From a480393bfda4d92d63c48909feec12bd7d274909 Mon Sep 17 00:00:00 2001 From: Till JS Date: Thu, 16 Apr 2026 16:25:17 +0200 Subject: [PATCH] =?UTF-8?q?fix(ai):=20P1=20batch=20=E2=80=94=20N+1=20queri?= =?UTF-8?q?es,=20vault-locked,=20debug=20hardening,=20timeout?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four P1 fixes from the AI Workbench audit: #3 N+1 junction queries → batch lookups: - TagLinkOps gains getTagIdsForMany(entityIds) — single where(field).anyOf(ids).toArray() instead of N calls. - filterBySceneScopeBatch() uses a pre-fetched Map. - All 4 module queries (notes, todo, contacts, calendar) migrated. - 500 notes now = 2 Dexie queries (records + junctions) instead of 501. #4 Vault-locked detection in readLocalNote: - Catches VaultLockedError from decryptRecords. - Throws descriptive "Vault ist gesperrt" instead of returning null. - Tools surface it as a clear error to the planner ("bitte Vault entsperren") instead of "Notiz nicht gefunden". #5 Debug log hardening: - Resolved-input content truncated to 500 chars before storage. - Time-based purge: entries older than 7 days auto-deleted. - Reduces privacy exposure if device is stolen/profile synced. #6 Timeout 90s → 180s: - 5 LLM calls on slow models (Ollama/GPU) regularly hit 90s. - 180s gives comfortable headroom for the reasoning loop. Audit doc updated with status markers. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../web/src/lib/data/ai/missions/debug.ts | 40 +++++++++++++++++-- .../web/src/lib/data/ai/missions/runner.ts | 6 ++- .../web/src/lib/modules/calendar/queries.ts | 12 +++--- .../web/src/lib/modules/contacts/queries.ts | 5 ++- .../apps/web/src/lib/modules/notes/queries.ts | 5 ++- .../apps/web/src/lib/modules/notes/tools.ts | 18 +++++++-- .../apps/web/src/lib/modules/todo/queries.ts | 19 ++++++--- .../web/src/lib/stores/scene-scope.svelte.ts | 27 +++++++++++++ packages/shared-stores/src/tag-links.ts | 21 ++++++++++ 9 files changed, 131 insertions(+), 22 deletions(-) diff --git a/apps/mana/apps/web/src/lib/data/ai/missions/debug.ts b/apps/mana/apps/web/src/lib/data/ai/missions/debug.ts index 3cc425878..bc02a1071 100644 --- a/apps/mana/apps/web/src/lib/data/ai/missions/debug.ts +++ b/apps/mana/apps/web/src/lib/data/ai/missions/debug.ts @@ -19,6 +19,12 @@ import type { ResolvedInput } from './planner/types'; const TABLE = '_aiDebugLog'; const STORAGE_KEY = 'mana.ai.debug'; const MAX_ENTRIES = 50; +/** Auto-purge entries older than this to limit exposure of decrypted + * content in the local-only table. */ +const MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000; // 7 days +/** Max chars per resolved-input content stored in debug. Longer content + * is truncated to reduce the privacy surface if the device is stolen. */ +const INPUT_CONTENT_LIMIT = 500; /** * Captured by `aiPlanTask` and passed back via the planner output so the @@ -83,11 +89,28 @@ export function setAiDebugEnabled(enabled: boolean): void { localStorage.setItem(STORAGE_KEY, enabled ? '1' : '0'); } -/** Persist one debug entry + trim oldest if over cap. Idempotent on - * iterationId — re-running an iteration overwrites the prior capture. */ +/** Truncate resolved-input content before persisting so the debug + * table doesn't store full decrypted note/kontext bodies at rest. */ +function sanitizeForStorage(entry: AiDebugEntry): AiDebugEntry { + return { + ...entry, + resolvedInputs: entry.resolvedInputs.map((inp) => ({ + ...inp, + content: + inp.content.length > INPUT_CONTENT_LIMIT + ? inp.content.slice(0, INPUT_CONTENT_LIMIT) + '\n… (truncated for privacy)' + : inp.content, + })), + }; +} + +/** Persist one debug entry + trim oldest if over cap + purge old entries. + * Idempotent on iterationId — re-running an iteration overwrites prior. */ export async function recordAiDebug(entry: AiDebugEntry): Promise { try { - await db.table(TABLE).put(entry); + await db.table(TABLE).put(sanitizeForStorage(entry)); + + // Count-based cap const total = await db.table(TABLE).count(); if (total > MAX_ENTRIES) { const overflow = total - MAX_ENTRIES; @@ -100,6 +123,17 @@ export async function recordAiDebug(entry: AiDebugEntry): Promise { await db.table(TABLE).bulkDelete(oldest); } } + + // Time-based purge: drop entries older than MAX_AGE_MS + const cutoff = new Date(Date.now() - MAX_AGE_MS).toISOString(); + const expired = await db + .table(TABLE) + .where('capturedAt') + .below(cutoff) + .primaryKeys(); + if (expired.length) { + await db.table(TABLE).bulkDelete(expired); + } } catch (err) { console.warn('[AiDebug] persist failed:', err); } diff --git a/apps/mana/apps/web/src/lib/data/ai/missions/runner.ts b/apps/mana/apps/web/src/lib/data/ai/missions/runner.ts index 676e3d28e..7689fe2cf 100644 --- a/apps/mana/apps/web/src/lib/data/ai/missions/runner.ts +++ b/apps/mana/apps/web/src/lib/data/ai/missions/runner.ts @@ -71,7 +71,11 @@ const KONTEXT_SINGLETON_ID = 'singleton'; * and finalises the iteration as failed. 90 s is comfortable for a * cloud-tier model but short enough that a wedged backend doesn't sit * in `running` indefinitely. */ -const ITERATION_TIMEOUT_MS = 90_000; +/** 180s gives the reasoning loop (up to 5 LLM calls) enough headroom + * even on slow models. Each call can take 10–30s on Ollama/GPU with + * network latency; the old 90s limit regularly timed out during the + * second loop round. */ +const ITERATION_TIMEOUT_MS = 180_000; class CancelledError extends Error { constructor(reason: string) { diff --git a/apps/mana/apps/web/src/lib/modules/calendar/queries.ts b/apps/mana/apps/web/src/lib/modules/calendar/queries.ts index cf47d5062..aced344e8 100644 --- a/apps/mana/apps/web/src/lib/modules/calendar/queries.ts +++ b/apps/mana/apps/web/src/lib/modules/calendar/queries.ts @@ -13,7 +13,7 @@ import { useLiveQueryWithDefault } from '@mana/local-store/svelte'; import { db } from '$lib/data/database'; import { decryptRecords } from '$lib/data/crypto'; -import { filterBySceneScope } from '$lib/stores/scene-scope.svelte'; +import { filterBySceneScopeBatch } from '$lib/stores/scene-scope.svelte'; import { eventTagOps } from './stores/tags.svelte'; import type { LocalCalendar, LocalEvent, Calendar, CalendarEvent } from './types'; import { timeBlockToCalendarEvent } from './types'; @@ -67,10 +67,12 @@ export function useAllCalendarItems() { eventsById.set(e.id, e); } - // Scene scope filter: only calendar-sourced blocks are tag-filterable. - const scopedBlocks = await filterBySceneScope(decryptedBlocks, async (b) => - b.sourceModule === 'calendar' && b.sourceId ? eventTagOps.getTagIds(b.sourceId) : [] - ); + // Scene scope filter: batch-fetch event tags for calendar-sourced blocks. + const calendarSourceIds = decryptedBlocks + .filter((b) => b.sourceModule === 'calendar' && b.sourceId) + .map((b) => b.sourceId); + const tagMap = await eventTagOps.getTagIdsForMany(calendarSourceIds); + const scopedBlocks = filterBySceneScopeBatch(decryptedBlocks, (b) => b.sourceId ?? '', tagMap); // Convert to CalendarEvent, joining event data for calendar blocks return scopedBlocks.map((block) => { diff --git a/apps/mana/apps/web/src/lib/modules/contacts/queries.ts b/apps/mana/apps/web/src/lib/modules/contacts/queries.ts index 68a6a6751..46edcda80 100644 --- a/apps/mana/apps/web/src/lib/modules/contacts/queries.ts +++ b/apps/mana/apps/web/src/lib/modules/contacts/queries.ts @@ -5,7 +5,7 @@ import { useLiveQueryWithDefault } from '@mana/local-store/svelte'; import { db } from '$lib/data/database'; import { decryptRecords } from '$lib/data/crypto'; -import { filterBySceneScope } from '$lib/stores/scene-scope.svelte'; +import { filterBySceneScopeBatch } from '$lib/stores/scene-scope.svelte'; import { contactTagOps } from './stores/tags.svelte'; import type { LocalContact, Contact, SortField, ContactFilter } from './types'; @@ -57,7 +57,8 @@ export function useAllContacts() { (c) => !c.deletedAt ); const decrypted = await decryptRecords('contacts', visible); - const scoped = await filterBySceneScope(decrypted, (c) => contactTagOps.getTagIds(c.id)); + const tagMap = await contactTagOps.getTagIdsForMany(decrypted.map((c) => c.id)); + const scoped = filterBySceneScopeBatch(decrypted, (c) => c.id, tagMap); return scoped.map(toContact); }, []); } diff --git a/apps/mana/apps/web/src/lib/modules/notes/queries.ts b/apps/mana/apps/web/src/lib/modules/notes/queries.ts index 17fb9cf75..228c5f371 100644 --- a/apps/mana/apps/web/src/lib/modules/notes/queries.ts +++ b/apps/mana/apps/web/src/lib/modules/notes/queries.ts @@ -17,7 +17,7 @@ import { useLiveQueryWithDefault } from '@mana/local-store/svelte'; import { db } from '$lib/data/database'; import { decryptRecords } from '$lib/data/crypto'; -import { filterBySceneScope } from '$lib/stores/scene-scope.svelte'; +import { filterBySceneScopeBatch } from '$lib/stores/scene-scope.svelte'; import { noteTagOps } from './stores/tags.svelte'; import type { LocalNote, Note } from './types'; @@ -51,7 +51,8 @@ export function useAllNotes() { // Locked vault returns the blobs untouched so the UI can render // a "🔒" placeholder where title/content would be. const decrypted = await decryptRecords('notes', visible); - const scoped = await filterBySceneScope(decrypted, (n) => noteTagOps.getTagIds(n.id)); + const tagMap = await noteTagOps.getTagIdsForMany(decrypted.map((n) => n.id)); + const scoped = filterBySceneScopeBatch(decrypted, (n) => n.id, tagMap); return scoped.map(toNote).sort((a, b) => { if (a.isPinned !== b.isPinned) return a.isPinned ? -1 : 1; return b.updatedAt.localeCompare(a.updatedAt); diff --git a/apps/mana/apps/web/src/lib/modules/notes/tools.ts b/apps/mana/apps/web/src/lib/modules/notes/tools.ts index bf6dcd24f..699829ca2 100644 --- a/apps/mana/apps/web/src/lib/modules/notes/tools.ts +++ b/apps/mana/apps/web/src/lib/modules/notes/tools.ts @@ -13,7 +13,7 @@ import type { ModuleTool } from '$lib/data/tools/types'; import { notesStore } from './stores/notes.svelte'; import { noteTagOps } from './stores/tags.svelte'; import { db } from '$lib/data/database'; -import { decryptRecords } from '$lib/data/crypto'; +import { decryptRecords, VaultLockedError } from '$lib/data/crypto'; import { filterByScope } from '$lib/data/ai/scope-context'; import type { LocalNote } from './types'; @@ -25,11 +25,23 @@ function excerptOf(content: string, max = 140): string { return flat.length <= max ? flat : flat.slice(0, max - 1) + '…'; } +/** Read + decrypt a single note. Throws a descriptive error when the + * vault is locked instead of returning null (which callers can't + * distinguish from "note doesn't exist"). */ async function readLocalNote(id: string): Promise { const local = await db.table('notes').get(id); if (!local || local.deletedAt) return null; - const [decrypted] = await decryptRecords('notes', [local]); - return decrypted ?? null; + try { + const [decrypted] = await decryptRecords('notes', [local]); + return decrypted ?? null; + } catch (err) { + if (err instanceof VaultLockedError) { + throw new Error( + `Vault ist gesperrt — Notiz ${id} kann nicht entschlüsselt werden. Bitte Vault entsperren.` + ); + } + throw err; + } } export const notesTools: ModuleTool[] = [ diff --git a/apps/mana/apps/web/src/lib/modules/todo/queries.ts b/apps/mana/apps/web/src/lib/modules/todo/queries.ts index 671515a63..16c8164c2 100644 --- a/apps/mana/apps/web/src/lib/modules/todo/queries.ts +++ b/apps/mana/apps/web/src/lib/modules/todo/queries.ts @@ -5,8 +5,7 @@ import { useLiveQueryWithDefault } from '@mana/local-store/svelte'; import { db } from '$lib/data/database'; import { decryptRecords } from '$lib/data/crypto'; -import { filterBySceneScope } from '$lib/stores/scene-scope.svelte'; -import { taskTagTable } from './collections'; +import { filterBySceneScopeBatch } from '$lib/stores/scene-scope.svelte'; import type { LocalTask, LocalBoardView, @@ -48,10 +47,18 @@ export function useAllTasks() { const locals = await db.table('tasks').orderBy('order').toArray(); const visible = locals.filter((t) => !t.deletedAt); const decrypted = await decryptRecords('tasks', visible); - const scoped = await filterBySceneScope(decrypted, async (t) => { - const links = await taskTagTable.where('taskId').equals(t.id).toArray(); - return links.filter((l) => !l.deletedAt).map((l) => l.tagId); - }); + // Batch tag lookup: 1 query instead of N + const ids = decrypted.map((t) => t.id); + const allLinks = + ids.length > 0 ? await db.table('taskLabels').where('taskId').anyOf(ids).toArray() : []; + const tagMap = new Map(); + for (const l of allLinks) { + if (l.deletedAt) continue; + const arr = tagMap.get(l.taskId as string); + if (arr) arr.push(l.tagId as string); + else tagMap.set(l.taskId as string, [l.tagId as string]); + } + const scoped = filterBySceneScopeBatch(decrypted, (t) => t.id, tagMap); return scoped.map(toTask); }, [] as Task[]); } diff --git a/apps/mana/apps/web/src/lib/stores/scene-scope.svelte.ts b/apps/mana/apps/web/src/lib/stores/scene-scope.svelte.ts index 7dfe943de..7fa93d754 100644 --- a/apps/mana/apps/web/src/lib/stores/scene-scope.svelte.ts +++ b/apps/mana/apps/web/src/lib/stores/scene-scope.svelte.ts @@ -35,6 +35,33 @@ export function getSceneScopeTagIds(): readonly string[] | undefined { * Synchronous if tagIds are already loaded; async variant for * junction-table lookups (same signature as the AI version). */ +/** + * Filter records by the active scene scope using a pre-fetched tag map. + * Pass the result of `tagOps.getTagIdsForMany(ids)` to avoid N+1 queries. + * + * @param getId - extract the record's ID (used as key into tagMap) + * @param tagMap - Map from getTagIdsForMany() + */ +export function filterBySceneScopeBatch( + records: T[], + getId: (record: T) => string, + tagMap: Map +): T[] { + const scope = _scopeTagIds; + if (!scope) return records; + + const scopeSet = new Set(scope); + return records.filter((r) => { + const tagIds = tagMap.get(getId(r)); + // Untagged records (not in map or empty) are globally visible + if (!tagIds || tagIds.length === 0) return true; + return tagIds.some((id) => scopeSet.has(id)); + }); +} + +/** + * Legacy per-record variant. Prefer `filterBySceneScopeBatch` for lists. + */ export async function filterBySceneScope( records: T[], getTagIdsForRecord: (record: T) => Promise diff --git a/packages/shared-stores/src/tag-links.ts b/packages/shared-stores/src/tag-links.ts index 9475a4ddb..22a611db0 100644 --- a/packages/shared-stores/src/tag-links.ts +++ b/packages/shared-stores/src/tag-links.ts @@ -42,6 +42,13 @@ export interface TagLinkOpsConfig { export interface TagLinkOps { /** Get all tag IDs linked to an entity */ getTagIds(entityId: string): Promise; + /** + * Batch variant: fetch tag IDs for many entities in ONE Dexie query. + * Returns a Map. Entities with no tags are absent + * from the map (not present with empty array). Use this in list views + * and scope filters to avoid N+1 queries. + */ + getTagIdsForMany(entityIds: string[]): Promise>; /** Add a tag to an entity (no-op if already linked) */ addTag(entityId: string, tagId: string): Promise; /** Remove a tag from an entity (soft-delete) */ @@ -66,6 +73,20 @@ export function createTagLinkOps(config: TagLinkOpsConfig): TagLinkOps { return links.map((l) => l.tagId); }, + async getTagIdsForMany(entityIds: string[]): Promise> { + if (entityIds.length === 0) return new Map(); + const all = await config.table().where(entityIdField).anyOf(entityIds).toArray(); + const active = all.filter((r) => !r.deletedAt); + const result = new Map(); + for (const link of active) { + const eid = link[entityIdField] as string; + const arr = result.get(eid); + if (arr) arr.push(link.tagId); + else result.set(eid, [link.tagId]); + } + return result; + }, + async addTag(entityId: string, tagId: string): Promise { const existing = await getActive(entityId); if (existing.some((l) => l.tagId === tagId)) return;