From 7bca16dfa7e85e18eeecc3397658ee85b4cbce8a Mon Sep 17 00:00:00 2001 From: Till JS Date: Tue, 28 Apr 2026 22:11:51 +0200 Subject: [PATCH] feat(articles): bulk-import schema + plan (Phase 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three new sync-tracked Dexie tables under the articles appId: articleImportJobs — job header (counters, status, lease metadata). articleImportItems — one row per URL in a job, state-machine driven. articleExtractPickup — short-lived server→client handoff inbox. URL stays plaintext on items by necessity — the server-worker reads it without master-key access, same rationale as articles.originalUrl. The extracted article eventually lands encrypted in the existing `articles` table; bulk-import rows hold only pointers. Plan: docs/plans/articles-bulk-import.md (full architecture, 7 phases, test matrix, edge-cases). Phase 2 already shipped in 5535f2da4 (worker); this commit lays the schema underneath it. Originally committed as b2f4e8314, lost during a parallel reset, here restored via cherry-pick. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../observatory/SeenplatteScene.svelte | 199 ------ .../observatory/atmosphere/Ambient.svelte | 126 ---- .../observatory/atmosphere/Sky.svelte | 83 --- .../lib/components/observatory/data/colors.ts | 84 --- .../lib/components/observatory/data/layout.ts | 171 ----- .../components/observatory/data/mockData.ts | 352 ----------- .../lib/components/observatory/data/types.ts | 69 --- .../observatory/plants/MossCluster.svelte | 63 -- .../observatory/plants/PlantFactory.svelte | 25 - .../observatory/plants/ReedPlant.svelte | 98 --- .../observatory/plants/Sprout.svelte | 81 --- .../observatory/plants/TreePlant.svelte | 117 ---- .../observatory/plants/WaterLily.svelte | 118 ---- .../observatory/terrain/Background.svelte | 67 -- .../observatory/terrain/Terrain.svelte | 59 -- .../observatory/terrain/WaterBody.svelte | 70 --- .../observatory/ui/CompareView.svelte | 359 ----------- .../observatory/ui/DetailPanel.svelte | 331 ---------- .../components/observatory/ui/LakeCard.svelte | 177 ------ .../observatory/ui/Leaderboard.svelte | 295 --------- .../observatory/ui/PlantCard.svelte | 185 ------ .../observatory/ui/PlantTooltip.svelte | 176 ------ .../observatory/ui/RadarChart.svelte | 120 ---- .../observatory/ui/RiverCard.svelte | 158 ----- .../observatory/ui/TrendsChart.svelte | 234 ------- .../observatory/water/RiverFlow.svelte | 69 --- .../lib/data/crypto/plaintext-allowlist.ts | 3 + apps/mana/apps/web/src/lib/data/database.ts | 44 +- .../src/lib/modules/articles/collections.ts | 13 +- .../src/lib/modules/articles/module.config.ts | 12 +- .../web/src/lib/modules/articles/types.ts | 140 ++++- .../{broadcast => broadcasts}/ListView.svelte | 0 .../modules/{broadcast => broadcasts}/api.ts | 0 .../audience/AudienceBuilder.svelte | 0 .../audience/segment-builder.test.ts | 0 .../audience/segment-builder.ts | 0 .../{broadcast => broadcasts}/collections.ts | 0 .../components/DnsCheckBanner.svelte | 0 .../components/SettingsForm.svelte | 0 .../{broadcast => broadcasts}/constants.ts | 0 .../editor/Editor.svelte | 0 .../{broadcast => broadcasts}/index.ts | 0 .../module.config.ts | 0 .../preview/EmailPreview.svelte | 0 .../preview/PreviewTabs.svelte | 0 .../{broadcast => broadcasts}/queries.ts | 0 .../render/email-html.test.ts | 0 .../render/email-html.ts | 0 .../render/plain-text.test.ts | 0 .../render/plain-text.ts | 0 .../stores/campaigns.svelte.ts | 0 .../stores/settings.svelte.ts | 0 .../{broadcast => broadcasts}/tools.ts | 0 .../{broadcast => broadcasts}/types.ts | 0 .../views/ComposeView.svelte | 0 .../views/DetailView.svelte | 0 .../widgets/BroadcastsWidget.svelte | 0 .../src/routes/(app)/observatory/+page.svelte | 351 ----------- docs/plans/articles-bulk-import.md | 585 ++++++++++++++++++ 59 files changed, 785 insertions(+), 4249 deletions(-) delete mode 100644 apps/mana/apps/web/src/lib/components/observatory/SeenplatteScene.svelte delete mode 100644 apps/mana/apps/web/src/lib/components/observatory/atmosphere/Ambient.svelte delete mode 100644 apps/mana/apps/web/src/lib/components/observatory/atmosphere/Sky.svelte delete mode 100644 apps/mana/apps/web/src/lib/components/observatory/data/colors.ts delete mode 100644 apps/mana/apps/web/src/lib/components/observatory/data/layout.ts delete mode 100644 apps/mana/apps/web/src/lib/components/observatory/data/mockData.ts delete mode 100644 apps/mana/apps/web/src/lib/components/observatory/data/types.ts delete mode 100644 apps/mana/apps/web/src/lib/components/observatory/plants/MossCluster.svelte delete mode 100644 apps/mana/apps/web/src/lib/components/observatory/plants/PlantFactory.svelte delete mode 100644 apps/mana/apps/web/src/lib/components/observatory/plants/ReedPlant.svelte delete mode 100644 apps/mana/apps/web/src/lib/components/observatory/plants/Sprout.svelte delete mode 100644 apps/mana/apps/web/src/lib/components/observatory/plants/TreePlant.svelte delete mode 100644 apps/mana/apps/web/src/lib/components/observatory/plants/WaterLily.svelte delete mode 100644 apps/mana/apps/web/src/lib/components/observatory/terrain/Background.svelte delete mode 100644 apps/mana/apps/web/src/lib/components/observatory/terrain/Terrain.svelte delete mode 100644 apps/mana/apps/web/src/lib/components/observatory/terrain/WaterBody.svelte delete mode 100644 apps/mana/apps/web/src/lib/components/observatory/ui/CompareView.svelte delete mode 100644 apps/mana/apps/web/src/lib/components/observatory/ui/DetailPanel.svelte delete mode 100644 apps/mana/apps/web/src/lib/components/observatory/ui/LakeCard.svelte delete mode 100644 apps/mana/apps/web/src/lib/components/observatory/ui/Leaderboard.svelte delete mode 100644 apps/mana/apps/web/src/lib/components/observatory/ui/PlantCard.svelte delete mode 100644 apps/mana/apps/web/src/lib/components/observatory/ui/PlantTooltip.svelte delete mode 100644 apps/mana/apps/web/src/lib/components/observatory/ui/RadarChart.svelte delete mode 100644 apps/mana/apps/web/src/lib/components/observatory/ui/RiverCard.svelte delete mode 100644 apps/mana/apps/web/src/lib/components/observatory/ui/TrendsChart.svelte delete mode 100644 apps/mana/apps/web/src/lib/components/observatory/water/RiverFlow.svelte rename apps/mana/apps/web/src/lib/modules/{broadcast => broadcasts}/ListView.svelte (100%) rename apps/mana/apps/web/src/lib/modules/{broadcast => broadcasts}/api.ts (100%) rename apps/mana/apps/web/src/lib/modules/{broadcast => broadcasts}/audience/AudienceBuilder.svelte (100%) rename apps/mana/apps/web/src/lib/modules/{broadcast => broadcasts}/audience/segment-builder.test.ts (100%) rename apps/mana/apps/web/src/lib/modules/{broadcast => broadcasts}/audience/segment-builder.ts (100%) rename apps/mana/apps/web/src/lib/modules/{broadcast => broadcasts}/collections.ts (100%) rename apps/mana/apps/web/src/lib/modules/{broadcast => broadcasts}/components/DnsCheckBanner.svelte (100%) rename apps/mana/apps/web/src/lib/modules/{broadcast => broadcasts}/components/SettingsForm.svelte (100%) rename apps/mana/apps/web/src/lib/modules/{broadcast => broadcasts}/constants.ts (100%) rename apps/mana/apps/web/src/lib/modules/{broadcast => broadcasts}/editor/Editor.svelte (100%) rename apps/mana/apps/web/src/lib/modules/{broadcast => broadcasts}/index.ts (100%) rename apps/mana/apps/web/src/lib/modules/{broadcast => broadcasts}/module.config.ts (100%) rename apps/mana/apps/web/src/lib/modules/{broadcast => broadcasts}/preview/EmailPreview.svelte (100%) rename apps/mana/apps/web/src/lib/modules/{broadcast => broadcasts}/preview/PreviewTabs.svelte (100%) rename apps/mana/apps/web/src/lib/modules/{broadcast => broadcasts}/queries.ts (100%) rename apps/mana/apps/web/src/lib/modules/{broadcast => broadcasts}/render/email-html.test.ts (100%) rename apps/mana/apps/web/src/lib/modules/{broadcast => broadcasts}/render/email-html.ts (100%) rename apps/mana/apps/web/src/lib/modules/{broadcast => broadcasts}/render/plain-text.test.ts (100%) rename apps/mana/apps/web/src/lib/modules/{broadcast => broadcasts}/render/plain-text.ts (100%) rename apps/mana/apps/web/src/lib/modules/{broadcast => broadcasts}/stores/campaigns.svelte.ts (100%) rename apps/mana/apps/web/src/lib/modules/{broadcast => broadcasts}/stores/settings.svelte.ts (100%) rename apps/mana/apps/web/src/lib/modules/{broadcast => broadcasts}/tools.ts (100%) rename apps/mana/apps/web/src/lib/modules/{broadcast => broadcasts}/types.ts (100%) rename apps/mana/apps/web/src/lib/modules/{broadcast => broadcasts}/views/ComposeView.svelte (100%) rename apps/mana/apps/web/src/lib/modules/{broadcast => broadcasts}/views/DetailView.svelte (100%) rename apps/mana/apps/web/src/lib/modules/{broadcast => broadcasts}/widgets/BroadcastsWidget.svelte (100%) delete mode 100644 apps/mana/apps/web/src/routes/(app)/observatory/+page.svelte create mode 100644 docs/plans/articles-bulk-import.md diff --git a/apps/mana/apps/web/src/lib/components/observatory/SeenplatteScene.svelte b/apps/mana/apps/web/src/lib/components/observatory/SeenplatteScene.svelte deleted file mode 100644 index 61d77dbfb..000000000 --- a/apps/mana/apps/web/src/lib/components/observatory/SeenplatteScene.svelte +++ /dev/null @@ -1,199 +0,0 @@ - - -
- - - - -
-

- Mana Seenplatte -

-

Ecosystem Observatory

-
- - -
- - - Mature - - - - Production - - - - Beta - - - - Alpha - -
- - - - - - - - - - - {#each RIVERS as river} - - {/each} - - - {#each LAKES as lake} - - {/each} - - - - - - - {#each apps.toSorted((a, b) => a.position.y - b.position.y) as app (app.id)} - handleAppHover(app, e)} - onmousemove={handleAppHoverMove} - onmouseleave={handleAppLeave} - > - handleAppClick(app)} /> - - {/each} - - - - {#if hoveredApp && !selectedApp} - - {/if} -
- - - diff --git a/apps/mana/apps/web/src/lib/components/observatory/atmosphere/Ambient.svelte b/apps/mana/apps/web/src/lib/components/observatory/atmosphere/Ambient.svelte deleted file mode 100644 index c8772fe9b..000000000 --- a/apps/mana/apps/web/src/lib/components/observatory/atmosphere/Ambient.svelte +++ /dev/null @@ -1,126 +0,0 @@ - - - -{#if !isNight} - {#each birds as bird} - - - - - - - - {/each} - - - {#each dragonflies as df} - - - - - - - {/each} -{/if} - - -{#if isNight} - {#each fireflies as ff} - - - - - - {/each} -{/if} - - -{#if !isNight} - - - - - - - - - - -{/if} diff --git a/apps/mana/apps/web/src/lib/components/observatory/atmosphere/Sky.svelte b/apps/mana/apps/web/src/lib/components/observatory/atmosphere/Sky.svelte deleted file mode 100644 index 545aeea90..000000000 --- a/apps/mana/apps/web/src/lib/components/observatory/atmosphere/Sky.svelte +++ /dev/null @@ -1,83 +0,0 @@ - - - - - - - - - - - - -{#if isNight} - - {#each Array(35) as _, i} - {@const sx = (i * 137 + 42) % SCENE.width} - {@const sy = (i * 89 + 17) % 250} - {@const sr = 0.5 + (i % 3) * 0.4} - - - - {/each} - -{/if} - - -{#if isDusk} - - - - -{/if} diff --git a/apps/mana/apps/web/src/lib/components/observatory/data/colors.ts b/apps/mana/apps/web/src/lib/components/observatory/data/colors.ts deleted file mode 100644 index 74856010b..000000000 --- a/apps/mana/apps/web/src/lib/components/observatory/data/colors.ts +++ /dev/null @@ -1,84 +0,0 @@ -// Natural color palette for the Seenplatte ecosystem - -export const sky = { - dayTop: '#87CEEB', - dayBottom: '#E0F0FF', - duskTop: '#2C1654', - duskBottom: '#E8956A', - nightTop: '#0B1026', - nightBottom: '#1A2444', -} as const; - -export const mountains = { - far: '#8BA4B8', - mid: '#6B8A9E', - near: '#4A7085', - snow: '#E8F0F4', -} as const; - -export const water = { - shallow: '#7EC8D9', - mid: '#4BA3B5', - deep: '#2A7A8C', - veryDeep: '#1A5666', - river: '#5BB5C5', - highlight: '#A8E4F0', - foam: '#E8F6FA', -} as const; - -export const terrain = { - meadow: '#7DB86A', - meadowLight: '#96CC86', - meadowDark: '#5E9A4D', - shore: '#C4B48A', - shoreDark: '#A89870', - path: '#D4C4A0', - rock: '#8C8C80', - rockLight: '#A8A89C', -} as const; - -export const vegetation = { - // Tree health colors based on ManaScore - healthyDark: '#2D6B30', - healthy: '#3E8B42', - healthyLight: '#5AAB5E', - moderate: '#7DB86A', - warning: '#C4B848', - stressed: '#D4944A', - critical: '#B85C4A', - - // Specific plant colors - trunk: '#6B4E3D', - trunkLight: '#8B6E5D', - reed: '#7DA868', - reedDark: '#5A8548', - lilyPad: '#4A8B50', - lilyFlower: '#E8B0D0', - lilyFlowerCenter: '#F0D060', - moss: '#5A8B4A', - mossLight: '#78A868', - sprout: '#90C880', - sproutStake: '#A89070', -} as const; - -/** - * Get tree crown color based on ManaScore (0-100) - */ -export function getHealthColor(score: number): string { - if (score >= 85) return vegetation.healthyDark; - if (score >= 70) return vegetation.healthy; - if (score >= 55) return vegetation.moderate; - if (score >= 40) return vegetation.warning; - if (score >= 25) return vegetation.stressed; - return vegetation.critical; -} - -/** - * Get water clarity based on error rate (0 = clear, 1 = murky) - */ -export function getWaterColor(clarity: number): string { - if (clarity > 0.8) return water.shallow; - if (clarity > 0.5) return water.mid; - if (clarity > 0.2) return water.deep; - return water.veryDeep; -} diff --git a/apps/mana/apps/web/src/lib/components/observatory/data/layout.ts b/apps/mana/apps/web/src/lib/components/observatory/data/layout.ts deleted file mode 100644 index 2fa98cb41..000000000 --- a/apps/mana/apps/web/src/lib/components/observatory/data/layout.ts +++ /dev/null @@ -1,171 +0,0 @@ -import type { LakeData, RiverData } from './types'; - -// Scene dimensions (SVG viewBox) -export const SCENE = { - width: 1600, - height: 900, - viewBox: '0 0 1600 900', -} as const; - -// Mountain layer paths (3 layers, back to front) -export const MOUNTAIN_PATHS = { - far: 'M0,280 Q200,180 400,240 Q500,210 600,250 Q750,160 900,220 Q1050,180 1200,230 Q1350,170 1500,210 L1600,240 L1600,350 L0,350 Z', - mid: 'M0,310 Q150,250 300,290 Q450,240 550,280 Q700,220 850,270 Q1000,230 1150,275 Q1300,240 1450,265 L1600,280 L1600,380 L0,380 Z', - near: 'M0,340 Q100,300 250,330 Q400,290 500,320 Q650,280 800,315 Q950,285 1100,320 Q1250,295 1400,310 L1600,320 L1600,400 L0,400 Z', -} as const; - -// Lake definitions with organic SVG paths -export const LAKES: LakeData[] = [ - { - id: 'auth', - name: 'auth', - label: 'Zentralsee', - path: 'M680,440 Q720,410 790,415 Q860,420 890,445 Q910,470 900,500 Q885,535 850,550 Q800,565 750,558 Q700,550 680,525 Q660,500 665,470 Q668,450 680,440 Z', - color: '#4BA3B5', - colorDeep: '#2A7A8C', - clarity: 1, - level: 0.8, - position: { x: 790, y: 485 }, - }, - { - id: 'redis', - name: 'redis', - label: 'Bergsee', - path: 'M280,390 Q310,370 350,375 Q390,380 405,400 Q415,420 405,440 Q390,455 355,460 Q320,462 295,450 Q275,435 270,415 Q268,398 280,390 Z', - color: '#7EC8D9', - colorDeep: '#4BA3B5', - clarity: 1, - level: 0.9, - position: { x: 340, y: 420 }, - }, - { - id: 'minio', - name: 'minio', - label: 'Stausee', - path: 'M1180,400 Q1220,375 1280,380 Q1340,385 1370,410 Q1390,435 1380,465 Q1360,490 1320,500 Q1270,510 1220,502 Q1185,492 1170,468 Q1158,445 1165,420 Q1170,405 1180,400 Z', - color: '#5BB5C5', - colorDeep: '#3A8A9A', - clarity: 0.9, - level: 0.7, - position: { x: 1275, y: 440 }, - }, - { - id: 'db-left', - name: 'postgres-left', - label: 'Waldsee West', - path: 'M180,580 Q220,555 280,560 Q340,565 365,590 Q380,615 370,640 Q355,665 310,675 Q260,682 215,670 Q180,658 165,635 Q155,612 165,595 Q170,582 180,580 Z', - color: '#3A8A9A', - colorDeep: '#1A5666', - clarity: 0.85, - level: 0.75, - position: { x: 270, y: 620 }, - }, - { - id: 'db-center', - name: 'postgres-center', - label: 'Waldsee Mitte', - path: 'M650,620 Q700,595 770,600 Q840,605 870,635 Q890,660 878,690 Q862,718 810,730 Q760,738 710,730 Q660,720 640,695 Q625,670 632,645 Q638,628 650,620 Z', - color: '#3A8A9A', - colorDeep: '#1A5666', - clarity: 0.9, - level: 0.8, - position: { x: 760, y: 665 }, - }, - { - id: 'db-right', - name: 'postgres-right', - label: 'Waldsee Ost', - path: 'M1120,590 Q1160,568 1220,572 Q1280,578 1305,600 Q1322,625 1315,652 Q1300,678 1255,690 Q1210,698 1165,688 Q1130,678 1115,655 Q1102,632 1108,610 Q1112,595 1120,590 Z', - color: '#3A8A9A', - colorDeep: '#1A5666', - clarity: 0.85, - level: 0.75, - position: { x: 1215, y: 630 }, - }, -]; - -// Rivers connecting the lakes -export const RIVERS: RiverData[] = [ - { - id: 'redis-to-auth', - from: 'redis', - to: 'auth', - path: 'M405,430 Q480,435 540,440 Q600,445 665,460', - flowSpeed: 0.8, - width: 8, - }, - { - id: 'minio-to-auth', - from: 'minio', - to: 'auth', - path: 'M1170,450 Q1100,455 1020,460 Q940,465 900,475', - flowSpeed: 0.6, - width: 8, - }, - { - id: 'auth-to-db-left', - from: 'auth', - to: 'db-left', - path: 'M710,555 Q620,570 520,580 Q420,590 365,595', - flowSpeed: 0.7, - width: 10, - }, - { - id: 'auth-to-db-center', - from: 'auth', - to: 'db-center', - path: 'M790,560 Q785,580 778,600 Q770,610 765,620', - flowSpeed: 0.7, - width: 10, - }, - { - id: 'auth-to-db-right', - from: 'auth', - to: 'db-right', - path: 'M870,548 Q960,565 1040,575 Q1080,582 1115,592', - flowSpeed: 0.7, - width: 10, - }, - { - id: 'inlet', - from: 'source', - to: 'auth', - path: 'M790,350 Q792,370 790,390 Q788,410 790,420', - flowSpeed: 0.9, - width: 6, - }, -]; - -// App positions around the lakes -export const APP_POSITIONS: Record = { - // Around Zentralsee (auth) - core/mature apps - mana: { x: 730, y: 410, lakeId: 'auth' }, - chat: { x: 860, y: 420, lakeId: 'auth' }, - picture: { x: 910, y: 480, lakeId: 'auth' }, - presi: { x: 660, y: 490, lakeId: 'auth' }, - - // Around Waldsee West (db-left) - calendar: { x: 170, y: 560, lakeId: 'db-left' }, - todo: { x: 330, y: 555, lakeId: 'db-left' }, - contacts: { x: 370, y: 630, lakeId: 'db-left' }, - storage: { x: 160, y: 650, lakeId: 'db-left' }, - - // Around Waldsee Mitte (db-center) - quotes: { x: 640, y: 600, lakeId: 'db-center' }, - music: { x: 850, y: 610, lakeId: 'db-center' }, - clock: { x: 880, y: 680, lakeId: 'db-center' }, - food: { x: 650, y: 720, lakeId: 'db-center' }, - - // Around Waldsee Ost (db-right) - photos: { x: 1110, y: 575, lakeId: 'db-right' }, - skilltree: { x: 1310, y: 590, lakeId: 'db-right' }, - context: { x: 1320, y: 660, lakeId: 'db-right' }, - plants: { x: 1115, y: 675, lakeId: 'db-right' }, - - // Around Bergsee (redis) - lightweight/cache - traces: { x: 400, y: 385, lakeId: 'redis' }, - - // Around Stausee (minio) - storage-heavy - cards: { x: 1180, y: 385, lakeId: 'minio' }, - questions: { x: 1370, y: 400, lakeId: 'minio' }, -}; diff --git a/apps/mana/apps/web/src/lib/components/observatory/data/mockData.ts b/apps/mana/apps/web/src/lib/components/observatory/data/mockData.ts deleted file mode 100644 index 2a283c87e..000000000 --- a/apps/mana/apps/web/src/lib/components/observatory/data/mockData.ts +++ /dev/null @@ -1,352 +0,0 @@ -import type { AppData, AppStatus, PlantType, CategoryScores } from './types'; -import { APP_POSITIONS } from './layout'; - -interface AppDefinition { - id: string; - displayName: string; - score: number; - status: AppStatus; - categories: CategoryScores; - previousScore?: number; -} - -function getPlantType(status: AppStatus, score: number): PlantType { - if (score >= 85) return 'oak'; - if (score >= 70) return 'birch'; - if (status === 'beta' && score >= 55) return 'youngTree'; - if (status === 'beta') return 'reed'; - if (status === 'alpha') return 'sprout'; - if (status === 'prototype') return 'sprout'; - return 'youngTree'; -} - -// Real ManaScore data from 2026-03-24 audits -const APP_DEFINITIONS: AppDefinition[] = [ - { - id: 'calendar', - displayName: 'Calendar', - score: 97, - previousScore: 82, - status: 'mature', - categories: { - backend: 95, - frontend: 96, - database: 92, - testing: 90, - deployment: 92, - documentation: 98, - security: 92, - ux: 95, - }, - }, - { - id: 'todo', - displayName: 'Todo', - score: 96, - previousScore: 80, - status: 'mature', - categories: { - backend: 94, - frontend: 95, - database: 88, - testing: 90, - deployment: 92, - documentation: 95, - security: 90, - ux: 94, - }, - }, - { - id: 'contacts', - displayName: 'Contacts', - score: 94, - status: 'production', - categories: { - backend: 92, - frontend: 90, - database: 88, - testing: 88, - deployment: 90, - documentation: 92, - security: 85, - ux: 85, - }, - }, - { - id: 'mana', - displayName: 'Mana', - score: 88, - status: 'production', - categories: { - backend: 55, - frontend: 90, - database: 70, - testing: 72, - deployment: 90, - documentation: 88, - security: 80, - ux: 92, - }, - }, - { - id: 'presi', - displayName: 'Presi', - score: 86, - status: 'mature', - categories: { - backend: 90, - frontend: 82, - database: 85, - testing: 82, - deployment: 75, - documentation: 90, - security: 85, - ux: 82, - }, - }, - { - id: 'storage', - displayName: 'Storage', - score: 84, - previousScore: 55, - status: 'production', - categories: { - backend: 88, - frontend: 84, - database: 82, - testing: 78, - deployment: 65, - documentation: 78, - security: 78, - ux: 75, - }, - }, - { - id: 'chat', - displayName: 'Chat', - score: 82, - status: 'production', - categories: { - backend: 90, - frontend: 82, - database: 95, - testing: 60, - deployment: 92, - documentation: 85, - security: 82, - ux: 80, - }, - }, - { - id: 'picture', - displayName: 'Picture', - score: 81, - status: 'production', - categories: { - backend: 90, - frontend: 80, - database: 92, - testing: 55, - deployment: 75, - documentation: 78, - security: 80, - ux: 78, - }, - }, - { - id: 'music', - displayName: 'Music', - score: 80, - status: 'beta', - categories: { - backend: 90, - frontend: 78, - database: 90, - testing: 65, - deployment: 85, - documentation: 80, - security: 78, - ux: 60, - }, - }, - { - id: 'food', - displayName: 'Food', - score: 63, - status: 'beta', - categories: { - backend: 78, - frontend: 62, - database: 80, - testing: 58, - deployment: 40, - documentation: 85, - security: 68, - ux: 55, - }, - }, - { - id: 'photos', - displayName: 'Photos', - score: 62, - status: 'beta', - categories: { - backend: 82, - frontend: 65, - database: 72, - testing: 0, - deployment: 85, - documentation: 78, - security: 65, - ux: 55, - }, - }, - { - id: 'quotes', - displayName: 'Quotes', - score: 62, - status: 'beta', - categories: { - backend: 72, - frontend: 78, - database: 75, - testing: 0, - deployment: 92, - documentation: 20, - security: 70, - ux: 75, - }, - }, - { - id: 'context', - displayName: 'Context', - score: 60, - status: 'beta', - categories: { - backend: 75, - frontend: 75, - database: 82, - testing: 55, - deployment: 25, - documentation: 85, - security: 68, - ux: 65, - }, - }, - { - id: 'clock', - displayName: 'Clock', - score: 58, - status: 'beta', - categories: { - backend: 75, - frontend: 70, - database: 72, - testing: 0, - deployment: 88, - documentation: 10, - security: 60, - ux: 55, - }, - }, - { - id: 'skilltree', - displayName: 'SkillTree', - score: 58, - status: 'beta', - categories: { - backend: 65, - frontend: 68, - database: 72, - testing: 28, - deployment: 55, - documentation: 62, - security: 65, - ux: 72, - }, - }, - { - id: 'plants', - displayName: 'Plants', - score: 50, - status: 'alpha', - categories: { - backend: 68, - frontend: 58, - database: 70, - testing: 0, - deployment: 45, - documentation: 62, - security: 55, - ux: 50, - }, - }, - { - id: 'cards', - displayName: 'Cards', - score: 48, - status: 'alpha', - categories: { - backend: 50, - frontend: 65, - database: 30, - testing: 18, - deployment: 80, - documentation: 25, - security: 55, - ux: 68, - }, - }, - { - id: 'questions', - displayName: 'Questions', - score: 48, - status: 'alpha', - categories: { - backend: 88, - frontend: 62, - database: 78, - testing: 0, - deployment: 10, - documentation: 72, - security: 55, - ux: 55, - }, - }, - { - id: 'traces', - displayName: 'Traces', - score: 35, - status: 'alpha', - categories: { - backend: 72, - frontend: 10, - database: 70, - testing: 0, - deployment: 8, - documentation: 45, - security: 55, - ux: 35, - }, - }, -]; - -export function createMockEcosystem(): AppData[] { - return APP_DEFINITIONS.map((def) => { - const pos = APP_POSITIONS[def.id] || { x: 800, y: 500, lakeId: 'auth' }; - const trend = def.previousScore ? def.score - def.previousScore : 0; - return { - id: def.id, - name: def.id, - displayName: def.displayName, - score: def.score, - status: def.status, - health: 'up' as const, - plantType: getPlantType(def.status, def.score), - categories: def.categories, - trend, - lakeId: pos.lakeId, - position: { x: pos.x, y: pos.y }, - }; - }); -} diff --git a/apps/mana/apps/web/src/lib/components/observatory/data/types.ts b/apps/mana/apps/web/src/lib/components/observatory/data/types.ts deleted file mode 100644 index 51d495acc..000000000 --- a/apps/mana/apps/web/src/lib/components/observatory/data/types.ts +++ /dev/null @@ -1,69 +0,0 @@ -export type PlantType = - | 'oak' - | 'birch' - | 'youngTree' - | 'reed' - | 'waterLily' - | 'moss' - | 'shrub' - | 'sprout' - | 'stump' - | 'swampCluster'; - -export type AppStatus = 'prototype' | 'alpha' | 'beta' | 'production' | 'mature'; - -export type HealthStatus = 'up' | 'degraded' | 'down' | 'unknown'; - -export interface CategoryScores { - backend: number; - frontend: number; - database: number; - testing: number; - deployment: number; - documentation: number; - security: number; - ux: number; -} - -export interface AppData { - id: string; - name: string; - displayName: string; - score: number; - status: AppStatus; - health: HealthStatus; - plantType: PlantType; - categories: CategoryScores; - trend: number; - lakeId: string; - position: { x: number; y: number }; -} - -export interface LakeData { - id: string; - name: string; - label: string; - path: string; - color: string; - colorDeep: string; - clarity: number; // 0-1, 1 = crystal clear - level: number; // 0-1, normalized fill level - position: { x: number; y: number }; -} - -export interface RiverData { - id: string; - from: string; - to: string; - path: string; - flowSpeed: number; // 0-1 - width: number; -} - -export interface EcosystemState { - apps: AppData[]; - lakes: LakeData[]; - rivers: RiverData[]; - timeOfDay: number; // 0-24 - systemHealth: 'sunny' | 'cloudy' | 'stormy'; -} diff --git a/apps/mana/apps/web/src/lib/components/observatory/plants/MossCluster.svelte b/apps/mana/apps/web/src/lib/components/observatory/plants/MossCluster.svelte deleted file mode 100644 index 724fdae38..000000000 --- a/apps/mana/apps/web/src/lib/components/observatory/plants/MossCluster.svelte +++ /dev/null @@ -1,63 +0,0 @@ - - - e.key === 'Enter' && onclick?.()} -> - {#each patches as patch} - - {/each} - - - - {app.displayName} - - diff --git a/apps/mana/apps/web/src/lib/components/observatory/plants/PlantFactory.svelte b/apps/mana/apps/web/src/lib/components/observatory/plants/PlantFactory.svelte deleted file mode 100644 index 74fd31aef..000000000 --- a/apps/mana/apps/web/src/lib/components/observatory/plants/PlantFactory.svelte +++ /dev/null @@ -1,25 +0,0 @@ - - -{#if app.plantType === 'oak' || app.plantType === 'birch' || app.plantType === 'youngTree'} - -{:else if app.plantType === 'reed'} - -{:else if app.plantType === 'waterLily'} - -{:else if app.plantType === 'sprout'} - -{:else if app.plantType === 'moss'} - -{:else} - - -{/if} diff --git a/apps/mana/apps/web/src/lib/components/observatory/plants/ReedPlant.svelte b/apps/mana/apps/web/src/lib/components/observatory/plants/ReedPlant.svelte deleted file mode 100644 index ccdfa9dfa..000000000 --- a/apps/mana/apps/web/src/lib/components/observatory/plants/ReedPlant.svelte +++ /dev/null @@ -1,98 +0,0 @@ - - - e.key === 'Enter' && onclick?.()} -> - - - - {#each stalkData as stalk} - - - - - {#if stalk.hasBulrush} - - {/if} - {/each} - - - - - - - - - {app.displayName} - - diff --git a/apps/mana/apps/web/src/lib/components/observatory/plants/Sprout.svelte b/apps/mana/apps/web/src/lib/components/observatory/plants/Sprout.svelte deleted file mode 100644 index 7d6872a75..000000000 --- a/apps/mana/apps/web/src/lib/components/observatory/plants/Sprout.svelte +++ /dev/null @@ -1,81 +0,0 @@ - - - e.key === 'Enter' && onclick?.()} -> - - - - - - - - - - - - - - - - - - - - {app.displayName} - - diff --git a/apps/mana/apps/web/src/lib/components/observatory/plants/TreePlant.svelte b/apps/mana/apps/web/src/lib/components/observatory/plants/TreePlant.svelte deleted file mode 100644 index 67f108a5f..000000000 --- a/apps/mana/apps/web/src/lib/components/observatory/plants/TreePlant.svelte +++ /dev/null @@ -1,117 +0,0 @@ - - - e.key === 'Enter' && onclick?.()} -> - - - - - - - - - - {#if isOak} - - - - - {/if} - - - - {#each blobs as blob, i} - {@const lightness = i % 3 === 0 ? 15 : i % 3 === 1 ? 0 : -10} - - {/each} - - - - - - - - - {app.displayName} - - - {app.score} - - diff --git a/apps/mana/apps/web/src/lib/components/observatory/plants/WaterLily.svelte b/apps/mana/apps/web/src/lib/components/observatory/plants/WaterLily.svelte deleted file mode 100644 index 6d74057ef..000000000 --- a/apps/mana/apps/web/src/lib/components/observatory/plants/WaterLily.svelte +++ /dev/null @@ -1,118 +0,0 @@ - - - e.key === 'Enter' && onclick?.()} -> - - - - - - - - - - - - - - - {#if app.score > 20} - - - {#each petalData as petal} - {@const px = Math.cos(petal.angle) * 5 * bloomAmount} - {@const py = Math.sin(petal.angle) * 3 * bloomAmount} - - {/each} - - - - - {/if} - - - - - {app.displayName} - - diff --git a/apps/mana/apps/web/src/lib/components/observatory/terrain/Background.svelte b/apps/mana/apps/web/src/lib/components/observatory/terrain/Background.svelte deleted file mode 100644 index 45c4f2167..000000000 --- a/apps/mana/apps/web/src/lib/components/observatory/terrain/Background.svelte +++ /dev/null @@ -1,67 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - {#each Array(60) as _, i} - {@const x = i * 28 + 10} - {@const baseY = 335 + Math.sin(i * 0.8) * 15} - {@const h = 12 + Math.sin(i * 1.3) * 5} - - {/each} - diff --git a/apps/mana/apps/web/src/lib/components/observatory/terrain/Terrain.svelte b/apps/mana/apps/web/src/lib/components/observatory/terrain/Terrain.svelte deleted file mode 100644 index 9438136d0..000000000 --- a/apps/mana/apps/web/src/lib/components/observatory/terrain/Terrain.svelte +++ /dev/null @@ -1,59 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - {#each Array(40) as _, i} - {@const x = i * 42 + 15} - {@const y = 820 + Math.sin(i * 2.1) * 20} - - {/each} - diff --git a/apps/mana/apps/web/src/lib/components/observatory/terrain/WaterBody.svelte b/apps/mana/apps/web/src/lib/components/observatory/terrain/WaterBody.svelte deleted file mode 100644 index ac7361670..000000000 --- a/apps/mana/apps/web/src/lib/components/observatory/terrain/WaterBody.svelte +++ /dev/null @@ -1,70 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - {lake.label} - - diff --git a/apps/mana/apps/web/src/lib/components/observatory/ui/CompareView.svelte b/apps/mana/apps/web/src/lib/components/observatory/ui/CompareView.svelte deleted file mode 100644 index 476e445e3..000000000 --- a/apps/mana/apps/web/src/lib/components/observatory/ui/CompareView.svelte +++ /dev/null @@ -1,359 +0,0 @@ - - -
- -
-

Apps auswahlen (max. 4):

-
- {#each apps as app (app.id)} - - {/each} -
-
- - {#if selectedApps.length >= 2} -
- -
- - {#each rings as ring} - - {/each} - {#each axes as axis} - - {/each} - - {#each selectedApps as app, i (app.id)} - - {/each} - - {#each labelPositions as lp} - - {lp.label} - - {/each} - - - -
- {#each selectedApps as app, i (app.id)} - - - {app.displayName} ({app.score}) - - {/each} -
-
- - -
- {#each categoryKeys as cat} -
- {cat} -
- {#each selectedApps as app, i (app.id)} -
-
-
-
- {app.categories[cat]} -
- {/each} -
-
- {/each} -
-
- {:else} -
-

Wahle mindestens 2 Apps zum Vergleichen

-
- {/if} -
- - diff --git a/apps/mana/apps/web/src/lib/components/observatory/ui/DetailPanel.svelte b/apps/mana/apps/web/src/lib/components/observatory/ui/DetailPanel.svelte deleted file mode 100644 index 152d5a8de..000000000 --- a/apps/mana/apps/web/src/lib/components/observatory/ui/DetailPanel.svelte +++ /dev/null @@ -1,331 +0,0 @@ - - - - -{#if app} - - - -
- -
-
-

{app.displayName}

- - {statusLabels[app.status]} - -
-
- {app.score} -
-
- - -
-
-
- - -
- -
- -
-
- - -
-
- {#each Object.entries(app.categories) as [key, value]} -
- {key} -
-
-
- {value} -
- {/each} -
-
- - -
- -
- - - -
-{/if} - - diff --git a/apps/mana/apps/web/src/lib/components/observatory/ui/LakeCard.svelte b/apps/mana/apps/web/src/lib/components/observatory/ui/LakeCard.svelte deleted file mode 100644 index e9e305cb4..000000000 --- a/apps/mana/apps/web/src/lib/components/observatory/ui/LakeCard.svelte +++ /dev/null @@ -1,177 +0,0 @@ - - -
- -
- - - - - - - - - - - - - - - -
- - -
-

{lake.label}

- {lakeIcons[lake.id] || lake.name} -

{lakeDescriptions[lake.id] || ''}

-
- - Klarheit - {Math.round(lake.clarity * 100)}% - - - Fullstand - {Math.round(lake.level * 100)}% - -
-
-
- - diff --git a/apps/mana/apps/web/src/lib/components/observatory/ui/Leaderboard.svelte b/apps/mana/apps/web/src/lib/components/observatory/ui/Leaderboard.svelte deleted file mode 100644 index e879e2c3f..000000000 --- a/apps/mana/apps/web/src/lib/components/observatory/ui/Leaderboard.svelte +++ /dev/null @@ -1,295 +0,0 @@ - - -
-
- - - - - - - - {#each categoryKeys as cat} - - {/each} - - - - {#each sorted() as app, i (app.id)} - onselect(app)}> - - - - - {#each categoryKeys as cat} - - {/each} - - {/each} - -
# toggleSort('name')}> - App{sortIndicator('name')} - toggleSort('score')}> - Score{sortIndicator('score')} - toggleSort('trend')}> - Trend{sortIndicator('trend')} - toggleSort(cat)}> - {shortLabels[cat]}{sortIndicator(cat)} -
{i + 1} - - {app.displayName} - - {app.score} - - {#if app.trend > 0} - +{app.trend} - {:else if app.trend < 0} - {app.trend} - {:else} - - - {/if} - -
-
-
- {app.categories[cat]} -
-
-
- - diff --git a/apps/mana/apps/web/src/lib/components/observatory/ui/PlantCard.svelte b/apps/mana/apps/web/src/lib/components/observatory/ui/PlantCard.svelte deleted file mode 100644 index 37bb8be1c..000000000 --- a/apps/mana/apps/web/src/lib/components/observatory/ui/PlantCard.svelte +++ /dev/null @@ -1,185 +0,0 @@ - - - - - diff --git a/apps/mana/apps/web/src/lib/components/observatory/ui/PlantTooltip.svelte b/apps/mana/apps/web/src/lib/components/observatory/ui/PlantTooltip.svelte deleted file mode 100644 index b3e8c91af..000000000 --- a/apps/mana/apps/web/src/lib/components/observatory/ui/PlantTooltip.svelte +++ /dev/null @@ -1,176 +0,0 @@ - - -
-
- {app.displayName} - {app.score} -
-
- - {statusLabels[app.status]} - - - {healthIcons[app.health]} - - {#if app.trend !== 0} - 0} class:negative={app.trend < 0}> - {app.trend > 0 ? '+' : ''}{app.trend} - - {/if} -
-
- {#each Object.entries(app.categories) as [key, value]} -
- {key} -
-
-
- {value} -
- {/each} -
-
- - diff --git a/apps/mana/apps/web/src/lib/components/observatory/ui/RadarChart.svelte b/apps/mana/apps/web/src/lib/components/observatory/ui/RadarChart.svelte deleted file mode 100644 index 725377545..000000000 --- a/apps/mana/apps/web/src/lib/components/observatory/ui/RadarChart.svelte +++ /dev/null @@ -1,120 +0,0 @@ - - - - - {#each rings as ring} - - {/each} - - - {#each axes as axis} - - {/each} - - - - - - {#each dataPoints as point} - - {/each} - - - {#each labelPositions as lp} - - {lp.label} - - {/each} - diff --git a/apps/mana/apps/web/src/lib/components/observatory/ui/RiverCard.svelte b/apps/mana/apps/web/src/lib/components/observatory/ui/RiverCard.svelte deleted file mode 100644 index ddbf16770..000000000 --- a/apps/mana/apps/web/src/lib/components/observatory/ui/RiverCard.svelte +++ /dev/null @@ -1,158 +0,0 @@ - - -
- -
- - - - - - - - - - - - - -
- -
-
- {fromLabel} - - {toLabel} -
-
- - Geschwindigkeit - {speedLabel} - - - Breite - {river.width}px - -
-
-
- - diff --git a/apps/mana/apps/web/src/lib/components/observatory/ui/TrendsChart.svelte b/apps/mana/apps/web/src/lib/components/observatory/ui/TrendsChart.svelte deleted file mode 100644 index dd589c204..000000000 --- a/apps/mana/apps/web/src/lib/components/observatory/ui/TrendsChart.svelte +++ /dev/null @@ -1,234 +0,0 @@ - - - - - diff --git a/apps/mana/apps/web/src/lib/components/observatory/water/RiverFlow.svelte b/apps/mana/apps/web/src/lib/components/observatory/water/RiverFlow.svelte deleted file mode 100644 index a1a3f3ec4..000000000 --- a/apps/mana/apps/web/src/lib/components/observatory/water/RiverFlow.svelte +++ /dev/null @@ -1,69 +0,0 @@ - - - - - - - - - - - - - - - - - - - diff --git a/apps/mana/apps/web/src/lib/data/crypto/plaintext-allowlist.ts b/apps/mana/apps/web/src/lib/data/crypto/plaintext-allowlist.ts index 4ec55c8da..b9f7b0d8e 100644 --- a/apps/mana/apps/web/src/lib/data/crypto/plaintext-allowlist.ts +++ b/apps/mana/apps/web/src/lib/data/crypto/plaintext-allowlist.ts @@ -22,6 +22,9 @@ export const PLAINTEXT_ALLOWLIST: readonly string[] = [ 'albumItems', // TODO: audit 'albums', // TODO: audit 'articleTags', // FK-only junction into globalTags (articleId, tagId). Tag names live in globalTags. + 'articleImportJobs', // Bulk-import job header (counters, status, lease metadata). Pure operational state, no user-typed content. See docs/plans/articles-bulk-import.md. + 'articleImportItems', // One row per URL in a bulk job. URL is plaintext by necessity — server-worker reads it without master-key access (same rationale as articles.originalUrl). + 'articleExtractPickup', // Short-lived server-write inbox; the client picks up the extracted payload, encrypts it into the articles table, deletes the row. Plaintext by necessity (server has no master key); empty in steady state. 'automations', // TODO: audit 'boardViews', // TODO: audit 'budgets', // TODO: audit diff --git a/apps/mana/apps/web/src/lib/data/database.ts b/apps/mana/apps/web/src/lib/data/database.ts index 30980ac19..a550c03eb 100644 --- a/apps/mana/apps/web/src/lib/data/database.ts +++ b/apps/mana/apps/web/src/lib/data/database.ts @@ -168,11 +168,6 @@ db.version(1).stores({ timeWorldClocks: 'id, sortOrder, timezone', entryTags: 'id, entryId, tagId, [entryId+tagId]', - // ─── Context (appId: 'context') ─── - contextSpaces: 'id, pinned, prefix', - documents: 'id, spaceId, type, pinned, title, [spaceId+type], updatedAt', - documentTags: 'id, documentId, tagId, [documentId+tagId]', - // ─── Questions (appId: 'questions') ─── qCollections: 'id, sortOrder, isDefault', questions: 'id, collectionId, status, priority, [collectionId+status]', @@ -673,11 +668,10 @@ db.version(30).stores({ _serverIterationExecutions: 'iterationId, missionId, executedAt', }); -// v31 — Rename the legacy `spaceId` field to `contextSpaceId` on four +// v31 — Rename the legacy `spaceId` field to `contextSpaceId` on three // tables that owned the term before the multi-tenancy Spaces foundation // arrived (v28): // - conversations (chat module's reference to a context-space folder) -// - documents (context module's parent context-space) // - spaceMembers (memoro's members of a context-space) // - memoSpaces (memoro's memo ↔ context-space join) // @@ -696,12 +690,11 @@ db.version(30).stores({ db.version(31) .stores({ conversations: 'id, isArchived, isPinned, contextSpaceId, templateId, updatedAt', - documents: 'id, contextSpaceId, type, pinned, title, [contextSpaceId+type], updatedAt', spaceMembers: 'id, contextSpaceId, userId', memoSpaces: 'id, memoId, contextSpaceId', }) .upgrade(async (tx) => { - const tables = ['conversations', 'documents', 'spaceMembers', 'memoSpaces'] as const; + const tables = ['conversations', 'spaceMembers', 'memoSpaces'] as const; for (const name of tables) { await tx .table(name) @@ -1404,6 +1397,39 @@ db.version(55).upgrade(async (tx) => { } }); +// v56 — Articles Bulk-Import (docs/plans/articles-bulk-import.md Phase 1). +// Three new tables that ride the standard sync pipeline under the +// articles appId: +// +// articleImportJobs — one row per bulk-import the user kicked off. +// Indexed on `status` for the JobsList tab filter, `[spaceId+status]` +// for the per-Space active-job query the worker projection runs, +// and `_updatedAtIndex` for chronological sort. Lease columns are +// scanned via JS filter — only ~tens of running jobs per user. +// articleImportItems — one row per URL inside a job. `[jobId+state]` +// is the hot index: the JobDetailView range-scans pending+running +// items per job, and the worker pulls "items in state=pending for +// these jobIds". `idx` is plain so the in-list display order +// scrolls without an extra sort key. `state` standalone is used by +// the cross-job retry-failed query. +// articleExtractPickup — short-lived inbox between server-worker +// write and client-pickup-consumer read. `itemId` indexed so the +// consumer can join back to the owning item row. Empty in steady +// state; server-side GC caps it at 24 h. +// +// All three are plaintext (encryption registry: plaintext-allowlist). +// `articleImportItems.url` and `articleExtractPickup.payload` ARE +// user-typed-adjacent content but stay plaintext by necessity — the +// server-side worker reads them without master-key access. Same +// rationale as articles.originalUrl. Once the article is persisted, +// the encrypted copy lives in `articles` and the item carries only an +// articleId pointer. +db.version(56).stores({ + articleImportJobs: 'id, status, [spaceId+status], _updatedAtIndex', + articleImportItems: 'id, jobId, [jobId+state], state, idx', + articleExtractPickup: 'id, itemId, _updatedAtIndex', +}); + // ─── Sync Routing ────────────────────────────────────────── // SYNC_APP_MAP, TABLE_TO_SYNC_NAME, TABLE_TO_APP, SYNC_NAME_TO_TABLE, // toSyncName() and fromSyncName() are now derived from per-module diff --git a/apps/mana/apps/web/src/lib/modules/articles/collections.ts b/apps/mana/apps/web/src/lib/modules/articles/collections.ts index ca0d00f33..267a72946 100644 --- a/apps/mana/apps/web/src/lib/modules/articles/collections.ts +++ b/apps/mana/apps/web/src/lib/modules/articles/collections.ts @@ -7,8 +7,19 @@ */ import { db } from '$lib/data/database'; -import type { LocalArticle, LocalHighlight, LocalArticleTag } from './types'; +import type { + LocalArticle, + LocalArticleExtractPickup, + LocalArticleImportItem, + LocalArticleImportJob, + LocalArticleTag, + LocalHighlight, +} from './types'; export const articleTable = db.table('articles'); export const articleHighlightTable = db.table('articleHighlights'); export const articleTagTable = db.table('articleTags'); +export const articleImportJobTable = db.table('articleImportJobs'); +export const articleImportItemTable = db.table('articleImportItems'); +export const articleExtractPickupTable = + db.table('articleExtractPickup'); diff --git a/apps/mana/apps/web/src/lib/modules/articles/module.config.ts b/apps/mana/apps/web/src/lib/modules/articles/module.config.ts index d34f3503d..11e5808f5 100644 --- a/apps/mana/apps/web/src/lib/modules/articles/module.config.ts +++ b/apps/mana/apps/web/src/lib/modules/articles/module.config.ts @@ -1,12 +1,19 @@ import type { ModuleConfig } from '$lib/data/module-registry'; /** - * Articles module — saved web articles + highlights + tag links. + * Articles module — saved web articles + highlights + tag links + bulk- + * import jobs. * * `articleTags` is a pure junction into globalTags (the core `tags` * appId). The junction itself syncs under `articles` appId with its * owning rows, the same pattern every other tagged module uses * (noteTags, eventTags, contactTags, placeTags, …). + * + * `articleImportJobs` + `articleImportItems` + `articleExtractPickup` + * implement the durable bulk-import pipeline (docs/plans/articles-bulk- + * import.md). All three sync under the articles appId so multi-device + * progress and server-worker state-transitions ride the standard + * sync_changes channel. */ export const articlesModuleConfig: ModuleConfig = { appId: 'articles', @@ -14,5 +21,8 @@ export const articlesModuleConfig: ModuleConfig = { { name: 'articles' }, { name: 'articleHighlights', syncName: 'highlights' }, { name: 'articleTags' }, + { name: 'articleImportJobs', syncName: 'importJobs' }, + { name: 'articleImportItems', syncName: 'importItems' }, + { name: 'articleExtractPickup', syncName: 'extractPickup' }, ], }; diff --git a/apps/mana/apps/web/src/lib/modules/articles/types.ts b/apps/mana/apps/web/src/lib/modules/articles/types.ts index f886a29e1..89cab9fdc 100644 --- a/apps/mana/apps/web/src/lib/modules/articles/types.ts +++ b/apps/mana/apps/web/src/lib/modules/articles/types.ts @@ -1,7 +1,7 @@ /** * Articles module — Pocket-style read-it-later. * - * Three Dexie tables: + * Six Dexie tables: * * articles — saved URLs + extracted Readability content * (encrypted: title, excerpt, content, htmlContent, @@ -12,6 +12,23 @@ * articleTags — pure junction into globalTags. No user-typed * content lives here — tag names/colors are in * the global tag system (appId: 'tags'). + * + * articleImportJobs — Bulk-Import job header. Plaintext: counters, + * status, lease metadata. See + * docs/plans/articles-bulk-import.md. + * articleImportItems — One row per URL in a bulk job. URL stays + * plaintext (server-worker reads it without + * master-key access — same rationale as + * articles.originalUrl). State machine: + * pending → extracting → extracted → + * (saved | duplicate | consent-wall | error | + * cancelled). + * articleExtractPickup — Server-write inbox: the worker drops the + * extracted payload here, the client picks it + * up, runs encryptRecord + articleTable.add, + * then deletes the row. Plaintext by necessity + * (server has no master key); empty in steady + * state. */ import type { BaseRecord } from '@mana/local-store'; @@ -115,3 +132,124 @@ export interface Highlight { createdAt: string; updatedAt: string; } + +// ─── Bulk Import (docs/plans/articles-bulk-import.md) ───── + +/** + * Job status — drives the index list filter and the JobDetailView's + * action bar. `running` is the only state where the worker actively + * pulls items; `paused` lets the user stop progress without losing the + * remaining queue, `cancelled` is a hard stop with all pending items + * flipped to terminal `cancelled`. + */ +export type ArticleImportJobStatus = 'queued' | 'running' | 'paused' | 'done' | 'cancelled'; + +/** + * Item state machine. Server-side transitions: pending → extracting → + * extracted (worker has dropped a pickup row). Client-side transitions: + * extracted → saved | duplicate | consent-wall (pickup-consumer + * applied the result). Both sides may transition to error (worker after + * 3 retries, client if encryptRecord/add fails). cancelled is terminal + * and only set when the parent job is cancelled before the item ran. + */ +export type ArticleImportItemState = + | 'pending' + | 'extracting' + | 'extracted' + | 'saved' + | 'duplicate' + | 'consent-wall' + | 'error' + | 'cancelled'; + +export interface LocalArticleImportJob extends BaseRecord { + totalUrls: number; + status: ArticleImportJobStatus; + /** Worker lease — workerId of the apps/api instance that claimed the job. */ + leasedBy: string | null; + /** ISO timestamp; lease is dead once `leasedUntil < now`. */ + leasedUntil: string | null; + startedAt: string | null; + finishedAt: string | null; + /** Counters mirror the per-item terminal states. Cache for fast list + * rendering — truth lives in the item rows. Worker stamps these on + * each transition. */ + savedCount: number; + duplicateCount: number; + errorCount: number; + warningCount: number; +} + +export interface LocalArticleImportItem extends BaseRecord { + jobId: string; + /** Original position in the user-provided URL list. Drives display order. */ + idx: number; + /** Plaintext — server worker reads it without master-key access. Same + * rationale as articles.originalUrl / newsArticles.originalUrl. */ + url: string; + state: ArticleImportItemState; + /** Pointer into `articles` table once the article is persisted. */ + articleId: string | null; + warning: 'probable_consent_wall' | null; + /** Plaintext technical error message ("502 Bad Gateway", "timeout"). */ + error: string | null; + attempts: number; + lastAttemptAt: string | null; +} + +/** + * Server → client handoff. Lives only between worker-write and + * pickup-consumer-read. Empty in steady state. + */ +export interface LocalArticleExtractPickup extends BaseRecord { + itemId: string; + /** The server's ExtractedArticle JSON, plaintext. Mirrors the shape + * in articles/api.ts but lives here as a structural type so the + * database layer doesn't import the API client. */ + payload: { + originalUrl: string; + title: string; + excerpt: string | null; + content: string; + htmlContent: string; + author: string | null; + siteName: string | null; + wordCount: number; + readingTimeMinutes: number; + warning?: 'probable_consent_wall'; + }; +} + +// Public DTOs used by views (livequery converters strip the BaseRecord +// internals + map state to display-friendly shapes). + +export interface ArticleImportJob { + id: string; + totalUrls: number; + status: ArticleImportJobStatus; + leasedBy: string | null; + leasedUntil: string | null; + startedAt: string | null; + finishedAt: string | null; + savedCount: number; + duplicateCount: number; + errorCount: number; + warningCount: number; + createdAt: string; + updatedAt: string; +} + +export interface ArticleImportItem { + id: string; + jobId: string; + idx: number; + url: string; + state: ArticleImportItemState; + articleId: string | null; + warning: 'probable_consent_wall' | null; + error: string | null; + attempts: number; + lastAttemptAt: string | null; + createdAt: string; + updatedAt: string; +} diff --git a/apps/mana/apps/web/src/lib/modules/broadcast/ListView.svelte b/apps/mana/apps/web/src/lib/modules/broadcasts/ListView.svelte similarity index 100% rename from apps/mana/apps/web/src/lib/modules/broadcast/ListView.svelte rename to apps/mana/apps/web/src/lib/modules/broadcasts/ListView.svelte diff --git a/apps/mana/apps/web/src/lib/modules/broadcast/api.ts b/apps/mana/apps/web/src/lib/modules/broadcasts/api.ts similarity index 100% rename from apps/mana/apps/web/src/lib/modules/broadcast/api.ts rename to apps/mana/apps/web/src/lib/modules/broadcasts/api.ts diff --git a/apps/mana/apps/web/src/lib/modules/broadcast/audience/AudienceBuilder.svelte b/apps/mana/apps/web/src/lib/modules/broadcasts/audience/AudienceBuilder.svelte similarity index 100% rename from apps/mana/apps/web/src/lib/modules/broadcast/audience/AudienceBuilder.svelte rename to apps/mana/apps/web/src/lib/modules/broadcasts/audience/AudienceBuilder.svelte diff --git a/apps/mana/apps/web/src/lib/modules/broadcast/audience/segment-builder.test.ts b/apps/mana/apps/web/src/lib/modules/broadcasts/audience/segment-builder.test.ts similarity index 100% rename from apps/mana/apps/web/src/lib/modules/broadcast/audience/segment-builder.test.ts rename to apps/mana/apps/web/src/lib/modules/broadcasts/audience/segment-builder.test.ts diff --git a/apps/mana/apps/web/src/lib/modules/broadcast/audience/segment-builder.ts b/apps/mana/apps/web/src/lib/modules/broadcasts/audience/segment-builder.ts similarity index 100% rename from apps/mana/apps/web/src/lib/modules/broadcast/audience/segment-builder.ts rename to apps/mana/apps/web/src/lib/modules/broadcasts/audience/segment-builder.ts diff --git a/apps/mana/apps/web/src/lib/modules/broadcast/collections.ts b/apps/mana/apps/web/src/lib/modules/broadcasts/collections.ts similarity index 100% rename from apps/mana/apps/web/src/lib/modules/broadcast/collections.ts rename to apps/mana/apps/web/src/lib/modules/broadcasts/collections.ts diff --git a/apps/mana/apps/web/src/lib/modules/broadcast/components/DnsCheckBanner.svelte b/apps/mana/apps/web/src/lib/modules/broadcasts/components/DnsCheckBanner.svelte similarity index 100% rename from apps/mana/apps/web/src/lib/modules/broadcast/components/DnsCheckBanner.svelte rename to apps/mana/apps/web/src/lib/modules/broadcasts/components/DnsCheckBanner.svelte diff --git a/apps/mana/apps/web/src/lib/modules/broadcast/components/SettingsForm.svelte b/apps/mana/apps/web/src/lib/modules/broadcasts/components/SettingsForm.svelte similarity index 100% rename from apps/mana/apps/web/src/lib/modules/broadcast/components/SettingsForm.svelte rename to apps/mana/apps/web/src/lib/modules/broadcasts/components/SettingsForm.svelte diff --git a/apps/mana/apps/web/src/lib/modules/broadcast/constants.ts b/apps/mana/apps/web/src/lib/modules/broadcasts/constants.ts similarity index 100% rename from apps/mana/apps/web/src/lib/modules/broadcast/constants.ts rename to apps/mana/apps/web/src/lib/modules/broadcasts/constants.ts diff --git a/apps/mana/apps/web/src/lib/modules/broadcast/editor/Editor.svelte b/apps/mana/apps/web/src/lib/modules/broadcasts/editor/Editor.svelte similarity index 100% rename from apps/mana/apps/web/src/lib/modules/broadcast/editor/Editor.svelte rename to apps/mana/apps/web/src/lib/modules/broadcasts/editor/Editor.svelte diff --git a/apps/mana/apps/web/src/lib/modules/broadcast/index.ts b/apps/mana/apps/web/src/lib/modules/broadcasts/index.ts similarity index 100% rename from apps/mana/apps/web/src/lib/modules/broadcast/index.ts rename to apps/mana/apps/web/src/lib/modules/broadcasts/index.ts diff --git a/apps/mana/apps/web/src/lib/modules/broadcast/module.config.ts b/apps/mana/apps/web/src/lib/modules/broadcasts/module.config.ts similarity index 100% rename from apps/mana/apps/web/src/lib/modules/broadcast/module.config.ts rename to apps/mana/apps/web/src/lib/modules/broadcasts/module.config.ts diff --git a/apps/mana/apps/web/src/lib/modules/broadcast/preview/EmailPreview.svelte b/apps/mana/apps/web/src/lib/modules/broadcasts/preview/EmailPreview.svelte similarity index 100% rename from apps/mana/apps/web/src/lib/modules/broadcast/preview/EmailPreview.svelte rename to apps/mana/apps/web/src/lib/modules/broadcasts/preview/EmailPreview.svelte diff --git a/apps/mana/apps/web/src/lib/modules/broadcast/preview/PreviewTabs.svelte b/apps/mana/apps/web/src/lib/modules/broadcasts/preview/PreviewTabs.svelte similarity index 100% rename from apps/mana/apps/web/src/lib/modules/broadcast/preview/PreviewTabs.svelte rename to apps/mana/apps/web/src/lib/modules/broadcasts/preview/PreviewTabs.svelte diff --git a/apps/mana/apps/web/src/lib/modules/broadcast/queries.ts b/apps/mana/apps/web/src/lib/modules/broadcasts/queries.ts similarity index 100% rename from apps/mana/apps/web/src/lib/modules/broadcast/queries.ts rename to apps/mana/apps/web/src/lib/modules/broadcasts/queries.ts diff --git a/apps/mana/apps/web/src/lib/modules/broadcast/render/email-html.test.ts b/apps/mana/apps/web/src/lib/modules/broadcasts/render/email-html.test.ts similarity index 100% rename from apps/mana/apps/web/src/lib/modules/broadcast/render/email-html.test.ts rename to apps/mana/apps/web/src/lib/modules/broadcasts/render/email-html.test.ts diff --git a/apps/mana/apps/web/src/lib/modules/broadcast/render/email-html.ts b/apps/mana/apps/web/src/lib/modules/broadcasts/render/email-html.ts similarity index 100% rename from apps/mana/apps/web/src/lib/modules/broadcast/render/email-html.ts rename to apps/mana/apps/web/src/lib/modules/broadcasts/render/email-html.ts diff --git a/apps/mana/apps/web/src/lib/modules/broadcast/render/plain-text.test.ts b/apps/mana/apps/web/src/lib/modules/broadcasts/render/plain-text.test.ts similarity index 100% rename from apps/mana/apps/web/src/lib/modules/broadcast/render/plain-text.test.ts rename to apps/mana/apps/web/src/lib/modules/broadcasts/render/plain-text.test.ts diff --git a/apps/mana/apps/web/src/lib/modules/broadcast/render/plain-text.ts b/apps/mana/apps/web/src/lib/modules/broadcasts/render/plain-text.ts similarity index 100% rename from apps/mana/apps/web/src/lib/modules/broadcast/render/plain-text.ts rename to apps/mana/apps/web/src/lib/modules/broadcasts/render/plain-text.ts diff --git a/apps/mana/apps/web/src/lib/modules/broadcast/stores/campaigns.svelte.ts b/apps/mana/apps/web/src/lib/modules/broadcasts/stores/campaigns.svelte.ts similarity index 100% rename from apps/mana/apps/web/src/lib/modules/broadcast/stores/campaigns.svelte.ts rename to apps/mana/apps/web/src/lib/modules/broadcasts/stores/campaigns.svelte.ts diff --git a/apps/mana/apps/web/src/lib/modules/broadcast/stores/settings.svelte.ts b/apps/mana/apps/web/src/lib/modules/broadcasts/stores/settings.svelte.ts similarity index 100% rename from apps/mana/apps/web/src/lib/modules/broadcast/stores/settings.svelte.ts rename to apps/mana/apps/web/src/lib/modules/broadcasts/stores/settings.svelte.ts diff --git a/apps/mana/apps/web/src/lib/modules/broadcast/tools.ts b/apps/mana/apps/web/src/lib/modules/broadcasts/tools.ts similarity index 100% rename from apps/mana/apps/web/src/lib/modules/broadcast/tools.ts rename to apps/mana/apps/web/src/lib/modules/broadcasts/tools.ts diff --git a/apps/mana/apps/web/src/lib/modules/broadcast/types.ts b/apps/mana/apps/web/src/lib/modules/broadcasts/types.ts similarity index 100% rename from apps/mana/apps/web/src/lib/modules/broadcast/types.ts rename to apps/mana/apps/web/src/lib/modules/broadcasts/types.ts diff --git a/apps/mana/apps/web/src/lib/modules/broadcast/views/ComposeView.svelte b/apps/mana/apps/web/src/lib/modules/broadcasts/views/ComposeView.svelte similarity index 100% rename from apps/mana/apps/web/src/lib/modules/broadcast/views/ComposeView.svelte rename to apps/mana/apps/web/src/lib/modules/broadcasts/views/ComposeView.svelte diff --git a/apps/mana/apps/web/src/lib/modules/broadcast/views/DetailView.svelte b/apps/mana/apps/web/src/lib/modules/broadcasts/views/DetailView.svelte similarity index 100% rename from apps/mana/apps/web/src/lib/modules/broadcast/views/DetailView.svelte rename to apps/mana/apps/web/src/lib/modules/broadcasts/views/DetailView.svelte diff --git a/apps/mana/apps/web/src/lib/modules/broadcast/widgets/BroadcastsWidget.svelte b/apps/mana/apps/web/src/lib/modules/broadcasts/widgets/BroadcastsWidget.svelte similarity index 100% rename from apps/mana/apps/web/src/lib/modules/broadcast/widgets/BroadcastsWidget.svelte rename to apps/mana/apps/web/src/lib/modules/broadcasts/widgets/BroadcastsWidget.svelte diff --git a/apps/mana/apps/web/src/routes/(app)/observatory/+page.svelte b/apps/mana/apps/web/src/routes/(app)/observatory/+page.svelte deleted file mode 100644 index a3901095f..000000000 --- a/apps/mana/apps/web/src/routes/(app)/observatory/+page.svelte +++ /dev/null @@ -1,351 +0,0 @@ - - - -
- -
- {#each tabs as tab} - - {/each} -
- - - {#if activeTab === 'scene'} -
- -
- {:else if activeTab === 'plants'} - - {:else if activeTab === 'lakes'} - - {:else if activeTab === 'rivers'} - - {:else if activeTab === 'leaderboard'} - - {:else if activeTab === 'compare'} - - {:else if activeTab === 'trends'} - - {/if} -
- - - (selectedApp = null)} /> -
- - diff --git a/docs/plans/articles-bulk-import.md b/docs/plans/articles-bulk-import.md new file mode 100644 index 000000000..1430f83a7 --- /dev/null +++ b/docs/plans/articles-bulk-import.md @@ -0,0 +1,585 @@ +# Articles — Bulk URL Import + +## Status (2026-04-28) + +**Phase 0 — Plan:** in progress. +**Phasen 1–7:** offen. + +## Ziel + +User wirft eine Liste von URLs in ein Textfeld (zeilengetrennt), Mana +extrahiert + speichert alle Artikel im Hintergrund. Funktioniert auch wenn +der Tab schließt, das Gerät wechselt, das Netz kurz weg ist. Der Job +überlebt Sessions und ist auf jedem Gerät sichtbar an dem der User +eingeloggt ist. + +Heute existiert nur Single-URL-Ingestion (`AddUrlForm`, `QuickAddInput`, +Bookmarklets v1+v2, Share-Target). Alle Pfade rufen am Ende +`articlesStore.saveFromUrl()` oder `saveFromExtracted()` auf. + +## Leitsätze + +1. **Job-State lebt in der synchronisierten DB**, nicht im Tab. Damit + fallen Tab-Close-Resilienz, Multi-Device-Sicht, Resume-after-Offline + und Audit automatisch ab. +2. **Server macht Extract, Client macht Encrypt.** Das ehrt das At-Rest- + Modell — der Master-Key bleibt clientseitig, der Server sieht den + extrahierten Text nur kurz in einer Pickup-Inbox (gleicher Threat- + Model wie heute schon der `/extract`-Endpoint). +3. **Eine Code-Bahn für jede Ingestion.** Single-URL und Bulk laufen + nach Phase 7 durch denselben Worker — der QuickAdd-Pfad legt unter + der Haube auch einen 1-URL-Job an. +4. **Soft → Hard.** Schema- und Semantik-Migrationen kommen in zwei + Commits: erst tolerant zu alten Rows, dann hartes Cleanup. + +## Architektur + +``` + ┌──────────────────────────────────────────┐ + │ /articles/import (List + JobDetail) │ + │ pure liveQuery-View, UI macht keine │ + │ Job-Logik selbst │ + └─────────────────┬────────────────────────┘ + │ liveQuery + ▼ + ┌────────────────────────────────────────────────────────┐ + │ Dexie + mana-sync (articles appId) │ + │ │ + │ articleImportJobs │ + │ id, spaceId, totalUrls, status, leasedBy, │ + │ leasedUntil, savedCount, duplicateCount, │ + │ errorCount, warningCount, finishedAt │ + │ │ + │ articleImportItems │ + │ id, jobId, spaceId, idx, url, state, articleId, │ + │ warning, error, attempts, lastAttemptAt │ + │ │ + │ articleExtractPickup (kurzlebige Inbox) │ + │ id, itemId, payload (extracted), createdAt │ + └─────────────────┬───────────────────────┬──────────────┘ + │ pending items │ pickup rows + ▼ ▼ + ┌─────────────────────────────┐ ┌──────────────────────────┐ + │ apps/api Extract-Worker │ │ Client Pickup-Consumer │ + │ • snapshot der Items │ │ • liveQuery auf Pickup │ + │ • Lease + Heartbeat │ │ • encryptRecord │ + │ • Concurrency 3 / User │ │ • articleTable.add() │ + │ • shared-rss extractFromUrl│ │ • item.state='saved' │ + │ • schreibt Pickup-Row │ │ • pickup.delete() │ + │ • setzt Item-State │ │ │ + └─────────────────────────────┘ └──────────────────────────┘ +``` + +## Datenmodell + +### `articleImportJobs` (synced, articles appId) + +```ts +interface LocalArticleImportJob extends BaseRecord { + totalUrls: number; + status: 'queued' | 'running' | 'paused' | 'done' | 'cancelled'; + + /** Worker-Lease — verhindert dass mehrere Worker denselben Job ziehen. + * Server-Worker stempelt seine workerId beim Claim, erneuert die + * leasedUntil per Heartbeat. Lease-Ablauf > 60s = Job ist verfügbar. */ + leasedBy: string | null; + leasedUntil: string | null; + + startedAt: string | null; + finishedAt: string | null; + + /** Counters werden vom Server beim Item-Übergang in einen Terminal-State + * inkrementiert. Pure Bookkeeping — Truth liegt in den Item-Rows, das + * hier ist die Cache-Spalte für die Liste. */ + savedCount: number; + duplicateCount: number; + errorCount: number; + warningCount: number; +} +``` + +### `articleImportItems` (synced, articles appId) + +```ts +type ImportItemState = + | 'pending' // wartet auf den Worker + | 'extracting' // Worker hat geclaimed + | 'extracted' // Pickup-Row liegt für den Client bereit + | 'saved' // im Article-Table angekommen + | 'duplicate' // Article mit dieser URL gabs schon + | 'consent-wall' // gespeichert, aber Cookie-Wand erkannt + | 'error' // X Versuche fehlgeschlagen + | 'cancelled'; // Job abgebrochen vor Verarbeitung + +interface LocalArticleImportItem extends BaseRecord { + jobId: string; + idx: number; // Reihenfolge aus der User-Eingabe + url: string; // PLAINTEXT — Server muss lesen können + state: ImportItemState; + articleId: string | null; // bei saved/duplicate gesetzt + warning: 'probable_consent_wall' | null; + error: string | null; + attempts: number; + lastAttemptAt: string | null; +} +``` + +**`url` bleibt bewusst plaintext** — der Server-Worker liest sie aus +`sync_changes` und kann nicht entschlüsseln. Gleiche Begründung wie bei +`articles.originalUrl` / `newsArticles.originalUrl` / `links.originalUrl`. + +`error` bleibt plaintext, weil Fehlertexte technisch sind ("502 Bad +Gateway") und keinen User-Inhalt enthalten. + +### `articleExtractPickup` (synced, articles appId, kurzlebig) + +```ts +interface LocalArticleExtractPickup extends BaseRecord { + itemId: string; // Pointer zum Item — auch dessen jobId + payload: ExtractedArticle; // PLAINTEXT — Server hat das eh + createdAt: string; +} +``` + +Inbox-Tabelle. Server schreibt rein, Client liest und löscht. Im +Steady-State leer. TTL serverseitig: 24 h, dann GC. + +**Warum eine eigene Tabelle statt direkt `articles` schreiben?** Der +Server hat keinen Master-Key — er kann den Article nicht verschlüsseln. +Pickup ist die Übergabe-Pufferzone, der Client holt sie ab und ruft die +existierende `saveFromExtracted()` auf, die `encryptRecord()` triggert. + +### Crypto-Registry + +```ts +// articleImportJobs: keine User-typed Inhalte → plaintext-allowlist +// articleImportItems: url + error sind plaintext, sonst nichts schützenswert → plaintext-allowlist +// articleExtractPickup: payload wird gleich nach Apply gelöscht → plaintext-allowlist +``` + +Alle drei landen auf der `plaintext-allowlist.ts`, nicht in +`ENCRYPTION_REGISTRY`. Items und Job-Rows enthalten keine User-typed- +Felder die nicht eh schon plaintext bleiben müssten (URL fürs Routing, +Counters, Foreign Keys). Der eigentliche Article-Inhalt wandert wie +bisher verschlüsselt in `articles`. + +### Module-Config + Sync + +`modules/articles/module.config.ts` bekommt drei neue Tabellen: + +```ts +tables: [ + { name: 'articles' }, + { name: 'articleHighlights', syncName: 'highlights' }, + { name: 'articleTags' }, + { name: 'articleImportJobs', syncName: 'importJobs' }, + { name: 'articleImportItems', syncName: 'importItems' }, + { name: 'articleExtractPickup', syncName: 'extractPickup' }, +] +``` + +Damit gehen sie automatisch durch den Standard-Sync-Pfad, RLS, +field-level LWW. Keine neue Sync-Infrastruktur. + +## Server-Worker + +### Wo + +**`apps/api/src/modules/articles/import-worker.ts`**, gestartet aus +`apps/api/src/index.ts` neben den Routes. Nicht in `services/mana-ai` +(falscher Scope) und nicht in `services/mana-research` (Provider- +Orchestrierung, kein Persistenz-Worker). + +### Konzept + +Standard-Pattern aus `services/mana-ai`: + +1. **Snapshot-Projektion** — eine kleine Tabelle in `mana_platform.articles_imports`-Schema + die `sync_changes` für `appId='articles'` und `tableName ∈ {articleImportJobs, articleImportItems}` + zu Live-Records faltet (field-level LWW). Refreshed sich pro Tick. +2. **Tick alle 2 s.** Liest die Snapshot, sucht: + - Jobs mit `status='running'` und (`leasedBy=null` OR `leasedUntil < now`) + - dazu Items mit `state='pending'` für diese Jobs +3. **Lease** — `leasedBy` auf eigene `workerId` setzen, `leasedUntil = now + 60s`. Schreiben als + `sync_changes`-Row mit `actor=system`, `origin=system`, `source='articles-import-worker'`. +4. **Concurrency 3 pro Job** — pro Tick max 3 Items in `state='extracting'` schalten, + `extractFromUrl()` aus `@mana/shared-rss` aufrufen. +5. **Pickup-Write** — bei Erfolg: `articleExtractPickup`-Row schreiben + + Item-State auf `extracted`. Bei Fehler: `attempts += 1`, wenn `attempts >= 3` + → `state='error'`, sonst zurück auf `pending`. +6. **Job-Completion** — wenn alle Items eines Jobs in einem Terminal-State sind + (`saved | duplicate | consent-wall | error | cancelled`), setze + `job.status='done'` + `finishedAt`. Counter-Spalten gleich mit aktualisieren. +7. **Heartbeat** — solange Items `extracting`, alle 30 s `leasedUntil` erneuern. + +### Single-Instance-Garantie + +`pg_advisory_lock()` über die Worker-Loop. Falls apps/api in mehreren +Instanzen läuft, nimmt nur eine den Lock und tickt. Andere idlen. + +### Counters: woher + +Worker tracked Item-Übergänge und stempelt: +- `pending → extracted`: keine Counter-Änderung +- `extracted → saved` (Client signalisiert): `savedCount += 1` +- `extracted → duplicate` (Client signalisiert): `duplicateCount += 1` +- `extracted → consent-wall` (Client signalisiert): `warningCount += 1` +- jeder Übergang → `error`: `errorCount += 1` + +Counter-Updates gehen als normale `articleImportJobs.update` durch +`sync_changes`, RLS-correct. + +### Server-side Cleanup + +Stündlicher GC-Job: +- Pickup-Rows älter 24 h löschen (Sicherheits-Cap) +- Jobs mit `status='done' AND finishedAt < now - 30d` archivieren + (späteres Polish — erst mal nur Cap) + +## Client-Pickup-Consumer + +`apps/mana/apps/web/src/lib/modules/articles/consume-pickup.ts`, +gestartet aus `data-layer-listeners.ts` zusammen mit den anderen +Listener-Wirings. + +Logik: + +```ts +liveQuery(() => articleExtractPickup + .filter(r => !r.deletedAt) + .toArray() +).subscribe(rows => { + for (const row of rows) { + void consumeOne(row); + } +}); + +async function consumeOne(row: LocalArticleExtractPickup) { + // Re-entrancy guard via in-memory Set so multiple liveQuery ticks + // don't race the same row. + if (inFlight.has(row.id)) return; + inFlight.add(row.id); + try { + const item = await articleImportItemTable.get(row.itemId); + if (!item || item.state !== 'extracted') { + await articleExtractPickupTable.delete(row.id); // Stale row + return; + } + // Dedupe-Check für den Fall dass der User die URL parallel + // single-saved hat während der Job lief. + const existing = await articlesStore.findByUrl(row.payload.originalUrl); + if (existing) { + await articleImportItemTable.update(item.id, { + state: 'duplicate', + articleId: existing.id, + }); + await articleExtractPickupTable.delete(row.id); + return; + } + const article = await articlesStore.saveFromExtracted(row.payload); + const nextState: ImportItemState = + row.payload.warning === 'probable_consent_wall' + ? 'consent-wall' + : 'saved'; + await articleImportItemTable.update(item.id, { + state: nextState, + articleId: article.id, + warning: row.payload.warning ?? null, + }); + await articleExtractPickupTable.delete(row.id); + } finally { + inFlight.delete(row.id); + } +} +``` + +Multi-Tab: alle Tabs sehen Pickup-Rows. Web-Lock `mana:articles:pickup` +sorgt dafür dass nur ein Tab gleichzeitig konsumiert. Andere Tabs sehen +die liveQuery, der Lock-halter pickt ab. + +## Store-API + +`modules/articles/stores/imports.svelte.ts` — neue Datei. + +```ts +export const articleImportsStore = { + /** Erzeugt Job + N Items in einem Dexie bulkAdd, returns jobId. */ + async createJob(urls: string[]): Promise { … }, + + async pauseJob(jobId: string): Promise { … }, + async resumeJob(jobId: string): Promise { … }, + async cancelJob(jobId: string): Promise { … }, + + /** Setzt alle Error-Items eines Jobs zurück auf pending. */ + async retryFailed(jobId: string): Promise { … }, + + /** Soft-Delete des Jobs + aller Items. Article-Rows bleiben. */ + async deleteJob(jobId: string): Promise { … }, +}; +``` + +`saveFromExtracted` in `modules/articles/stores/articles.svelte.ts` +bleibt der gemeinsame Kern — der Pickup-Consumer ruft sie genauso auf wie +der existierende Single-URL-Pfad. + +URL-Parser steht im Store, nicht im Component: + +```ts +export function parseUrls(raw: string): { + valid: string[]; + invalid: string[]; + duplicates: string[]; +} { … } +``` + +Pure Funktion, unit-testbar. + +## UI + +### `/articles/import` — Index + Eingabe (`+page.svelte`) + +- `` Komponente mit `