From 07365c31b2659c913b336f2007c9c7409fdaf8b4 Mon Sep 17 00:00:00 2001 From: Till JS Date: Mon, 23 Mar 2026 11:15:29 +0100 Subject: [PATCH] chore: remove stale docs and outdated design plans - docs/AUTOMATED_TESTING_SYSTEM.md (implementation report, workflow already exists) - docs/README_ENV_AUDIT.md (index for already-deleted audit files) - docs/DEPENDENCY_ALIGNMENT.md (version tracking, immediately outdated) - .claude/plans/mana-search-service.md (draft from Jan 2025, service already built) - .claude/plans/questions-app.md (draft from Jan 2025, app already built) Co-Authored-By: Claude Opus 4.6 (1M context) --- .claude/plans/mana-search-service.md | 1597 -------------------------- .claude/plans/questions-app.md | 1471 ------------------------ docs/AUTOMATED_TESTING_SYSTEM.md | 583 ---------- docs/DEPENDENCY_ALIGNMENT.md | 180 --- docs/README_ENV_AUDIT.md | 264 ----- 5 files changed, 4095 deletions(-) delete mode 100644 .claude/plans/mana-search-service.md delete mode 100644 .claude/plans/questions-app.md delete mode 100644 docs/AUTOMATED_TESTING_SYSTEM.md delete mode 100644 docs/DEPENDENCY_ALIGNMENT.md delete mode 100644 docs/README_ENV_AUDIT.md diff --git a/.claude/plans/mana-search-service.md b/.claude/plans/mana-search-service.md deleted file mode 100644 index 39b58d5d7..000000000 --- a/.claude/plans/mana-search-service.md +++ /dev/null @@ -1,1597 +0,0 @@ -# Mana Search Service - Design Document - -> **Status**: Entwurf - Zur Überprüfung -> **Erstellt**: 2025-01-28 -> **Autor**: Claude Code -> **Abhängigkeit**: questions-app.md - ---- - -## 1. Übersicht - -### 1.1 Was ist der Mana Search Service? - -Ein zentraler Microservice, der **Web-Suche und Content-Extraktion** für alle ManaCore Apps bereitstellt. - -``` -┌─────────────────────────────────────────────────────────────────────────┐ -│ MANA SEARCH SERVICE │ -├─────────────────────────────────────────────────────────────────────────┤ -│ │ -│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ -│ │ Questions │ │ Chat │ │ Future │ │ -│ │ App │ │ App │ │ Apps │ │ -│ └──────┬──────┘ └──────┬──────┘ └──────┬──────┘ │ -│ │ │ │ │ -│ └────────────┬────┴────────────────┘ │ -│ ▼ │ -│ ┌────────────────────────┐ │ -│ │ mana-search (3021) │ ◄── NestJS API │ -│ │ ┌────────────────┐ │ │ -│ │ │ Search API │ │ ◄── Unified Interface │ -│ │ │ Extract API │ │ ◄── Content Extraction │ -│ │ │ Cache Layer │ │ ◄── Redis Caching │ -│ │ └───────┬────────┘ │ │ -│ └───────────┼────────────┘ │ -│ ▼ │ -│ ┌────────────────────────┐ │ -│ │ SearXNG (internal) │ ◄── Meta-Suchmaschine │ -│ │ Port 8080 (Docker) │ │ -│ └───────────┬────────────┘ │ -│ ▼ │ -│ ┌────────────────────────────────────────┐ │ -│ │ Google │ Bing │ DuckDuckGo │ Brave... │ │ -│ └────────────────────────────────────────┘ │ -│ │ -└─────────────────────────────────────────────────────────────────────────┘ -``` - -### 1.2 Warum ein zentraler Service? - -| Vorteil | Beschreibung | -|---------|--------------| -| **Einmalige Konfiguration** | SearXNG nur einmal aufsetzen und optimieren | -| **Caching** | Ergebnisse über alle Apps hinweg cachen | -| **Rate Limiting** | Zentrale Kontrolle über Anfragen an Suchmaschinen | -| **Content Extraction** | Einheitliche Extraktion, einmal optimiert | -| **Monitoring** | Zentrale Metriken für Suche | -| **Wartung** | Updates nur an einer Stelle | -| **Kostenersparnis** | Falls später paid APIs hinzukommen → zentral | - -### 1.3 Consumer Apps - -| App | Use Case | -|-----|----------| -| **Questions** | Recherche zu Nutzer-Fragen | -| **Chat** | Web-Grounding für AI-Antworten | -| **Project Doc Bot** | Technische Dokumentations-Suche | -| **Future Apps** | Jede App mit Recherche-Bedarf | - ---- - -## 2. Architektur - -### 2.1 Service-Struktur - -``` -services/mana-search/ -├── src/ -│ ├── main.ts # NestJS Entry Point -│ ├── app.module.ts # Root Module -│ │ -│ ├── config/ -│ │ └── configuration.ts # Environment Config -│ │ -│ ├── search/ -│ │ ├── search.module.ts -│ │ ├── search.controller.ts # /api/v1/search -│ │ ├── search.service.ts # Business Logic -│ │ ├── dto/ -│ │ │ ├── search-request.dto.ts -│ │ │ └── search-response.dto.ts -│ │ └── providers/ -│ │ ├── search-provider.interface.ts -│ │ ├── searxng.provider.ts # SearXNG Implementation -│ │ └── brave.provider.ts # Fallback (optional) -│ │ -│ ├── extract/ -│ │ ├── extract.module.ts -│ │ ├── extract.controller.ts # /api/v1/extract -│ │ ├── extract.service.ts # Content Extraction -│ │ └── extractors/ -│ │ ├── article.extractor.ts # News/Blog Articles -│ │ ├── docs.extractor.ts # Documentation Sites -│ │ └── generic.extractor.ts # Fallback -│ │ -│ ├── cache/ -│ │ ├── cache.module.ts -│ │ └── cache.service.ts # Redis Caching -│ │ -│ ├── health/ -│ │ ├── health.module.ts -│ │ └── health.controller.ts # /health -│ │ -│ └── metrics/ -│ ├── metrics.module.ts -│ └── metrics.service.ts # Prometheus Metrics -│ -├── searxng/ -│ ├── Dockerfile # SearXNG Container -│ ├── settings.yml # Engine Config -│ └── limiter.toml # Rate Limits -│ -├── Dockerfile # NestJS Service -├── docker-compose.yml # Local Development -├── package.json -├── tsconfig.json -├── nest-cli.json -└── CLAUDE.md -``` - -### 2.2 Docker Compose Setup - -```yaml -# services/mana-search/docker-compose.yml -version: '3.8' - -services: - # NestJS API Service - mana-search: - build: - context: . - dockerfile: Dockerfile - ports: - - "3021:3021" - environment: - NODE_ENV: development - PORT: 3021 - SEARXNG_URL: http://searxng:8080 - REDIS_HOST: redis - REDIS_PORT: 6379 - CACHE_TTL: 3600 # 1 hour - depends_on: - - searxng - - redis - networks: - - manacore-network - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:3021/health"] - interval: 30s - timeout: 10s - retries: 3 - - # SearXNG Meta-Suchmaschine - searxng: - image: searxng/searxng:latest - volumes: - - ./searxng/settings.yml:/etc/searxng/settings.yml:ro - - ./searxng/limiter.toml:/etc/searxng/limiter.toml:ro - environment: - SEARXNG_BASE_URL: http://localhost:8080 - SEARXNG_SECRET: ${SEARXNG_SECRET:-change-me-in-production} - networks: - - manacore-network - # Kein Port-Mapping - nur intern erreichbar - healthcheck: - test: ["CMD", "wget", "-q", "--spider", "http://localhost:8080/healthz"] - interval: 30s - timeout: 10s - retries: 3 - - # Redis Cache - redis: - image: redis:7-alpine - command: redis-server --appendonly yes - volumes: - - redis-data:/data - networks: - - manacore-network - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 10s - timeout: 5s - retries: 3 - -volumes: - redis-data: - -networks: - manacore-network: - external: true -``` - -### 2.3 SearXNG Konfiguration - -```yaml -# services/mana-search/searxng/settings.yml -use_default_settings: true - -general: - instance_name: "ManaCore Search" - debug: false - privacypolicy_url: false - donation_url: false - contact_url: false - enable_metrics: true - -search: - safe_search: 0 - autocomplete: "google" - default_lang: "de-DE" - formats: - - html - - json # Wichtig für API-Nutzung - -server: - secret_key: ${SEARXNG_SECRET} - limiter: true - image_proxy: false - method: "GET" - # Nur lokaler Zugriff - bind_address: "0.0.0.0" - port: 8080 - -ui: - static_use_hash: true - default_theme: simple - theme_args: - simple_style: dark - -# Aktivierte Suchmaschinen -engines: - # Web Search - - name: google - engine: google - shortcut: g - disabled: false - weight: 1.2 - - - name: bing - engine: bing - shortcut: b - disabled: false - weight: 1.0 - - - name: duckduckgo - engine: duckduckgo - shortcut: d - disabled: false - weight: 0.9 - - - name: brave - engine: brave - shortcut: br - disabled: false - weight: 1.0 - - - name: qwant - engine: qwant - shortcut: q - disabled: false - weight: 0.8 - - # IT/Developer Search - - name: github - engine: github - shortcut: gh - disabled: false - categories: [it] - - - name: stackoverflow - engine: stackoverflow - shortcut: so - disabled: false - categories: [it] - - - name: npm - engine: npm - shortcut: npm - disabled: false - categories: [it] - - # Wissenschaft - - name: arxiv - engine: arxiv - shortcut: ar - disabled: false - categories: [science] - - - name: google scholar - engine: google_scholar - shortcut: gs - disabled: false - categories: [science] - - - name: semantic scholar - engine: semantic_scholar - shortcut: ss - disabled: false - categories: [science] - - - name: pubmed - engine: pubmed - shortcut: pm - disabled: false - categories: [science, health] - - # Wikipedia - - name: wikipedia - engine: wikipedia - shortcut: w - disabled: false - weight: 1.1 - - # News - - name: google news - engine: google_news - shortcut: gn - disabled: false - categories: [news] - - - name: bing news - engine: bing_news - shortcut: bn - disabled: false - categories: [news] - -outgoing: - # Timeouts - request_timeout: 5.0 - max_request_timeout: 15.0 - # User Agent Rotation - useragent_suffix: "" - # Proxy (optional) - # proxies: - # http: "socks5://tor:9050" - # https: "socks5://tor:9050" - -# Kategorien -categories_as_tabs: - general: - images: - videos: - news: - science: - it: -``` - -### 2.4 Rate Limiter - -```toml -# services/mana-search/searxng/limiter.toml -[botdetection.ip_limit] -# Limit pro IP-Adresse -link_token = true -# Suchen pro Minute pro IP -limit = 30 -# Burst erlaubt -burst = 10 - -[botdetection.ip_lists] -# Interne IPs (Docker) erlauben -pass_ip = [ - "172.16.0.0/12", # Docker internal - "192.168.0.0/16", # Private networks - "10.0.0.0/8", # Private networks - "127.0.0.1", # Localhost -] -``` - ---- - -## 3. API-Endpoints - -### 3.1 Search API - -```typescript -// ============================================ -// SEARCH ENDPOINTS -// ============================================ - -/** - * Web-Suche durchführen - * POST /api/v1/search - */ -interface SearchRequest { - query: string; // Suchbegriff (required) - options?: { - categories?: SearchCategory[]; // ['general', 'news', 'science', 'it'] - engines?: string[]; // ['google', 'bing', 'duckduckgo'] - language?: string; // 'de-DE', 'en-US' - timeRange?: TimeRange; // 'day', 'week', 'month', 'year' - safeSearch?: 0 | 1 | 2; // 0=off, 1=moderate, 2=strict - limit?: number; // Max results (default: 10, max: 50) - }; - cache?: { - enabled?: boolean; // Default: true - ttl?: number; // Cache TTL in seconds - }; -} - -interface SearchResponse { - results: SearchResult[]; - meta: { - query: string; - totalResults: number; - engines: string[]; // Welche Engines geantwortet haben - duration: number; // Dauer in ms - cached: boolean; - cacheKey?: string; - }; -} - -interface SearchResult { - url: string; - title: string; - snippet: string; // Kurzer Textauszug - engine: string; // Welche Engine - score: number; // Relevanz-Score (0-1) - publishedDate?: string; // Falls verfügbar - thumbnail?: string; // Vorschaubild URL - category: string; // 'general', 'news', etc. -} - -type SearchCategory = 'general' | 'news' | 'science' | 'it' | 'images' | 'videos'; -type TimeRange = 'day' | 'week' | 'month' | 'year'; - -// ============================================ -// Beispiel-Aufruf -// ============================================ - -POST /api/v1/search -{ - "query": "quantum computing basics", - "options": { - "categories": ["general", "science"], - "engines": ["google", "bing", "wikipedia"], - "language": "en-US", - "limit": 10 - } -} - -// Response -{ - "results": [ - { - "url": "https://en.wikipedia.org/wiki/Quantum_computing", - "title": "Quantum computing - Wikipedia", - "snippet": "Quantum computing is a type of computation that harnesses...", - "engine": "wikipedia", - "score": 0.95, - "category": "general" - }, - // ... - ], - "meta": { - "query": "quantum computing basics", - "totalResults": 10, - "engines": ["google", "bing", "wikipedia"], - "duration": 1234, - "cached": false - } -} -``` - -### 3.2 Extract API - -```typescript -// ============================================ -// CONTENT EXTRACTION ENDPOINTS -// ============================================ - -/** - * Inhalt einer URL extrahieren - * POST /api/v1/extract - */ -interface ExtractRequest { - url: string; // URL zum Extrahieren - options?: { - includeHtml?: boolean; // Original HTML mitliefern - includeMarkdown?: boolean; // Als Markdown konvertieren - maxLength?: number; // Max Zeichen (default: unlimited) - extractImages?: boolean; // Bilder extrahieren - extractLinks?: boolean; // Links extrahieren - timeout?: number; // Timeout in ms (default: 10000) - }; -} - -interface ExtractResponse { - success: boolean; - content?: ExtractedContent; - error?: string; - meta: { - url: string; - duration: number; - cached: boolean; - contentType: string; - }; -} - -interface ExtractedContent { - title: string; - description?: string; - author?: string; - publishedDate?: string; - siteName?: string; - - // Content - text: string; // Plain text - markdown?: string; // Markdown (wenn angefordert) - html?: string; // Original HTML (wenn angefordert) - - // Word/Read Stats - wordCount: number; - readingTime: number; // Minuten - - // Media - images?: ExtractedImage[]; - links?: ExtractedLink[]; - - // Open Graph / Meta - ogImage?: string; - ogType?: string; - language?: string; -} - -interface ExtractedImage { - url: string; - alt?: string; - width?: number; - height?: number; -} - -interface ExtractedLink { - url: string; - text: string; - isExternal: boolean; -} - -// ============================================ -// Beispiel-Aufruf -// ============================================ - -POST /api/v1/extract -{ - "url": "https://example.com/article", - "options": { - "includeMarkdown": true, - "extractImages": true, - "maxLength": 5000 - } -} - -// Response -{ - "success": true, - "content": { - "title": "Understanding Quantum Computing", - "author": "Dr. Jane Smith", - "publishedDate": "2025-01-15", - "text": "Quantum computing represents a fundamental...", - "markdown": "# Understanding Quantum Computing\n\nQuantum computing...", - "wordCount": 1523, - "readingTime": 7, - "images": [ - { "url": "https://...", "alt": "Qubit diagram" } - ] - }, - "meta": { - "url": "https://example.com/article", - "duration": 856, - "cached": false, - "contentType": "text/html" - } -} -``` - -### 3.3 Bulk Operations - -```typescript -// ============================================ -// BULK ENDPOINTS -// ============================================ - -/** - * Mehrere URLs gleichzeitig extrahieren - * POST /api/v1/extract/bulk - */ -interface BulkExtractRequest { - urls: string[]; // Max 20 URLs - options?: ExtractOptions; // Gleiche Optionen für alle - concurrency?: number; // Parallele Requests (default: 5) -} - -interface BulkExtractResponse { - results: Array<{ - url: string; - success: boolean; - content?: ExtractedContent; - error?: string; - }>; - meta: { - total: number; - successful: number; - failed: number; - duration: number; - }; -} - -/** - * Suche + Extraktion in einem Schritt - * POST /api/v1/search-and-extract - */ -interface SearchAndExtractRequest { - query: string; - searchOptions?: SearchOptions; - extractOptions?: ExtractOptions; - extractLimit?: number; // Wie viele Results extrahieren (default: 5) -} - -interface SearchAndExtractResponse { - results: Array; - meta: SearchMeta & { - extracted: number; - extractFailed: number; - }; -} -``` - -### 3.4 Admin/Health Endpoints - -```typescript -// ============================================ -// ADMIN ENDPOINTS -// ============================================ - -// Health Check -GET /health -Response: { - status: "ok" | "degraded" | "error", - service: "mana-search", - version: "1.0.0", - timestamp: "2025-01-28T12:00:00Z", - components: { - searxng: { status: "ok", latency: 45 }, - redis: { status: "ok", latency: 2 }, - extraction: { status: "ok" } - } -} - -// SearXNG Engine Status -GET /api/v1/admin/engines -Response: { - engines: [ - { name: "google", status: "ok", avgLatency: 234 }, - { name: "bing", status: "ok", avgLatency: 189 }, - { name: "duckduckgo", status: "degraded", avgLatency: 1234 }, - // ... - ] -} - -// Cache Stats -GET /api/v1/admin/cache/stats -Response: { - hits: 12453, - misses: 3421, - hitRate: 0.78, - size: "45MB", - keys: 8234 -} - -// Cache leeren -DELETE /api/v1/admin/cache -Response: { cleared: true, keysRemoved: 8234 } - -// Prometheus Metrics -GET /metrics -Response: # Prometheus format -mana_search_requests_total{endpoint="/search"} 12453 -mana_search_latency_seconds{quantile="0.5"} 1.2 -mana_search_cache_hits_total 9842 -mana_search_extraction_success_total 8234 -... -``` - ---- - -## 4. Implementierung - -### 4.1 Search Service - -```typescript -// src/search/search.service.ts -import { Injectable, Logger } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { CacheService } from '../cache/cache.service'; -import { SearxngProvider } from './providers/searxng.provider'; -import { SearchRequest, SearchResponse } from './dto'; - -@Injectable() -export class SearchService { - private readonly logger = new Logger(SearchService.name); - - constructor( - private readonly configService: ConfigService, - private readonly cacheService: CacheService, - private readonly searxngProvider: SearxngProvider, - ) {} - - async search(request: SearchRequest): Promise { - const startTime = Date.now(); - - // 1. Cache prüfen - const cacheKey = this.buildCacheKey(request); - if (request.cache?.enabled !== false) { - const cached = await this.cacheService.get(cacheKey); - if (cached) { - this.logger.debug(`Cache hit for: ${request.query}`); - return { ...cached, meta: { ...cached.meta, cached: true } }; - } - } - - // 2. SearXNG abfragen - const results = await this.searxngProvider.search({ - q: request.query, - categories: request.options?.categories?.join(','), - engines: request.options?.engines?.join(','), - language: request.options?.language || 'de-DE', - time_range: request.options?.timeRange, - safesearch: request.options?.safeSearch ?? 0, - format: 'json', - }); - - // 3. Results normalisieren & ranken - const normalizedResults = this.normalizeResults(results, request.options?.limit); - - // 4. Response bauen - const response: SearchResponse = { - results: normalizedResults, - meta: { - query: request.query, - totalResults: normalizedResults.length, - engines: [...new Set(normalizedResults.map(r => r.engine))], - duration: Date.now() - startTime, - cached: false, - cacheKey, - }, - }; - - // 5. Cachen - if (request.cache?.enabled !== false) { - const ttl = request.cache?.ttl || this.configService.get('cache.ttl', 3600); - await this.cacheService.set(cacheKey, response, ttl); - } - - return response; - } - - private buildCacheKey(request: SearchRequest): string { - const parts = [ - 'search', - request.query, - request.options?.categories?.sort().join('-') || 'all', - request.options?.engines?.sort().join('-') || 'all', - request.options?.language || 'de-DE', - request.options?.timeRange || 'any', - ]; - return parts.join(':'); - } - - private normalizeResults( - rawResults: SearxngResult[], - limit = 10, - ): SearchResult[] { - return rawResults - .map((r) => ({ - url: r.url, - title: r.title, - snippet: r.content || '', - engine: r.engine, - score: r.score || 0.5, - publishedDate: r.publishedDate, - thumbnail: r.thumbnail, - category: r.category || 'general', - })) - .sort((a, b) => b.score - a.score) - .slice(0, Math.min(limit, 50)); - } -} -``` - -### 4.2 SearXNG Provider - -```typescript -// src/search/providers/searxng.provider.ts -import { Injectable, Logger, HttpException, HttpStatus } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; - -interface SearxngQuery { - q: string; - categories?: string; - engines?: string; - language?: string; - time_range?: string; - safesearch?: number; - format: 'json'; -} - -interface SearxngResponse { - query: string; - results: SearxngResult[]; - suggestions: string[]; - infoboxes: any[]; -} - -export interface SearxngResult { - url: string; - title: string; - content?: string; - engine: string; - score?: number; - category?: string; - publishedDate?: string; - thumbnail?: string; -} - -@Injectable() -export class SearxngProvider { - private readonly logger = new Logger(SearxngProvider.name); - private readonly baseUrl: string; - - constructor(private readonly configService: ConfigService) { - this.baseUrl = this.configService.get('searxng.url', 'http://searxng:8080'); - } - - async search(query: SearxngQuery): Promise { - const url = new URL('/search', this.baseUrl); - - // Query-Parameter setzen - Object.entries(query).forEach(([key, value]) => { - if (value !== undefined) { - url.searchParams.set(key, String(value)); - } - }); - - this.logger.debug(`SearXNG request: ${url.toString()}`); - - try { - const response = await fetch(url.toString(), { - method: 'GET', - headers: { - Accept: 'application/json', - }, - signal: AbortSignal.timeout(15000), // 15s timeout - }); - - if (!response.ok) { - throw new HttpException( - `SearXNG error: ${response.status}`, - HttpStatus.BAD_GATEWAY, - ); - } - - const data: SearxngResponse = await response.json(); - return data.results; - } catch (error) { - this.logger.error(`SearXNG search failed: ${error.message}`); - throw new HttpException( - 'Search service unavailable', - HttpStatus.SERVICE_UNAVAILABLE, - ); - } - } - - async healthCheck(): Promise<{ status: string; latency: number }> { - const start = Date.now(); - try { - const response = await fetch(`${this.baseUrl}/healthz`, { - signal: AbortSignal.timeout(5000), - }); - return { - status: response.ok ? 'ok' : 'error', - latency: Date.now() - start, - }; - } catch { - return { status: 'error', latency: Date.now() - start }; - } - } -} -``` - -### 4.3 Content Extractor - -```typescript -// src/extract/extract.service.ts -import { Injectable, Logger } from '@nestjs/common'; -import { extract } from '@extractus/article-extractor'; -import { CacheService } from '../cache/cache.service'; -import { ExtractRequest, ExtractResponse, ExtractedContent } from './dto'; -import TurndownService from 'turndown'; - -@Injectable() -export class ExtractService { - private readonly logger = new Logger(ExtractService.name); - private readonly turndown = new TurndownService(); - - constructor(private readonly cacheService: CacheService) { - // Turndown konfigurieren - this.turndown.addRule('codeBlocks', { - filter: ['pre', 'code'], - replacement: (content) => `\`\`\`\n${content}\n\`\`\``, - }); - } - - async extract(request: ExtractRequest): Promise { - const startTime = Date.now(); - - // Cache prüfen - const cacheKey = `extract:${request.url}`; - const cached = await this.cacheService.get(cacheKey); - if (cached) { - return { ...cached, meta: { ...cached.meta, cached: true } }; - } - - try { - // Artikel extrahieren - const article = await extract(request.url, { - timeout: request.options?.timeout || 10000, - }); - - if (!article) { - return { - success: false, - error: 'Could not extract content from URL', - meta: { - url: request.url, - duration: Date.now() - startTime, - cached: false, - contentType: 'unknown', - }, - }; - } - - // Content aufbereiten - let text = article.content || ''; - text = this.cleanText(text); - - // Optional: Länge begrenzen - if (request.options?.maxLength && text.length > request.options.maxLength) { - text = text.substring(0, request.options.maxLength) + '...'; - } - - const content: ExtractedContent = { - title: article.title || '', - description: article.description, - author: article.author, - publishedDate: article.published, - siteName: article.source, - - text, - wordCount: text.split(/\s+/).length, - readingTime: Math.ceil(text.split(/\s+/).length / 200), - - ogImage: article.image, - language: article.language, - }; - - // Markdown generieren - if (request.options?.includeMarkdown) { - content.markdown = this.turndown.turndown(article.content || ''); - } - - // HTML beibehalten - if (request.options?.includeHtml) { - content.html = article.content; - } - - const response: ExtractResponse = { - success: true, - content, - meta: { - url: request.url, - duration: Date.now() - startTime, - cached: false, - contentType: 'text/html', - }, - }; - - // Cachen (24h) - await this.cacheService.set(cacheKey, response, 86400); - - return response; - } catch (error) { - this.logger.error(`Extraction failed for ${request.url}: ${error.message}`); - return { - success: false, - error: error.message, - meta: { - url: request.url, - duration: Date.now() - startTime, - cached: false, - contentType: 'unknown', - }, - }; - } - } - - private cleanText(html: string): string { - // HTML-Tags entfernen, aber Struktur erhalten - return html - .replace(/)<[^<]*)*<\/script>/gi, '') - .replace(/)<[^<]*)*<\/style>/gi, '') - .replace(/<[^>]+>/g, ' ') - .replace(/\s+/g, ' ') - .trim(); - } -} -``` - -### 4.4 Cache Service - -```typescript -// src/cache/cache.service.ts -import { Injectable, Logger, OnModuleInit } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import Redis from 'ioredis'; - -@Injectable() -export class CacheService implements OnModuleInit { - private readonly logger = new Logger(CacheService.name); - private client: Redis; - - private stats = { - hits: 0, - misses: 0, - }; - - constructor(private readonly configService: ConfigService) {} - - async onModuleInit() { - this.client = new Redis({ - host: this.configService.get('redis.host', 'redis'), - port: this.configService.get('redis.port', 6379), - keyPrefix: 'mana-search:', - }); - - this.client.on('error', (err) => { - this.logger.error(`Redis error: ${err.message}`); - }); - - this.client.on('connect', () => { - this.logger.log('Connected to Redis'); - }); - } - - async get(key: string): Promise { - try { - const data = await this.client.get(key); - if (data) { - this.stats.hits++; - return JSON.parse(data); - } - this.stats.misses++; - return null; - } catch (error) { - this.logger.error(`Cache get error: ${error.message}`); - return null; - } - } - - async set(key: string, value: any, ttlSeconds: number): Promise { - try { - await this.client.setex(key, ttlSeconds, JSON.stringify(value)); - } catch (error) { - this.logger.error(`Cache set error: ${error.message}`); - } - } - - async delete(key: string): Promise { - await this.client.del(key); - } - - async clear(): Promise { - const keys = await this.client.keys('mana-search:*'); - if (keys.length > 0) { - await this.client.del(...keys); - } - return keys.length; - } - - getStats() { - const total = this.stats.hits + this.stats.misses; - return { - hits: this.stats.hits, - misses: this.stats.misses, - hitRate: total > 0 ? this.stats.hits / total : 0, - }; - } - - async healthCheck(): Promise<{ status: string; latency: number }> { - const start = Date.now(); - try { - await this.client.ping(); - return { status: 'ok', latency: Date.now() - start }; - } catch { - return { status: 'error', latency: Date.now() - start }; - } - } -} -``` - ---- - -## 5. Client-Integration - -### 5.1 Shared Package - -```typescript -// packages/shared-search-client/src/index.ts -export * from './search-client'; -export * from './types'; - -// packages/shared-search-client/src/types.ts -export interface SearchOptions { - categories?: ('general' | 'news' | 'science' | 'it')[]; - engines?: string[]; - language?: string; - timeRange?: 'day' | 'week' | 'month' | 'year'; - limit?: number; -} - -export interface SearchResult { - url: string; - title: string; - snippet: string; - engine: string; - score: number; - publishedDate?: string; -} - -export interface ExtractOptions { - includeMarkdown?: boolean; - maxLength?: number; -} - -export interface ExtractedContent { - title: string; - text: string; - markdown?: string; - wordCount: number; - author?: string; - publishedDate?: string; -} - -// packages/shared-search-client/src/search-client.ts -export class ManaSearchClient { - constructor(private readonly baseUrl: string) {} - - async search(query: string, options?: SearchOptions): Promise { - const response = await fetch(`${this.baseUrl}/api/v1/search`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ query, options }), - }); - - if (!response.ok) { - throw new Error(`Search failed: ${response.status}`); - } - - const data = await response.json(); - return data.results; - } - - async extract(url: string, options?: ExtractOptions): Promise { - const response = await fetch(`${this.baseUrl}/api/v1/extract`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ url, options }), - }); - - if (!response.ok) { - throw new Error(`Extract failed: ${response.status}`); - } - - const data = await response.json(); - if (!data.success) { - throw new Error(data.error); - } - - return data.content; - } - - async searchAndExtract( - query: string, - searchOptions?: SearchOptions, - extractOptions?: ExtractOptions, - extractLimit = 5, - ): Promise> { - const response = await fetch(`${this.baseUrl}/api/v1/search-and-extract`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - query, - searchOptions, - extractOptions, - extractLimit, - }), - }); - - if (!response.ok) { - throw new Error(`Search-and-extract failed: ${response.status}`); - } - - const data = await response.json(); - return data.results; - } -} -``` - -### 5.2 Nutzung in Questions App - -```typescript -// apps/questions/apps/backend/src/research/research.service.ts -import { Injectable } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { ManaSearchClient } from '@manacore/shared-search-client'; - -@Injectable() -export class ResearchService { - private readonly searchClient: ManaSearchClient; - - constructor(private readonly configService: ConfigService) { - this.searchClient = new ManaSearchClient( - this.configService.get('MANA_SEARCH_URL', 'http://mana-search:3021'), - ); - } - - async research(question: Question, depth: ResearchDepth) { - // 1. Suchen + Extrahieren in einem Schritt - const results = await this.searchClient.searchAndExtract( - question.title, - { - categories: ['general', 'science'], - limit: this.getLimit(depth), - }, - { includeMarkdown: true }, - this.getExtractLimit(depth), - ); - - // 2. AI Synthesis mit extrahiertem Content - const synthesis = await this.aiService.synthesize( - question, - results.map(r => ({ - url: r.url, - title: r.title, - content: r.content?.text || r.snippet, - })), - ); - - return synthesis; - } - - private getLimit(depth: ResearchDepth): number { - return { quick: 5, standard: 10, deep: 20 }[depth]; - } - - private getExtractLimit(depth: ResearchDepth): number { - return { quick: 2, standard: 5, deep: 10 }[depth]; - } -} -``` - -### 5.3 Nutzung in Chat App - -```typescript -// apps/chat/apps/backend/src/ai/web-grounding.service.ts -import { Injectable } from '@nestjs/common'; -import { ManaSearchClient } from '@manacore/shared-search-client'; - -@Injectable() -export class WebGroundingService { - private readonly searchClient: ManaSearchClient; - - constructor() { - this.searchClient = new ManaSearchClient( - process.env.MANA_SEARCH_URL || 'http://mana-search:3021', - ); - } - - async groundWithWebSearch(userMessage: string): Promise { - // Suche durchführen - const results = await this.searchClient.search(userMessage, { - categories: ['general'], - limit: 5, - }); - - // Context für AI bauen - const context = results - .map((r, i) => `[${i + 1}] ${r.title}\n${r.snippet}\nSource: ${r.url}`) - .join('\n\n'); - - return context; - } -} -``` - ---- - -## 6. Deployment - -### 6.1 Production Docker Compose - -```yaml -# docker-compose.yml (Root-Level, Production) -services: - mana-search: - image: ghcr.io/manacore/mana-search:latest - restart: unless-stopped - environment: - NODE_ENV: production - PORT: 3021 - SEARXNG_URL: http://searxng:8080 - REDIS_HOST: redis - REDIS_PORT: 6379 - CACHE_TTL: 3600 - depends_on: - - searxng - - redis - networks: - - manacore-network - labels: - - "traefik.enable=true" - - "traefik.http.routers.search.rule=Host(`search.mana.internal`)" - - "traefik.http.services.search.loadbalancer.server.port=3021" - - searxng: - image: searxng/searxng:latest - restart: unless-stopped - volumes: - - ./services/mana-search/searxng/settings.yml:/etc/searxng/settings.yml:ro - - ./services/mana-search/searxng/limiter.toml:/etc/searxng/limiter.toml:ro - environment: - SEARXNG_SECRET: ${SEARXNG_SECRET} - networks: - - manacore-network - # Kein externes Port-Mapping - nur intern -``` - -### 6.2 Development Script - -```bash -# Root package.json scripts -{ - "scripts": { - "dev:search": "docker-compose -f services/mana-search/docker-compose.yml up -d", - "dev:search:logs": "docker-compose -f services/mana-search/docker-compose.yml logs -f", - "dev:search:stop": "docker-compose -f services/mana-search/docker-compose.yml down" - } -} -``` - -### 6.3 Environment Variables - -```env -# .env.development -MANA_SEARCH_URL=http://localhost:3021 - -# Production -MANA_SEARCH_URL=http://mana-search:3021 -SEARXNG_SECRET=your-production-secret -``` - ---- - -## 7. Monitoring - -### 7.1 Prometheus Metrics - -```typescript -// src/metrics/metrics.service.ts -import { Injectable } from '@nestjs/common'; -import { Counter, Histogram, Gauge, Registry } from 'prom-client'; - -@Injectable() -export class MetricsService { - private readonly registry = new Registry(); - - // Request Counter - private readonly requestsTotal = new Counter({ - name: 'mana_search_requests_total', - help: 'Total number of search requests', - labelNames: ['endpoint', 'status'], - registers: [this.registry], - }); - - // Latency Histogram - private readonly latency = new Histogram({ - name: 'mana_search_latency_seconds', - help: 'Request latency in seconds', - labelNames: ['endpoint'], - buckets: [0.1, 0.5, 1, 2, 5, 10], - registers: [this.registry], - }); - - // Cache Metrics - private readonly cacheHits = new Counter({ - name: 'mana_search_cache_hits_total', - help: 'Total cache hits', - registers: [this.registry], - }); - - private readonly cacheMisses = new Counter({ - name: 'mana_search_cache_misses_total', - help: 'Total cache misses', - registers: [this.registry], - }); - - // SearXNG Engine Status - private readonly engineStatus = new Gauge({ - name: 'mana_search_engine_status', - help: 'SearXNG engine status (1=ok, 0=error)', - labelNames: ['engine'], - registers: [this.registry], - }); - - recordRequest(endpoint: string, status: number, duration: number) { - this.requestsTotal.inc({ endpoint, status: String(status) }); - this.latency.observe({ endpoint }, duration / 1000); - } - - recordCacheHit() { - this.cacheHits.inc(); - } - - recordCacheMiss() { - this.cacheMisses.inc(); - } - - async getMetrics(): Promise { - return this.registry.metrics(); - } -} -``` - -### 7.2 Grafana Dashboard - -```json -{ - "title": "Mana Search Service", - "panels": [ - { - "title": "Requests/min", - "type": "graph", - "targets": [ - { - "expr": "rate(mana_search_requests_total[1m])", - "legendFormat": "{{endpoint}}" - } - ] - }, - { - "title": "Latency p95", - "type": "graph", - "targets": [ - { - "expr": "histogram_quantile(0.95, rate(mana_search_latency_seconds_bucket[5m]))", - "legendFormat": "{{endpoint}}" - } - ] - }, - { - "title": "Cache Hit Rate", - "type": "gauge", - "targets": [ - { - "expr": "rate(mana_search_cache_hits_total[5m]) / (rate(mana_search_cache_hits_total[5m]) + rate(mana_search_cache_misses_total[5m]))" - } - ] - } - ] -} -``` - ---- - -## 8. Implementierungs-Roadmap - -### Phase 1: Core Setup (3-4 Tage) - -``` -[ ] Projekt-Struktur erstellen - [ ] services/mana-search/ - [ ] NestJS Basis-Setup - [ ] Docker Compose - -[ ] SearXNG Setup - [ ] Dockerfile/Image - [ ] settings.yml mit Engines - [ ] limiter.toml - [ ] Health-Check - -[ ] Redis Setup - [ ] Docker Container - [ ] Connection Service -``` - -### Phase 2: Search API (2-3 Tage) - -``` -[ ] SearXNG Provider - [ ] API-Integration - [ ] Error Handling - [ ] Retry Logic - -[ ] Search Service - [ ] Query-Verarbeitung - [ ] Result-Normalisierung - [ ] Caching - -[ ] Search Controller - [ ] POST /api/v1/search - [ ] Validation (DTOs) -``` - -### Phase 3: Extract API (2-3 Tage) - -``` -[ ] Article Extractor - [ ] @extractus/article-extractor Integration - [ ] Markdown Conversion - [ ] Error Handling - -[ ] Extract Service - [ ] URL-Verarbeitung - [ ] Caching - -[ ] Bulk Operations - [ ] POST /api/v1/extract/bulk - [ ] POST /api/v1/search-and-extract -``` - -### Phase 4: Shared Package (1-2 Tage) - -``` -[ ] @manacore/shared-search-client - [ ] ManaSearchClient Klasse - [ ] TypeScript Types - [ ] NPM Package Config -``` - -### Phase 5: Monitoring & Polish (1-2 Tage) - -``` -[ ] Health Endpoints -[ ] Prometheus Metrics -[ ] Grafana Dashboard -[ ] Dokumentation (CLAUDE.md) -[ ] Root package.json Scripts -``` - ---- - -## 9. Offene Fragen - -1. **Tor-Integration**: Soll SearXNG über Tor laufen für mehr Anonymität? -2. **API-Key-Auth**: Braucht der Service Authentifizierung oder ist er nur intern? -3. **Fallback-Provider**: Soll bei SearXNG-Ausfall automatisch auf Brave API gewechselt werden? -4. **Content-Limits**: Maximale Textlänge für Extraktion? - ---- - -## 10. Ressourcen-Schätzung - -| Komponente | RAM | CPU | Storage | -|------------|-----|-----|---------| -| mana-search (NestJS) | 256MB | 0.25 | - | -| SearXNG | 512MB | 0.5 | - | -| Redis | 128MB | 0.1 | 100MB | -| **Total** | **~1GB** | **~1 vCPU** | **100MB** | - -Passt problemlos auf bestehende Infrastruktur (Mac Mini). - ---- - -*Dokument-Status: Entwurf - Zur Überprüfung* diff --git a/.claude/plans/questions-app.md b/.claude/plans/questions-app.md deleted file mode 100644 index e4b12c8d0..000000000 --- a/.claude/plans/questions-app.md +++ /dev/null @@ -1,1471 +0,0 @@ -# ManaCore Questions App - Design Document - -> **Status**: Entwurf - Zur Überprüfung -> **Erstellt**: 2025-01-28 -> **Autor**: Claude Code - ---- - -## 1. Übersicht & Ziele - -### 1.1 Was ist die Questions App? - -Eine intelligente Fragen-Management-App, die Nutzern hilft: -- **Offene Fragen sammeln** - Gedanken, Ideen und Wissenslücken festhalten -- **KI-gestützte Recherche** - Automatische Recherche zu den Fragen durchführen -- **Wissen organisieren** - Antworten, Quellen und Erkenntnisse strukturiert speichern -- **Lernfortschritt tracken** - Verstehen, welche Fragen beantwortet wurden - -### 1.2 Use Cases - -| Use Case | Beschreibung | -|----------|--------------| -| **Lern-Begleiter** | Student notiert Fragen während der Vorlesung, App recherchiert später | -| **Recherche-Tool** | Journalist sammelt Fragen zu einem Thema für Deep-Dive | -| **Wissens-Management** | Professional trackt offene Fragen und deren Beantwortung | -| **Curiosity Journal** | Privatperson sammelt alltägliche "Ich frage mich..."-Momente | - -### 1.3 Kernfeatures - -``` -┌─────────────────────────────────────────────────────────────────────────┐ -│ QUESTIONS APP FEATURES │ -├─────────────────────────────────────────────────────────────────────────┤ -│ │ -│ [1] FRAGEN-MANAGEMENT │ -│ ├── Schnelles Erfassen von Fragen │ -│ ├── Kategorisierung & Tagging │ -│ ├── Priorisierung (dringend, wichtig, irgendwann) │ -│ └── Status-Tracking (offen, in Recherche, beantwortet) │ -│ │ -│ [2] KI-RECHERCHE │ -│ ├── Automatische Web-Recherche │ -│ ├── Zusammenfassung von Quellen │ -│ ├── Quellenangaben & Zitate │ -│ └── Follow-up Fragen generieren │ -│ │ -│ [3] WISSENS-ORGANISATION │ -│ ├── Antworten strukturiert speichern │ -│ ├── Quellen verwalten & kategorisieren │ -│ ├── Notizen & eigene Erkenntnisse hinzufügen │ -│ └── Export als Markdown/PDF │ -│ │ -│ [4] KOLLABORATION │ -│ ├── Fragen-Sammlungen teilen │ -│ ├── Team-Recherche │ -│ └── Diskussionen zu Fragen │ -│ │ -└─────────────────────────────────────────────────────────────────────────┘ -``` - -### 1.4 Abgrenzung zu anderen Apps - -| App | Fokus | Unterschied zu Questions | -|-----|-------|-------------------------| -| **Chat** | Konversation mit AI | Questions: Strukturierte Fragen-DB statt Chatverläufe | -| **Todo** | Aufgaben | Questions: Wissenslücken, nicht Todos | -| **Zitare** | Zitate sammeln | Questions: Aktive Recherche, nicht passive Sammlung | - ---- - -## 2. Architektur - -### 2.1 Projekt-Struktur - -``` -apps/questions/ -├── apps/ -│ ├── backend/ # NestJS API (Port 3010) -│ │ ├── src/ -│ │ │ ├── question/ # Fragen CRUD -│ │ │ ├── research/ # KI-Recherche -│ │ │ ├── source/ # Quellen-Management -│ │ │ ├── collection/ # Fragen-Sammlungen -│ │ │ ├── answer/ # Antworten & Notizen -│ │ │ ├── db/ # Drizzle Schema & Migrations -│ │ │ └── health/ # Health-Checks -│ │ └── drizzle/ -│ ├── mobile/ # Expo React Native App -│ │ └── app/ -│ │ ├── (tabs)/ # Tab-Navigation -│ │ ├── question/ # Frage-Detail -│ │ └── research/ # Recherche-Ansicht -│ ├── web/ # SvelteKit Web App -│ │ └── src/ -│ │ ├── routes/ -│ │ │ ├── (app)/ # Authenticated Routes -│ │ │ └── (auth)/ # Login/Register -│ │ ├── lib/ -│ │ │ ├── components/ -│ │ │ └── stores/ -│ │ └── app.css -│ └── landing/ # Astro Landing Page -├── packages/ -│ └── shared/ # Shared Types & Utils -└── package.json -``` - -### 2.2 Technologie-Stack - -| Layer | Technologie | Details | -|-------|-------------|---------| -| **Backend** | NestJS 10 | REST API, Drizzle ORM, PostgreSQL | -| **Web** | SvelteKit 2 + Svelte 5 | Runes Mode, Tailwind CSS | -| **Mobile** | Expo SDK 54, React Native | NativeWind, Expo Router | -| **Landing** | Astro 5 | Static Site, Tailwind CSS | -| **Auth** | Mana Core Auth | JWT (EdDSA), Port 3001 | -| **AI** | Ollama + OpenRouter | Dual-Provider (wie Chat) | -| **Search** | Web Search API | Externe Such-API für Recherche | - -### 2.3 System-Architektur - -``` -┌─────────────┐ ┌─────────────┐ ┌────────────────┐ -│ Client │────>│ Questions │────>│ mana-core-auth │ -│ (Web/Mobile)│ │ Backend │ │ (port 3001) │ -└─────────────┘ │ (port 3010) │ └────────────────┘ - └──────┬──────┘ - │ - ┌──────────────┼──────────────┐ - ▼ ▼ ▼ - ┌──────────┐ ┌──────────┐ ┌──────────┐ - │ Ollama │ │OpenRouter│ │ Web │ - │ (local) │ │ (cloud) │ │ Search │ - └──────────┘ └──────────┘ └──────────┘ -``` - ---- - -## 3. Datenbank-Schema - -### 3.1 Core Tables - -```sql --- ============================================ --- QUESTIONS SCHEMA --- ============================================ - --- Fragen (Haupttabelle) -CREATE TABLE questions ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - user_id TEXT NOT NULL, -- Mana Core Auth User ID - collection_id UUID REFERENCES collections(id) ON DELETE SET NULL, - - -- Inhalt - title TEXT NOT NULL, -- Kurze Frage - description TEXT, -- Ausführliche Beschreibung/Kontext - - -- Status & Priorisierung - status TEXT NOT NULL DEFAULT 'open', -- 'open', 'researching', 'answered', 'archived' - priority TEXT DEFAULT 'normal', -- 'low', 'normal', 'high', 'urgent' - - -- Kategorisierung - tags TEXT[] DEFAULT '{}', -- User-definierte Tags - category TEXT, -- Optionale Kategorie - - -- Research Config - research_depth TEXT DEFAULT 'quick', -- 'quick', 'standard', 'deep' - auto_research BOOLEAN DEFAULT false, -- Auto-Recherche aktiviert - - -- Timestamps - created_at TIMESTAMPTZ DEFAULT now(), - updated_at TIMESTAMPTZ DEFAULT now(), - answered_at TIMESTAMPTZ, -- Wann als beantwortet markiert - - -- Soft Delete - is_archived BOOLEAN DEFAULT false, - archived_at TIMESTAMPTZ -); - -CREATE INDEX questions_user_idx ON questions(user_id); -CREATE INDEX questions_status_idx ON questions(user_id, status); -CREATE INDEX questions_collection_idx ON questions(collection_id); -CREATE INDEX questions_tags_idx ON questions USING GIN(tags); - --- Fragen-Sammlungen -CREATE TABLE collections ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - user_id TEXT NOT NULL, - - name TEXT NOT NULL, - description TEXT, - color TEXT DEFAULT '#6366f1', -- Indigo als Default - icon TEXT DEFAULT 'folder', - - -- Sharing - is_shared BOOLEAN DEFAULT false, - share_token TEXT UNIQUE, -- Für öffentliche Links - - created_at TIMESTAMPTZ DEFAULT now(), - updated_at TIMESTAMPTZ DEFAULT now() -); - -CREATE INDEX collections_user_idx ON collections(user_id); - --- KI-Recherche-Ergebnisse -CREATE TABLE research_results ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - question_id UUID NOT NULL REFERENCES questions(id) ON DELETE CASCADE, - - -- Recherche-Metadaten - model_id TEXT NOT NULL, -- Verwendetes AI-Modell - provider TEXT NOT NULL, -- 'ollama' oder 'openrouter' - research_depth TEXT NOT NULL, -- 'quick', 'standard', 'deep' - - -- Ergebnis - summary TEXT NOT NULL, -- KI-generierte Zusammenfassung - key_points JSONB DEFAULT '[]', -- Wichtige Erkenntnisse als Array - follow_up_questions TEXT[] DEFAULT '{}', -- Vorgeschlagene Folgefragen - - -- Token-Tracking (wie bei Chat) - prompt_tokens INTEGER, - completion_tokens INTEGER, - estimated_cost DECIMAL(10, 6), - - -- Timestamps - created_at TIMESTAMPTZ DEFAULT now(), - duration_ms INTEGER -- Dauer der Recherche -); - -CREATE INDEX research_results_question_idx ON research_results(question_id); - --- Quellen (von Recherche gefunden) -CREATE TABLE sources ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - research_id UUID NOT NULL REFERENCES research_results(id) ON DELETE CASCADE, - question_id UUID NOT NULL REFERENCES questions(id) ON DELETE CASCADE, - - -- Quelle - url TEXT, - title TEXT NOT NULL, - source_type TEXT DEFAULT 'web', -- 'web', 'book', 'paper', 'video', 'manual' - - -- Inhalt - excerpt TEXT, -- Relevanter Ausschnitt - summary TEXT, -- KI-Zusammenfassung - - -- Bewertung - relevance_score REAL, -- 0.0 - 1.0 - user_rating INTEGER, -- 1-5 Sterne (optional) - - -- Metadaten - author TEXT, - published_at DATE, - accessed_at TIMESTAMPTZ DEFAULT now(), - - created_at TIMESTAMPTZ DEFAULT now() -); - -CREATE INDEX sources_research_idx ON sources(research_id); -CREATE INDEX sources_question_idx ON sources(question_id); - --- Antworten & Notizen (User-generiert) -CREATE TABLE answers ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - question_id UUID NOT NULL REFERENCES questions(id) ON DELETE CASCADE, - user_id TEXT NOT NULL, - - -- Inhalt - content TEXT NOT NULL, -- Markdown-formatierte Antwort - answer_type TEXT DEFAULT 'note', -- 'note', 'answer', 'insight' - - -- Optional: Basiert auf Recherche - research_id UUID REFERENCES research_results(id), - source_ids UUID[] DEFAULT '{}', -- Referenzierte Quellen - - created_at TIMESTAMPTZ DEFAULT now(), - updated_at TIMESTAMPTZ DEFAULT now() -); - -CREATE INDEX answers_question_idx ON answers(question_id); -CREATE INDEX answers_user_idx ON answers(user_id); - --- Frage-Verlauf (Änderungen tracken) -CREATE TABLE question_history ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - question_id UUID NOT NULL REFERENCES questions(id) ON DELETE CASCADE, - - field_changed TEXT NOT NULL, -- 'status', 'title', 'description', etc. - old_value TEXT, - new_value TEXT, - changed_by TEXT NOT NULL, -- User ID - - created_at TIMESTAMPTZ DEFAULT now() -); - -CREATE INDEX question_history_question_idx ON question_history(question_id); -``` - -### 3.2 Drizzle Schema (TypeScript) - -```typescript -// db/schema/questions.schema.ts -import { pgTable, uuid, text, boolean, timestamp, real, integer, jsonb } from 'drizzle-orm/pg-core'; - -export const questions = pgTable('questions', { - id: uuid('id').primaryKey().defaultRandom(), - userId: text('user_id').notNull(), - collectionId: uuid('collection_id').references(() => collections.id, { onDelete: 'set null' }), - - title: text('title').notNull(), - description: text('description'), - - status: text('status').notNull().default('open'), - priority: text('priority').default('normal'), - - tags: text('tags').array().default([]), - category: text('category'), - - researchDepth: text('research_depth').default('quick'), - autoResearch: boolean('auto_research').default(false), - - createdAt: timestamp('created_at', { withTimezone: true }).defaultNow(), - updatedAt: timestamp('updated_at', { withTimezone: true }).defaultNow(), - answeredAt: timestamp('answered_at', { withTimezone: true }), - - isArchived: boolean('is_archived').default(false), - archivedAt: timestamp('archived_at', { withTimezone: true }), -}); - -export const collections = pgTable('collections', { - id: uuid('id').primaryKey().defaultRandom(), - userId: text('user_id').notNull(), - - name: text('name').notNull(), - description: text('description'), - color: text('color').default('#6366f1'), - icon: text('icon').default('folder'), - - isShared: boolean('is_shared').default(false), - shareToken: text('share_token').unique(), - - createdAt: timestamp('created_at', { withTimezone: true }).defaultNow(), - updatedAt: timestamp('updated_at', { withTimezone: true }).defaultNow(), -}); - -export const researchResults = pgTable('research_results', { - id: uuid('id').primaryKey().defaultRandom(), - questionId: uuid('question_id').notNull().references(() => questions.id, { onDelete: 'cascade' }), - - modelId: text('model_id').notNull(), - provider: text('provider').notNull(), - researchDepth: text('research_depth').notNull(), - - summary: text('summary').notNull(), - keyPoints: jsonb('key_points').default([]), - followUpQuestions: text('follow_up_questions').array().default([]), - - promptTokens: integer('prompt_tokens'), - completionTokens: integer('completion_tokens'), - estimatedCost: real('estimated_cost'), - - createdAt: timestamp('created_at', { withTimezone: true }).defaultNow(), - durationMs: integer('duration_ms'), -}); - -export const sources = pgTable('sources', { - id: uuid('id').primaryKey().defaultRandom(), - researchId: uuid('research_id').notNull().references(() => researchResults.id, { onDelete: 'cascade' }), - questionId: uuid('question_id').notNull().references(() => questions.id, { onDelete: 'cascade' }), - - url: text('url'), - title: text('title').notNull(), - sourceType: text('source_type').default('web'), - - excerpt: text('excerpt'), - summary: text('summary'), - - relevanceScore: real('relevance_score'), - userRating: integer('user_rating'), - - author: text('author'), - publishedAt: timestamp('published_at'), - accessedAt: timestamp('accessed_at', { withTimezone: true }).defaultNow(), - - createdAt: timestamp('created_at', { withTimezone: true }).defaultNow(), -}); - -export const answers = pgTable('answers', { - id: uuid('id').primaryKey().defaultRandom(), - questionId: uuid('question_id').notNull().references(() => questions.id, { onDelete: 'cascade' }), - userId: text('user_id').notNull(), - - content: text('content').notNull(), - answerType: text('answer_type').default('note'), - - researchId: uuid('research_id').references(() => researchResults.id), - sourceIds: uuid('source_ids').array().default([]), - - createdAt: timestamp('created_at', { withTimezone: true }).defaultNow(), - updatedAt: timestamp('updated_at', { withTimezone: true }).defaultNow(), -}); -``` - ---- - -## 4. API-Endpoints - -### 4.1 Questions Controller - -```typescript -// ============================================ -// QUESTIONS ENDPOINTS -// ============================================ - -// Liste aller Fragen -GET /api/v1/questions -Query: ?status=open&priority=high&collection_id=xxx&tags=tag1,tag2&search=text&limit=20&offset=0 -Response: { - questions: Question[], - pagination: { total, limit, offset } -} - -// Einzelne Frage mit Details -GET /api/v1/questions/:id -Response: { - question: Question, - researchResults: ResearchResult[], - sources: Source[], - answers: Answer[] -} - -// Frage erstellen -POST /api/v1/questions -Body: { - title: string, // Required - description?: string, - collectionId?: string, - tags?: string[], - category?: string, - priority?: 'low' | 'normal' | 'high' | 'urgent', - researchDepth?: 'quick' | 'standard' | 'deep', - autoResearch?: boolean // Direkt Recherche starten -} -Response: { question: Question } - -// Frage aktualisieren -PATCH /api/v1/questions/:id -Body: { - title?: string, - description?: string, - status?: 'open' | 'researching' | 'answered' | 'archived', - priority?: string, - tags?: string[], - category?: string, - collectionId?: string | null -} -Response: { question: Question } - -// Frage archivieren -PATCH /api/v1/questions/:id/archive -Response: { question: Question } - -// Frage wiederherstellen -PATCH /api/v1/questions/:id/unarchive -Response: { question: Question } - -// Frage als beantwortet markieren -PATCH /api/v1/questions/:id/mark-answered -Response: { question: Question } - -// Frage löschen -DELETE /api/v1/questions/:id -Response: { success: true } - -// Bulk-Operationen -POST /api/v1/questions/bulk -Body: { - action: 'archive' | 'delete' | 'move' | 'tag', - questionIds: string[], - data?: { collectionId?: string, tags?: string[] } -} -Response: { affected: number } -``` - -### 4.2 Research Controller - -```typescript -// ============================================ -// RESEARCH ENDPOINTS -// ============================================ - -// Recherche starten -POST /api/v1/research -Body: { - questionId: string, - depth?: 'quick' | 'standard' | 'deep', // Default: question.researchDepth - modelId?: string, // Optional: Override default model - additionalContext?: string // Extra Kontext für bessere Ergebnisse -} -Response: { - research: ResearchResult, - sources: Source[], - creditsUsed: number -} - -// Recherche-Status prüfen (für Polling bei async) -GET /api/v1/research/:id/status -Response: { - status: 'pending' | 'processing' | 'completed' | 'failed', - progress?: number, // 0-100 - research?: ResearchResult // Wenn completed -} - -// Recherche-Ergebnisse abrufen -GET /api/v1/research/question/:questionId -Response: { - results: ResearchResult[], - sources: Source[] -} - -// Folgefrage aus Recherche erstellen -POST /api/v1/research/:id/follow-up -Body: { - questionText: string // Vorgeschlagene oder neue Folgefrage -} -Response: { question: Question } - -// Verfügbare Modelle für Recherche -GET /api/v1/research/models -Response: { - models: Array<{ - id: string, - name: string, - provider: 'ollama' | 'openrouter', - description: string, - costPerToken?: number, - recommended: boolean - }> -} -``` - -### 4.3 Sources Controller - -```typescript -// ============================================ -// SOURCES ENDPOINTS -// ============================================ - -// Quellen einer Frage -GET /api/v1/sources/question/:questionId -Query: ?type=web&sort=relevance -Response: { sources: Source[] } - -// Manuelle Quelle hinzufügen -POST /api/v1/sources -Body: { - questionId: string, - url?: string, - title: string, - sourceType: 'web' | 'book' | 'paper' | 'video' | 'manual', - excerpt?: string, - author?: string, - publishedAt?: string -} -Response: { source: Source } - -// Quelle bewerten -PATCH /api/v1/sources/:id/rate -Body: { rating: 1 | 2 | 3 | 4 | 5 } -Response: { source: Source } - -// Quelle entfernen -DELETE /api/v1/sources/:id -Response: { success: true } -``` - -### 4.4 Answers Controller - -```typescript -// ============================================ -// ANSWERS ENDPOINTS -// ============================================ - -// Antworten einer Frage -GET /api/v1/answers/question/:questionId -Response: { answers: Answer[] } - -// Antwort/Notiz erstellen -POST /api/v1/answers -Body: { - questionId: string, - content: string, // Markdown - answerType: 'note' | 'answer' | 'insight', - researchId?: string, // Basiert auf Recherche - sourceIds?: string[] // Referenzierte Quellen -} -Response: { answer: Answer } - -// Antwort aktualisieren -PATCH /api/v1/answers/:id -Body: { - content?: string, - sourceIds?: string[] -} -Response: { answer: Answer } - -// Antwort löschen -DELETE /api/v1/answers/:id -Response: { success: true } -``` - -### 4.5 Collections Controller - -```typescript -// ============================================ -// COLLECTIONS ENDPOINTS -// ============================================ - -// Alle Sammlungen -GET /api/v1/collections -Response: { - collections: Array -} - -// Sammlung mit Fragen -GET /api/v1/collections/:id -Query: ?includeQuestions=true -Response: { - collection: Collection, - questions?: Question[] -} - -// Sammlung erstellen -POST /api/v1/collections -Body: { - name: string, - description?: string, - color?: string, - icon?: string -} -Response: { collection: Collection } - -// Sammlung aktualisieren -PATCH /api/v1/collections/:id -Body: { - name?: string, - description?: string, - color?: string, - icon?: string -} -Response: { collection: Collection } - -// Sammlung teilen -POST /api/v1/collections/:id/share -Response: { - collection: Collection, - shareUrl: string -} - -// Teilen beenden -DELETE /api/v1/collections/:id/share -Response: { collection: Collection } - -// Öffentliche Sammlung abrufen -GET /api/v1/collections/shared/:token -Response: { - collection: Collection, - questions: Question[] -} - -// Sammlung löschen -DELETE /api/v1/collections/:id -Response: { success: true } -``` - -### 4.6 Export Controller - -```typescript -// ============================================ -// EXPORT ENDPOINTS -// ============================================ - -// Frage als Markdown exportieren -GET /api/v1/export/question/:id/markdown -Response: text/markdown - -// Frage als PDF exportieren -GET /api/v1/export/question/:id/pdf -Response: application/pdf - -// Sammlung exportieren -GET /api/v1/export/collection/:id -Query: ?format=markdown|pdf|json -Response: Entsprechendes Format - -// Bibliography exportieren (alle Quellen einer Frage) -GET /api/v1/export/question/:id/bibliography -Query: ?style=apa|mla|chicago|bibtex -Response: text/plain -``` - ---- - -## 5. KI-Recherche-System - -### 5.1 Recherche-Tiefen - -| Depth | Beschreibung | Dauer | Credits | Use Case | -|-------|--------------|-------|---------|----------| -| **quick** | Schnelle Antwort, 1-2 Quellen | ~10s | 5 | Faktenfragen, Quick Check | -| **standard** | Ausführlich, 3-5 Quellen | ~30s | 15 | Normale Recherche | -| **deep** | Umfassend, 5-10 Quellen | ~60s | 30 | Tiefgehende Analyse | - -### 5.2 Recherche-Ablauf - -``` -┌─────────────────────────────────────────────────────────────────────────┐ -│ RESEARCH PIPELINE │ -├─────────────────────────────────────────────────────────────────────────┤ -│ │ -│ [1] QUERY EXPANSION │ -│ │ Frage → KI optimiert Suchbegriffe │ -│ │ "Was ist Quantum Computing?" → │ -│ │ ["quantum computing basics", "qubits explanation", │ -│ │ "quantum vs classical computing"] │ -│ ▼ │ -│ [2] WEB SEARCH │ -│ │ Suchbegriffe → Web Search API │ -│ │ → URLs + Snippets zurück │ -│ ▼ │ -│ [3] CONTENT EXTRACTION │ -│ │ URLs → Fetch & Parse HTML │ -│ │ → Clean Text extraktion │ -│ ▼ │ -│ [4] RELEVANCE SCORING │ -│ │ KI bewertet Relevanz jeder Quelle │ -│ │ → Top N Quellen auswählen │ -│ ▼ │ -│ [5] SYNTHESIS │ -│ │ KI fasst alle Quellen zusammen │ -│ │ → Strukturierte Antwort mit: │ -│ │ - Summary (Hauptantwort) │ -│ │ - Key Points (Bullet Points) │ -│ │ - Follow-up Questions │ -│ ▼ │ -│ [6] CITATION GENERATION │ -│ │ Quellen → Formatierte Zitate │ -│ │ → Zuordnung zu Aussagen │ -│ ▼ │ -│ [RESULT] │ -│ ResearchResult + Sources │ -│ │ -└─────────────────────────────────────────────────────────────────────────┘ -``` - -### 5.3 System-Prompts - -```typescript -// Query Expansion Prompt -const QUERY_EXPANSION_PROMPT = ` -Du bist ein Such-Experte. Gegeben ist eine Frage, generiere 3-5 optimierte Suchbegriffe. - -Frage: {question} -Kontext: {description} - -Generiere Suchbegriffe die: -- Die Kernfrage abdecken -- Verschiedene Aspekte beleuchten -- Für Web-Suche optimiert sind - -Ausgabe als JSON-Array: ["term1", "term2", ...] -`; - -// Synthesis Prompt -const SYNTHESIS_PROMPT = ` -Du bist ein Recherche-Assistent. Analysiere die folgenden Quellen und beantworte die Frage. - -Frage: {question} -Kontext: {description} - -Quellen: -{sources} - -Erstelle eine strukturierte Antwort mit: -1. **Summary**: Hauptantwort in 2-3 Absätzen -2. **Key Points**: 3-5 wichtige Erkenntnisse als Bullet Points -3. **Follow-up Questions**: 2-3 weiterführende Fragen - -Zitiere Quellen mit [1], [2], etc. -Ausgabe als JSON. -`; -``` - -### 5.4 Web Search Integration - -```typescript -// Research Service -@Injectable() -export class ResearchService { - constructor( - private readonly searchService: WebSearchService, - private readonly aiService: AiCompletionService, - private readonly contentExtractor: ContentExtractorService, - ) {} - - async research(questionId: string, depth: ResearchDepth): Promise { - const question = await this.getQuestion(questionId); - - // 1. Query Expansion - const searchQueries = await this.aiService.expandQuery(question); - - // 2. Web Search - const searchResults = await Promise.all( - searchQueries.map(q => this.searchService.search(q, { limit: this.getLimit(depth) })) - ); - - // 3. Content Extraction - const contents = await Promise.all( - searchResults.flat().map(r => this.contentExtractor.extract(r.url)) - ); - - // 4. Relevance Scoring - const scoredSources = await this.aiService.scoreRelevance(question, contents); - const topSources = scoredSources.slice(0, this.getSourceLimit(depth)); - - // 5. Synthesis - const synthesis = await this.aiService.synthesize(question, topSources); - - // 6. Save Results - return this.saveResults(questionId, synthesis, topSources); - } - - private getLimit(depth: ResearchDepth): number { - return { quick: 3, standard: 6, deep: 10 }[depth]; - } - - private getSourceLimit(depth: ResearchDepth): number { - return { quick: 2, standard: 5, deep: 8 }[depth]; - } -} -``` - ---- - -## 6. Web App (SvelteKit) - -### 6.1 Routen-Struktur - -``` -src/routes/ -├── (auth)/ -│ ├── login/+page.svelte -│ └── register/+page.svelte -├── (app)/ -│ ├── +layout.svelte # App Shell mit Sidebar -│ ├── +page.svelte # Dashboard / Alle Fragen -│ ├── questions/ -│ │ ├── +page.svelte # Fragen-Liste -│ │ ├── [id]/ -│ │ │ ├── +page.svelte # Frage-Detail -│ │ │ └── +page.server.ts # SSR Data Loading -│ │ └── new/+page.svelte # Neue Frage -│ ├── collections/ -│ │ ├── +page.svelte # Sammlungen-Liste -│ │ └── [id]/+page.svelte # Sammlung-Detail -│ ├── research/ -│ │ └── [id]/+page.svelte # Recherche-Ergebnis-Ansicht -│ └── settings/+page.svelte # Einstellungen -└── health/+server.ts # Health Endpoint -``` - -### 6.2 Komponenten-Übersicht - -``` -src/lib/components/ -├── questions/ -│ ├── QuestionCard.svelte # Frage in Liste -│ ├── QuestionForm.svelte # Erstellen/Bearbeiten -│ ├── QuestionDetail.svelte # Vollständige Ansicht -│ ├── QuestionFilters.svelte # Filter & Suche -│ └── QuestionStatus.svelte # Status-Badge -├── research/ -│ ├── ResearchPanel.svelte # Recherche-Ergebnisse -│ ├── ResearchProgress.svelte # Fortschritts-Anzeige -│ ├── SourceCard.svelte # Einzelne Quelle -│ └── SourceList.svelte # Quellen-Liste -├── answers/ -│ ├── AnswerEditor.svelte # Markdown-Editor -│ ├── AnswerCard.svelte # Antwort-Anzeige -│ └── AnswerList.svelte # Antworten-Liste -├── collections/ -│ ├── CollectionCard.svelte # Sammlung in Liste -│ ├── CollectionForm.svelte # Erstellen/Bearbeiten -│ └── CollectionPicker.svelte # Auswahl-Dropdown -├── common/ -│ ├── TagInput.svelte # Tag-Eingabe -│ ├── PriorityBadge.svelte # Prioritäts-Anzeige -│ ├── EmptyState.svelte # Leerer Zustand -│ └── ConfirmDialog.svelte # Bestätigungs-Dialog -└── layout/ - ├── Sidebar.svelte # Navigation - ├── Header.svelte # Top-Bar - └── QuickCapture.svelte # Schnell-Eingabe (global) -``` - -### 6.3 Haupt-Ansichten - -#### Dashboard (Fragen-Liste) - -```svelte - - - -
- - - - - - - -
- {#if questions.length === 0} - - {:else} -
- {#each questions as question (question.id)} - - {/each} -
- {/if} -
-
-``` - -#### Frage-Detail - -```svelte - - - -
- -
-
-
-

{question.title}

-
- - - {#each question.tags as tag} - {tag} - {/each} -
-
- -
- {#if question.status !== 'answered'} - - - - startResearch('quick')}> - Quick (~10s, 5 Credits) - - startResearch('standard')}> - Standard (~30s, 15 Credits) - - startResearch('deep')}> - Deep (~60s, 30 Credits) - - - - - - {/if} -
-
- - {#if question.description} -

{question.description}

- {/if} -
- - - - - - Recherche ({researchResults.length}) - - - Antworten ({answers.length}) - - - Quellen ({sources.length}) - - - - - createFollowUp(text)} - /> - - - - - - - - - - -
-``` - -### 6.4 Stores (Svelte 5 Runes) - -```typescript -// src/lib/stores/questions.svelte.ts -import { questionsApi } from '$lib/api/questions'; - -interface QuestionsState { - questions: Question[]; - loading: boolean; - error: string | null; -} - -function createQuestionsStore() { - let state = $state({ - questions: [], - loading: false, - error: null - }); - - return { - get questions() { return state.questions; }, - get loading() { return state.loading; }, - get error() { return state.error; }, - - // Filtered questions - filtered(filters: QuestionFilters) { - return state.questions.filter(q => { - if (filters.status && q.status !== filters.status) return false; - if (filters.priority && q.priority !== filters.priority) return false; - if (filters.collectionId && q.collectionId !== filters.collectionId) return false; - if (filters.search && !q.title.toLowerCase().includes(filters.search.toLowerCase())) return false; - if (filters.tags.length && !filters.tags.some(t => q.tags.includes(t))) return false; - return true; - }); - }, - - // Load questions - async load() { - state.loading = true; - state.error = null; - try { - const result = await questionsApi.getAll(); - state.questions = result.questions; - } catch (e) { - state.error = e.message; - } finally { - state.loading = false; - } - }, - - // Create question - async create(data: CreateQuestionDto) { - const { question } = await questionsApi.create(data); - state.questions = [question, ...state.questions]; - return question; - }, - - // Update question - async update(id: string, data: UpdateQuestionDto) { - const { question } = await questionsApi.update(id, data); - state.questions = state.questions.map(q => - q.id === id ? question : q - ); - return question; - }, - - // Delete question - async delete(id: string) { - await questionsApi.delete(id); - state.questions = state.questions.filter(q => q.id !== id); - } - }; -} - -export const questionsStore = createQuestionsStore(); -``` - ---- - -## 7. Mobile App (Expo) - -### 7.1 Screen-Struktur - -``` -app/ -├── (tabs)/ -│ ├── _layout.tsx # Tab Navigator -│ ├── index.tsx # Fragen-Liste (Home) -│ ├── collections.tsx # Sammlungen -│ └── settings.tsx # Einstellungen -├── question/ -│ ├── [id].tsx # Frage-Detail -│ └── new.tsx # Neue Frage -├── research/ -│ └── [id].tsx # Recherche-Ergebnis -└── _layout.tsx # Root Layout -``` - -### 7.2 Haupt-Screens - -```tsx -// app/(tabs)/index.tsx -import { View, FlatList, RefreshControl } from 'react-native'; -import { useQuestions } from '@/hooks/useQuestions'; -import { QuestionCard } from '@/components/QuestionCard'; -import { QuickCaptureInput } from '@/components/QuickCaptureInput'; -import { FilterBar } from '@/components/FilterBar'; - -export default function QuestionsScreen() { - const { questions, loading, refetch, createQuestion } = useQuestions(); - const [filters, setFilters] = useState({ status: 'open' }); - - const filteredQuestions = useMemo(() => - questions.filter(q => !filters.status || q.status === filters.status), - [questions, filters] - ); - - return ( - - createQuestion({ title })} - placeholder="Neue Frage eingeben..." - /> - - - - ( - router.push(`/question/${item.id}`)} - /> - )} - keyExtractor={(item) => item.id} - refreshControl={ - - } - contentContainerClassName="p-4 gap-3" - /> - - ); -} -``` - -### 7.3 Quick Capture Widget - -```tsx -// components/QuickCaptureInput.tsx -import { View, TextInput, Pressable } from 'react-native'; -import { Send, Sparkles } from 'lucide-react-native'; -import { useState } from 'react'; - -interface Props { - onSubmit: (title: string, options?: { autoResearch?: boolean }) => void; - placeholder?: string; -} - -export function QuickCaptureInput({ onSubmit, placeholder }: Props) { - const [text, setText] = useState(''); - const [autoResearch, setAutoResearch] = useState(false); - - const handleSubmit = () => { - if (!text.trim()) return; - onSubmit(text.trim(), { autoResearch }); - setText(''); - }; - - return ( - - - - setAutoResearch(!autoResearch)} - > - - - - - - - - ); -} -``` - ---- - -## 8. Implementierungs-Roadmap - -### Phase 1: Foundation (1-2 Wochen) - -``` -[ ] Projekt-Struktur erstellen - [ ] apps/questions/apps/backend (NestJS) - [ ] apps/questions/apps/web (SvelteKit) - [ ] apps/questions/apps/mobile (Expo) - [ ] apps/questions/packages/shared - -[ ] Datenbank-Schema - [ ] Drizzle Schema definieren - [ ] Migrations erstellen - [ ] Seed-Daten vorbereiten - -[ ] Backend: Core CRUD - [ ] QuestionModule (CRUD) - [ ] CollectionModule (CRUD) - [ ] HealthModule - [ ] Auth-Integration (@manacore/shared-nestjs-auth) - -[ ] Web: Basis-UI - [ ] Layout mit Sidebar - [ ] Fragen-Liste - [ ] Frage erstellen/bearbeiten - [ ] Auth-Flow -``` - -### Phase 2: Research Engine (1-2 Wochen) - -``` -[ ] Backend: Research-System - [ ] ResearchModule - [ ] WebSearchService (externe API-Integration) - [ ] ContentExtractorService - [ ] AiCompletionService (Ollama + OpenRouter) - [ ] SourceModule - -[ ] Research-Pipeline - [ ] Query Expansion - [ ] Web Search - [ ] Content Extraction - [ ] Relevance Scoring - [ ] Synthesis - [ ] Citation Generation - -[ ] Web: Research-UI - [ ] Research starten (Button + Depth-Auswahl) - [ ] Progress-Anzeige - [ ] Ergebnis-Darstellung - [ ] Quellen-Liste -``` - -### Phase 3: Answers & Notes (1 Woche) - -``` -[ ] Backend: Answers - [ ] AnswerModule (CRUD) - [ ] Markdown-Verarbeitung - -[ ] Web: Answer-UI - [ ] Markdown-Editor - [ ] Antwort-Liste - [ ] Quellen-Referenzierung -``` - -### Phase 4: Mobile App (1-2 Wochen) - -``` -[ ] Expo Setup - [ ] Navigation - [ ] Auth-Integration - [ ] API-Client - -[ ] Screens - [ ] Fragen-Liste - [ ] Quick Capture - [ ] Frage-Detail - [ ] Research-Anzeige - [ ] Sammlungen -``` - -### Phase 5: Advanced Features (1 Woche) - -``` -[ ] Export - [ ] Markdown-Export - [ ] PDF-Export - [ ] Bibliography-Export - -[ ] Sharing - [ ] Öffentliche Sammlungen - [ ] Share-Links - -[ ] Bulk-Operationen - [ ] Multi-Select - [ ] Bulk-Archive - [ ] Bulk-Move -``` - -### Phase 6: Polish & Launch (1 Woche) - -``` -[ ] Landing Page (Astro) -[ ] Performance-Optimierung -[ ] Error-Handling -[ ] Loading-States -[ ] Dokumentation -[ ] Deploy-Konfiguration -``` - ---- - -## 9. Technische Entscheidungen - -### 9.1 Web Search API - -**Optionen:** - -| Service | Preis | Qualität | Integration | -|---------|-------|----------|-------------| -| **SearXNG (self-hosted)** | Kostenlos | Gut | Docker-Container | -| **Brave Search API** | $5/1000 queries | Sehr gut | REST API | -| **Google Custom Search** | $5/1000 queries | Excellent | REST API | -| **Tavily AI** | $5/1000 queries | Gut für AI | REST API | - -**Empfehlung:** SearXNG für Development, Brave/Tavily für Production - -### 9.2 Content Extraction - -```typescript -// Verwendung von @extractus/article-extractor -import { extract } from '@extractus/article-extractor'; - -async function extractContent(url: string) { - const article = await extract(url); - return { - title: article?.title, - content: article?.content, - author: article?.author, - published: article?.published - }; -} -``` - -### 9.3 Credit-Verbrauch - -| Aktion | Credits | -|--------|---------| -| Quick Research | 5 | -| Standard Research | 15 | -| Deep Research | 30 | -| Manual Source Add | 0 | -| Export | 0 | - ---- - -## 10. Offene Fragen - -1. **Web Search API**: Welchen Dienst verwenden? SearXNG vs. Brave vs. Tavily? -2. **Offline-Support**: Sollen Fragen offline erfasst werden können? -3. **Collaboration**: Sollen Teams gemeinsam an Fragen arbeiten können? -4. **Voice Input**: Soll Spracheingabe (STT) integriert werden? -5. **Auto-Research Schedule**: Sollen Fragen automatisch recherchiert werden (z.B. täglich)? - ---- - -## 11. Metriken & KPIs - -| Metrik | Ziel | Tracking | -|--------|------|----------| -| Fragen pro User/Woche | 10+ | Weekly | -| Recherchen pro Frage | 1.5 | Weekly | -| Antwort-Rate | 50% | Weekly | -| Research Satisfaction (Rating) | 4.0/5.0 | Per Research | -| Time to Answer | < 1 Woche | Weekly | - ---- - -*Dokument-Status: Entwurf - Zur Überprüfung* diff --git a/docs/AUTOMATED_TESTING_SYSTEM.md b/docs/AUTOMATED_TESTING_SYSTEM.md deleted file mode 100644 index a19cdcc7b..000000000 --- a/docs/AUTOMATED_TESTING_SYSTEM.md +++ /dev/null @@ -1,583 +0,0 @@ -# Automated Testing System - Implementation Summary - -Complete automated daily test execution system with monitoring and reporting for the ManaCore monorepo. - -## Overview - -This document provides an overview of the automated testing infrastructure implemented for continuous quality assurance. - -**Implementation Date**: 2025-12-25 -**Status**: Ready for deployment - -## Components Delivered - -### 1. GitHub Actions Workflow - -**File**: `.github/workflows/daily-tests.yml` - -**Features**: -- Scheduled daily execution at 2 AM UTC -- Manual trigger with configurable parameters -- Parallel test execution across multiple test suites -- Automatic database setup/teardown per suite -- Coverage enforcement (80% minimum) -- Test result aggregation and reporting -- Flaky test detection -- Performance metrics tracking -- Failure notifications (GitHub issues, Slack) - -**Test Matrix**: -- Backend tests (Jest + PostgreSQL + Redis) -- Mobile tests (Jest + React Native) -- Web tests (Vitest + Svelte) -- Integration tests (E2E flows) - -### 2. Test Execution Scripts - -**Directory**: `/scripts/` - -#### `/scripts/run-tests-with-coverage.sh` -Comprehensive test execution script with coverage reporting. - -**Usage**: -```bash -# Run all tests -./scripts/run-tests-with-coverage.sh - -# Run specific package -./scripts/run-tests-with-coverage.sh mana-core-auth -./scripts/run-tests-with-coverage.sh chat-backend -``` - -**Features**: -- Automatic Docker verification -- Database setup per package -- Coverage threshold checking -- Colored terminal output -- Detailed summary report - -### 3. Test Reporting Scripts - -**Directory**: `/scripts/test-reporting/` - -#### `aggregate-coverage.js` -Merges coverage reports from multiple test suites. - -**Outputs**: -- `total-coverage.json`: Aggregated coverage data -- `summary.md`: Markdown coverage summary - -#### `generate-summary.js` -Creates GitHub Actions summary with test results. - -**Features**: -- Coverage breakdown by suite -- Pass/fail statistics -- Recommendations for improvement - -#### `detect-flaky-tests.js` -Identifies tests that fail intermittently. - -**Configuration**: -- Flaky threshold: 10% failure rate -- Minimum runs: 3 -- History retention: 30 runs per test - -**Outputs**: -- `flaky-tests.json`: List of flaky tests -- `test-history.json`: Historical test data - -#### `track-metrics.js` -Records test performance metrics over time. - -**Tracks**: -- Total test execution time -- Average test duration -- Slowest tests -- Suite-level metrics -- Performance regressions (>20% increase) - -**Outputs**: -- `metrics.json`: Current metrics -- `metrics-report.md`: Formatted report -- `metrics-history.json`: 90-day history - -#### `format-metrics.js` -Formats metrics for GitHub Actions summary display. - -### 4. Test Data Management - -**Directory**: `/scripts/test-data/` - -#### `seed-test-data.sh` -Seeds databases with consistent test data. - -**Usage**: -```bash -# Seed all services -./scripts/test-data/seed-test-data.sh - -# Seed specific service -./scripts/test-data/seed-test-data.sh auth -./scripts/test-data/seed-test-data.sh chat -``` - -**Provides**: -- Deterministic test user accounts -- Pre-configured AI models (chat) -- Consistent credit balances - -**Test Users**: -| Email | Password | ID | Role | -|-------|----------|-----|------| -| test-user-1@example.com | TestPassword123! | 00000000-0000-0000-0000-000000000001 | user | -| test-user-2@example.com | TestPassword123! | 00000000-0000-0000-0000-000000000002 | user | -| admin@example.com | AdminPassword123! | 00000000-0000-0000-0000-000000000003 | admin | - -#### `cleanup-test-data.sh` -Removes test data and resets databases. - -**Usage**: -```bash -# Clean all databases -./scripts/test-data/cleanup-test-data.sh - -# Clean specific database -./scripts/test-data/cleanup-test-data.sh auth -``` - -### 5. Documentation - -#### `docs/TESTING_GUIDE.md` -Comprehensive testing documentation (4000+ words). - -**Contents**: -- Test types and strategies -- Local testing instructions -- Automated daily tests overview -- Writing tests best practices -- Test data management -- Coverage requirements -- Troubleshooting guide -- CI/CD integration - -#### `docs/TESTING_QUICK_REFERENCE.md` -Quick reference for common testing tasks. - -**Contents**: -- Quick commands -- Test patterns and templates -- Coverage viewing -- Test data reference -- Troubleshooting shortcuts -- Best practices summary - -#### `scripts/test-reporting/README.md` -Documentation for test reporting scripts. - -**Contents**: -- Script overview and usage -- Data format specifications -- Development guide -- Integration examples -- Troubleshooting - -### 6. Package.json Updates - -**File**: `/package.json` - -Added convenience scripts: -```json -{ - "test:cov": "./scripts/run-tests-with-coverage.sh", - "test:seed": "./scripts/test-data/seed-test-data.sh", - "test:cleanup": "./scripts/test-data/cleanup-test-data.sh" -} -``` - -## Architecture - -### Workflow Execution Flow - -``` -┌─────────────────────────────────────────┐ -│ Daily Tests Workflow (2 AM UTC) │ -└─────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────┐ -│ 1. Setup Job │ -│ - Detect test suites │ -│ - Generate test matrices │ -└─────────────────────────────────────────┘ - │ - ┌─────────────┴─────────────┬──────────────┐ - ▼ ▼ ▼ -┌──────────┐ ┌──────────────┐ ┌──────────┐ -│ Backend │ │ Mobile │ │ Web │ -│ Tests │ │ Tests │ │ Tests │ -│(Parallel)│ │ (Parallel) │ │(Parallel)│ -└──────────┘ └──────────────┘ └──────────┘ - │ │ │ - └─────────────┬─────────────┴──────────────┘ - ▼ -┌─────────────────────────────────────────┐ -│ Integration Tests │ -│ - Full E2E flows │ -│ - Auth + Database │ -└─────────────────────────────────────────┘ - │ - ┌─────────────┴─────────────┬──────────────┐ - ▼ ▼ ▼ -┌──────────┐ ┌──────────────┐ ┌──────────┐ -│ Report │ │ Detect Flaky │ │ Metrics │ -│ Job │ │ Tests │ │ Tracking │ -└──────────┘ └──────────────┘ └──────────┘ - │ │ │ - └─────────────┬─────────────┴──────────────┘ - ▼ -┌─────────────────────────────────────────┐ -│ Notify Job (on failure) │ -│ - GitHub issue │ -│ - Slack notification │ -└─────────────────────────────────────────┘ -``` - -### Test Data Flow - -``` -┌──────────────┐ -│ Test Suite │ -└──────────────┘ - │ - ▼ -┌──────────────────────┐ -│ Setup Database │ -│ - Run migrations │ -│ - Seed test data │ -└──────────────────────┘ - │ - ▼ -┌──────────────────────┐ -│ Execute Tests │ -│ - Unit tests │ -│ - Integration tests │ -└──────────────────────┘ - │ - ▼ -┌──────────────────────┐ -│ Generate Coverage │ -│ - coverage-summary │ -│ - HTML report │ -└──────────────────────┘ - │ - ▼ -┌──────────────────────┐ -│ Cleanup │ -│ - Remove test data │ -│ - Close connections │ -└──────────────────────┘ -``` - -## Usage - -### Running Tests Locally - -```bash -# Quick commands -pnpm test # Run all tests -pnpm test:cov # Run with coverage -pnpm test:seed # Seed test data -pnpm test:cleanup # Clean test data - -# Within a package -cd services/mana-core-auth -pnpm test # Run tests -pnpm test:cov # With coverage -pnpm test:watch # Watch mode -``` - -### Triggering Daily Tests Manually - -1. Navigate to GitHub Actions -2. Select "Daily Tests" workflow -3. Click "Run workflow" -4. (Optional) Configure parameters: - - Coverage threshold (default: 80%) - - Verbose output (default: false) -5. Click "Run workflow" button - -### Viewing Test Results - -**Coverage Reports**: -- Download from GitHub Actions artifacts -- Retention: 30 days -- Format: HTML + JSON - -**Aggregated Coverage**: -- Download "aggregated-coverage-report" artifact -- Retention: 90 days -- Includes: `total-coverage.json`, `summary.md` - -**Test Metrics**: -- Download "test-metrics" artifact -- Retention: 365 days -- Includes: `metrics.json`, `metrics-history.json` - -**Flaky Test Reports**: -- Download "flaky-test-report" artifact -- Retention: 90 days -- Format: JSON with failure rates - -## Configuration - -### Coverage Thresholds - -**Global** (all packages): -- Lines: 80% -- Statements: 80% -- Functions: 80% -- Branches: 80% - -**Critical Paths** (100% required): -- `services/mana-core-auth/src/auth/auth.service.ts` -- `services/mana-core-auth/src/credits/credits.service.ts` -- `services/mana-core-auth/src/common/guards/jwt-auth.guard.ts` - -### Flaky Test Detection - -- **Threshold**: 10% failure rate -- **Minimum Runs**: 3 runs required -- **History**: Last 30 runs per test -- **Action**: GitHub issue created automatically - -### Performance Metrics - -- **Regression Threshold**: 20% duration increase -- **Suite Threshold**: 30% duration increase -- **History**: 90 days retained -- **Action**: Workflow fails on regression - -## Monitoring and Alerts - -### Automated Notifications - -**GitHub Issues**: -- Created on test failure -- Created on flaky test detection -- Labels: `testing`, `failure`, `flaky-test`, `automated` - -**Slack** (if configured): -- Daily test failure notifications -- Sent to configured webhook -- Includes workflow run link - -### Metrics Dashboard - -Track trends via artifacts: - -1. **Coverage Trends**: - - Download aggregated coverage from multiple runs - - Compare `total-coverage.json` over time - -2. **Flaky Tests**: - - Review `flaky-tests.json` artifact - - Track failure rates - -3. **Performance**: - - Check `metrics-history.json` - - Monitor execution time trends - -## Best Practices - -### Writing Tests - -✅ **DO**: -- Write tests for all new features -- Use descriptive test names -- Keep tests isolated -- Mock external services -- Maintain 80%+ coverage - -❌ **DON'T**: -- Skip tests for "simple" code -- Create order-dependent tests -- Make real API calls -- Hardcode IDs or timestamps -- Commit failing tests - -### Test Data - -✅ **DO**: -- Use deterministic test data -- Clean up after tests -- Use test factories -- Seed consistent data - -❌ **DON'T**: -- Share state between tests -- Use production data -- Leave test data behind -- Use random values without seeds - -### Coverage - -✅ **DO**: -- Aim for high coverage (80%+) -- Test critical paths thoroughly -- Review coverage reports -- Fix coverage drops quickly - -❌ **DON'T**: -- Ignore coverage warnings -- Write tests just for coverage -- Skip edge cases -- Rely solely on coverage metrics - -## Troubleshooting - -### Common Issues - -**Tests fail with database connection error**: -```bash -# Solution: Start Docker -pnpm docker:up -``` - -**Coverage below threshold**: -```bash -# Solution: View uncovered code -cd services/mana-core-auth -pnpm test:cov -open coverage/lcov-report/index.html -``` - -**Flaky tests detected**: -```bash -# Solution: Review test isolation -# - Check for timing issues -# - Verify proper async/await -# - Ensure cleanup in afterEach -``` - -**Performance regression**: -```bash -# Solution: Profile slow tests -# - Check test-results/metrics.json -# - Identify slowest tests -# - Optimize or split large tests -``` - -## Maintenance - -### Regular Tasks - -**Weekly**: -- Review flaky test reports -- Address failing tests -- Check coverage trends - -**Monthly**: -- Review performance metrics -- Update test data as needed -- Clean up old artifacts - -**Quarterly**: -- Audit test coverage -- Update testing documentation -- Review and improve test quality - -### Updating Scripts - -When modifying reporting scripts: - -1. Test locally with mock data -2. Update script README -3. Test in workflow with manual trigger -4. Monitor first automated run -5. Update documentation if needed - -## Future Enhancements - -### Planned Improvements - -1. **E2E Tests with Playwright**: - - Browser-based testing - - Visual regression testing - - Cross-browser validation - -2. **Test Parallelization**: - - Optimize parallel execution - - Reduce total workflow time - - Smart test splitting - -3. **Coverage Visualization**: - - Interactive coverage dashboard - - Historical trend charts - - Per-developer coverage stats - -4. **Advanced Flaky Detection**: - - ML-based prediction - - Auto-retry flaky tests - - Root cause analysis - -5. **Performance Baselines**: - - Establish performance budgets - - Block slow test commits - - Automated optimization suggestions - -## Support - -### Documentation - -- **Comprehensive Guide**: `/docs/TESTING_GUIDE.md` -- **Quick Reference**: `/docs/TESTING_QUICK_REFERENCE.md` -- **Script Docs**: `/scripts/test-reporting/README.md` - -### Getting Help - -- **GitHub Issues**: Label with `testing` -- **Team Chat**: #testing channel -- **Documentation**: Check docs first - -## Metrics and Success Criteria - -### Key Performance Indicators - -| Metric | Target | Current | -|--------|--------|---------| -| Overall Coverage | 80%+ | TBD (after first run) | -| Daily Test Success Rate | 95%+ | TBD | -| Flaky Test Count | <5 | TBD | -| Average Test Duration | <60s per suite | TBD | -| Mean Time to Fix | <24 hours | TBD | - -### Success Criteria - -✅ **Workflow runs successfully daily** -✅ **All test suites execute in parallel** -✅ **Coverage reports generated and aggregated** -✅ **Flaky tests identified and tracked** -✅ **Performance metrics recorded** -✅ **Failures trigger notifications** -✅ **Documentation complete and accessible** - -## Conclusion - -The automated testing system provides comprehensive quality assurance for the ManaCore monorepo with: - -- **Automated Execution**: Daily scheduled runs at 2 AM UTC -- **Parallel Testing**: Fast execution across multiple suites -- **Coverage Enforcement**: 80% minimum threshold -- **Flaky Detection**: Identify unreliable tests -- **Performance Tracking**: Monitor test execution trends -- **Failure Notifications**: Immediate alerts on issues -- **Comprehensive Documentation**: Complete guides and references - -The system is ready for deployment and will ensure continuous quality as the monorepo grows. - ---- - -**Implementation**: Hive Mind Swarm (Tester Agent) -**Date**: 2025-12-25 -**Status**: Complete ✅ diff --git a/docs/DEPENDENCY_ALIGNMENT.md b/docs/DEPENDENCY_ALIGNMENT.md deleted file mode 100644 index 2a9929469..000000000 --- a/docs/DEPENDENCY_ALIGNMENT.md +++ /dev/null @@ -1,180 +0,0 @@ -# Dependency Alignment Guide - -This document tracks critical dependencies across all projects and their target versions for alignment. - -## Critical Dependencies - -### High Priority - Version Mismatches - -| Package | Target Version | Current Versions | Notes | -| ----------------------- | -------------- | ----------------- | -------------------------------------- | -| `@supabase/supabase-js` | **2.81.1** | 2.38.4 - 2.81.1 | Significant spread, alignment critical | -| `typescript` | **5.9.2** | 5.3.3 - 5.9.2 | Update all to latest | -| `react` | **19.1.0** | 18.3.1 - 19.1.0 | Mixed versions | -| `expo` | **54.x** | 52.0.39 - 54.0.21 | Manacore needs update | -| `expo-router` | **6.x** | 4.0.19 - 6.0.14 | Manacore needs update | -| `astro` | **5.16.0** | 5.3.0 - 5.16.0 | Memoro landing needs update | - -### Current Status by Project - -#### Supabase Versions - -``` -maerchenzauber: - - backend: 2.50.3 - - mobile: 2.50.3 - -manacore: - - mobile: 2.38.4 ❌ (very outdated) - - web: 2.49.2 - -manadeck: - - backend: 2.58.0 - - mobile: 2.38.4 ❌ (very outdated) - - web: 2.81.1 ✅ - -memoro: - - mobile: 2.49.4 - - web: 2.76.1 -``` - -#### Expo/React Native Versions - -``` -maerchenzauber: - - expo: 54.0.21 ✅ - - react-native: 0.81.5 ✅ - - expo-router: 6.0.14 ✅ - -manacore: - - expo: 52.0.39 ❌ (SDK 52, needs update to 54) - - react-native: 0.76.7 ❌ - - expo-router: 4.0.19 ❌ - -manadeck: - - expo: 54.0.13 ✅ - - react-native: 0.81.4 ✅ - - expo-router: 6.0.10 ✅ - -memoro: - - expo: 54.0.0 ✅ - - react-native: 0.81.4 ✅ - - expo-router: 6.0.8 ✅ -``` - -#### NestJS Versions (Backends) - -``` -maerchenzauber: NestJS 10.0.0 -manadeck: NestJS 11.0.1 - -Target: Align to NestJS 11.x -``` - -## Migration Priority - -### Phase 1: Critical Alignments (Week 1) - -1. **Manacore Expo Update** (High Risk) - - Upgrade from Expo SDK 52 → 54 - - Update expo-router 4.x → 6.x - - Update react-native 0.76.7 → 0.81.x - - This is the most significant update needed - -2. **Supabase Alignment** - - Update all projects to 2.81.1 - - Test auth flows after update - - Check for breaking changes in RLS - -3. **TypeScript Alignment** - - Update all to 5.9.2 - - Run type-check across all projects - -### Phase 2: Secondary Alignments (Week 2) - -1. **Astro Updates** - - Update Memoro landing from 5.3.0 → 5.16.0 - -2. **SvelteKit/Vite Updates** - - Align Vite versions (6.0.7 → 7.1.10) - - Align SvelteKit versions - -3. **NestJS Alignment** - - Update Maerchenzauber backend 10.x → 11.x - - Test all API endpoints - -## Alignment Commands - -### Update Supabase across all projects: - -```bash -# Maerchenzauber -cd maerchenzauber/apps/backend && pnpm update @supabase/supabase-js@2.81.1 -cd maerchenzauber/apps/mobile && pnpm update @supabase/supabase-js@2.81.1 - -# Manacore -cd manacore/apps/mobile && pnpm update @supabase/supabase-js@2.81.1 -cd manacore/apps/web && pnpm update @supabase/supabase-js@2.81.1 - -# Manadeck -cd manadeck/backend && pnpm update @supabase/supabase-js@2.81.1 -cd manadeck/apps/mobile && pnpm update @supabase/supabase-js@2.81.1 - -# Memoro -cd memoro/apps/mobile && pnpm update @supabase/supabase-js@2.81.1 -cd memoro/apps/web && pnpm update @supabase/supabase-js@2.81.1 -``` - -### Update TypeScript across all projects: - -```bash -pnpm update typescript@5.9.2 --recursive -``` - -## Testing After Alignment - -After updating dependencies, verify: - -1. **Build succeeds**: `pnpm run build` -2. **Type checks pass**: `pnpm run type-check` -3. **Tests pass**: `pnpm run test` -4. **Auth flows work** (especially after Supabase updates) -5. **Mobile apps launch** (especially after Expo updates) - -## Breaking Changes to Watch - -### Supabase 2.38 → 2.81 - -- Auth session handling may have changed -- Check `onAuthStateChange` listeners -- Verify RLS policies still work - -### Expo SDK 52 → 54 - -- Check expo-router migration guide -- New navigation patterns in 6.x -- Screen options changes -- Layout changes - -### NestJS 10 → 11 - -- Decorator changes -- Module resolution changes -- Check middleware compatibility - -## Recommended Tools - -- **Renovate/Dependabot**: Auto-update dependencies -- **pnpm outdated**: Check for outdated packages -- **turbo prune**: Create minimal installs for specific projects - -## Tracking Updates - -Mark completed updates: - -- [ ] Supabase alignment (all projects) -- [ ] TypeScript alignment (all projects) -- [ ] Manacore Expo SDK update -- [ ] Astro alignment -- [ ] NestJS alignment -- [ ] SvelteKit/Vite alignment diff --git a/docs/README_ENV_AUDIT.md b/docs/README_ENV_AUDIT.md deleted file mode 100644 index ca55d6f45..000000000 --- a/docs/README_ENV_AUDIT.md +++ /dev/null @@ -1,264 +0,0 @@ -# Environment Configuration Audit - Complete Documentation - -This folder contains a comprehensive audit of all backend environment variable configurations in the Mana Universe monorepo. - -## Documents - -### 1. [ENV_CONFIGURATION_AUDIT.md](ENV_CONFIGURATION_AUDIT.md) - MAIN REPORT - -**The complete audit with all findings and detailed analysis** - -- **Section 1:** Port Assignment Matrix (identifies 2 port conflicts) -- **Section 2:** Auth Environment Variables (missing variables, inconsistent naming) -- **Section 3:** Environment Variable Mapping Audit (coverage analysis) -- **Section 4:** Hardcoded Values & Security Concerns (DEV_USER_ID, CORS) -- **Section 5:** Configuration Best Practices Compliance (validation schemas) -- **Section 6:** Critical Issues Summary (8 issues identified) -- **Section 7:** Recommended Actions (3 implementation phases) -- **Section 8:** Updated Port Assignments (proposed fixes) -- **Section 9:** Environment Variable Summary Tables -- **Section 10:** Implementation Checklist (16 action items) - -**Read this if:** You need the complete, detailed analysis with code examples and full context. - -**Lines:** 408 | **Size:** 14KB - ---- - -### 2. [ENV_AUDIT_SUMMARY.md](ENV_AUDIT_SUMMARY.md) - QUICK START GUIDE - -**Executive summary with actionable checklists and next steps** - -- **Quick Issue Overview:** Blocking, Major, and Medium issues at a glance -- **Phase-Based Checklist:** Quick fix checklist organized by priority -- **Port Mapping:** Visual comparison of current vs. recommended ports -- **Environment Variable Status:** What's working and what needs work -- **Files to Modify:** Concrete list of files that need changes -- **Testing Instructions:** How to verify fixes -- **Additional Resources:** Links to full documentation - -**Read this if:** You need a quick overview and want to start fixing issues immediately. - -**Lines:** 166 | **Size:** 5KB - ---- - -### 3. [ENV_BACKEND_MATRIX.md](ENV_BACKEND_MATRIX.md) - DETAILED MATRIX VISUALIZATION - -**Backend configuration status visualized in detailed tables and matrices** - -- **Backend Status Matrix:** Port, Auth URL, Dev Bypass, Validation status -- **Database Configuration:** Which backends have database URLs -- **CORS Configuration:** How CORS is implemented (hardcoded vs. environment) -- **Port Availability & Conflicts:** Visual representation of port assignments -- **Environment Variable Generation Map:** How variables flow from .env.development -- **Severity Matrix:** Issue counts and time estimates -- **Best Practices Scorecard:** Overall quality assessment (5.1/10) -- **Variable Standardization Guide:** Current inconsistencies and path to consistency - -**Read this if:** You want to understand the full scope of backend configurations visually. - -**Lines:** 234 | **Size:** 8KB - ---- - -## Key Findings Summary - -### BLOCKING ISSUES (Fix Immediately) - -1. **Port 3002 Conflict:** Chat and Nutriphi both use port 3002 -2. **Port 3003 Conflict:** Picture and Maerchenzauber both use port 3003 -3. **Hardcoded DEV_USER_ID:** Chat backend hardcodes `17cb0be7-058a-4964-9e18-1fe7055fd014` - -### MAJOR ISSUES (Fix Soon) - -4. **Auth URL Naming Inconsistency:** - - Manadeck uses `MANA_SERVICE_URL` (should be `MANA_CORE_AUTH_URL`) - - Nutriphi uses `MANACORE_AUTH_URL` (should be `MANA_CORE_AUTH_URL`) - - Chat, Picture, Zitare, Presi use correct `MANA_CORE_AUTH_URL` - -5. **Hardcoded CORS Origins:** 4 backends hardcode allowed origins instead of using environment variable - -6. **Missing Configuration Examples:** - - No `.env.example` for Zitare backend - - No `.env.example` for Presi backend - -### MEDIUM ISSUES (Improve Quality) - -7. **Missing Validation Schemas:** Chat, Picture, Zitare, Presi lack Joi validation schemas - -8. **Inconsistent Dev Bypass Auth:** Only Chat backend implements DEV_BYPASS_AUTH - ---- - -## Quick Fix Timeline - -| Phase | Tasks | Time | Impact | -| ------- | -------------------------------------- | --------- | ------------------------------------------------- | -| Phase 1 | Fix ports + add DEV_USER_ID | 15-30 min | CRITICAL - Enables simultaneous backend execution | -| Phase 2 | Standardize naming + add .env examples | 30 min | MAJOR - Improves consistency | -| Phase 3 | Add validation schemas + extract CORS | 2-3 hours | QUALITY - Code quality improvement | - -**Total estimated time to fix all issues: 6-8 hours** - ---- - -## Which Document Should I Read? - -### I want to... - -**...quickly understand what's wrong** -→ Read [ENV_AUDIT_SUMMARY.md](ENV_AUDIT_SUMMARY.md) (5 min read) - -**...get detailed analysis with code examples** -→ Read [ENV_CONFIGURATION_AUDIT.md](ENV_CONFIGURATION_AUDIT.md) (20 min read) - -**...see all backend configurations visually** -→ Read [ENV_BACKEND_MATRIX.md](ENV_BACKEND_MATRIX.md) (10 min read) - -**...start fixing issues immediately** -→ Read [ENV_AUDIT_SUMMARY.md](ENV_AUDIT_SUMMARY.md) "Quick Fix Checklist" section - -**...understand the complete scope** -→ Read all three documents in order (1 → 2 → 3) - ---- - -## Implementation Roadmap - -### If you have 30 minutes - -1. Read ENV_AUDIT_SUMMARY.md -2. Fix port conflicts in .env.development -3. Add DEV_USER_ID variable - -### If you have 1-2 hours - -1. Complete Phase 1 fixes -2. Update generate-env.mjs variable names -3. Create .env.example files for Zitare and Presi - -### If you have 4+ hours - -1. Complete all Phase 1 & 2 fixes -2. Add validation schemas to all backends -3. Extract CORS origins to environment variables -4. Test all backends can run simultaneously - ---- - -## Files Analyzed in This Audit - -**Configuration Files:** - -- .env.development (202 lines) -- scripts/generate-env.mjs (433 lines) -- services/mana-core-auth/.env.example -- apps/chat/apps/backend/.env.example -- apps/picture/apps/backend/.env.example -- apps/manadeck/apps/backend/.env.example - -**Backend Configuration:** - -- 6 app.module.ts files (NestJS configuration) -- 5 main.ts files (server bootstrap & CORS) -- 1 validation.schema.ts file (Manadeck) -- Multiple JWT auth guard files - -**Total Files Analyzed:** 25+ -**Total Lines Reviewed:** 2,000+ -**Issues Identified:** 8 critical/major items, 17 total issues - ---- - -## Recommendations by Priority - -### Priority 1: BLOCKING (Do Today) - -- [ ] Fix PICTURE_BACKEND_PORT: 3003 → 3005 -- [ ] Fix NUTRIPHI_BACKEND_PORT: 3002 → 3006 -- [ ] Add DEV_USER_ID to .env.development -- [ ] Update Chat backend to read DEV_USER_ID from ConfigService - -### Priority 2: MAJOR (Do This Week) - -- [ ] Rename MANA_SERVICE_URL to MANA_CORE_AUTH_URL in Manadeck -- [ ] Rename MANACORE_AUTH_URL to MANA_CORE_AUTH_URL in Nutriphi -- [ ] Create .env.example for Zitare backend -- [ ] Create .env.example for Presi backend - -### Priority 3: MEDIUM (Plan This Week) - -- [ ] Add validation schemas to 4 backends (Chat, Picture, Zitare, Presi) -- [ ] Extract CORS origins to CORS_ORIGINS environment variable -- [ ] Update all backends to use env variable for CORS -- [ ] Document final port assignments in project CLAUDE.md files - -### Priority 4: LONG-TERM (Future Improvement) - -- [ ] Implement consistent dev bypass auth pattern across all backends -- [ ] Add comprehensive integration tests for all backends -- [ ] Document environment configuration in deployment guide -- [ ] Set up CI/CD to validate .env configuration changes - ---- - -## Success Criteria - -After implementing all recommendations, you should be able to: - -1. **Run all 8 active backends simultaneously without port conflicts** - - ```bash - pnpm dev:auth & - pnpm dev:chat:backend & - pnpm dev:picture:backend & - pnpm dev:manadeck:backend & - pnpm dev:zitare:backend & - pnpm dev:presi:backend & - ``` - -2. **Every backend loads from .env without warnings** - - All required variables present - - All variables properly typed/validated - -3. **Consistent naming conventions across all backends** - - Same auth URL variable name used everywhere - - Same database URL pattern - - Same CORS configuration approach - -4. **All backends properly documented** - - Each has .env.example file - - Each has configuration validation schema - - Port assignments documented in CLAUDE.md - -5. **Security best practices enforced** - - No hardcoded values in source code - - All secrets loaded from environment - - Configuration validated on startup - ---- - -## Contact & Questions - -If you have questions about any findings: - -1. **Detailed findings** → See ENV_CONFIGURATION_AUDIT.md section numbers -2. **Implementation guidance** → See ENV_AUDIT_SUMMARY.md "Files to Modify" -3. **Visual reference** → See ENV_BACKEND_MATRIX.md tables - ---- - -## Document Metadata - -**Audit Date:** December 1, 2025 -**Auditor:** Environment Configuration Auditor Agent -**Swarm Role:** Claude Flow Swarm - Configuration Validation Team -**Monorepo Version:** manacore-monorepo (main branch) -**Total Issues:** 8 critical/major + 9 medium/quality issues - -**Status:** READY FOR IMPLEMENTATION - ---- - -**Next Action:** Read ENV_AUDIT_SUMMARY.md and start with Phase 1 fixes.