diff --git a/.env.development b/.env.development index e50583d77..f782ea9e0 100644 --- a/.env.development +++ b/.env.development @@ -380,6 +380,32 @@ MANA_GAMES_GITHUB_TOKEN=your_github_token_here MANA_GAMES_GITHUB_OWNER=tillschneider MANA_GAMES_GITHUB_REPO=mana-games +# ============================================ +# MANA-RESEARCH SERVICE (Port 3068) +# ============================================ +# Unified web research orchestration across 16+ providers. +# Phase 1: SearXNG, DuckDuckGo, Brave, Tavily. + +MANA_RESEARCH_PORT=3068 +MANA_RESEARCH_DATABASE_URL=postgresql://mana:devpassword@localhost:5432/mana_platform +MANA_RESEARCH_CACHE_TTL_SECONDS=3600 + +# Search APIs (pay-per-use only, no subscriptions) +BRAVE_API_KEY= +TAVILY_API_KEY= +EXA_API_KEY= +SERPER_API_KEY= + +# Extract APIs +JINA_API_KEY= +FIRECRAWL_API_KEY= +SCRAPINGBEE_API_KEY= + +# Research Agents (Phase 3) +PERPLEXITY_API_KEY= +ANTHROPIC_API_KEY= +OPENAI_API_KEY= + # ============================================ # FINANCE PROJECT # ============================================ diff --git a/CLAUDE.md b/CLAUDE.md index 3adb8deaa..17f1d1afe 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -33,7 +33,7 @@ docs/ # Long-form docs (deployment, hardware, postmortems, etc.) ### Active services (`services/`) -`mana-auth` (3001), `mana-sync` (3050), `mana-credits`, `mana-user`, `mana-subscriptions`, `mana-analytics`, `mana-search` (3021), `mana-crawler`, `mana-api-gateway`, `mana-notify`, `mana-media`, `mana-llm`, `mana-image-gen`, `mana-video-gen`, `mana-stt`, `mana-tts`, `mana-voice-bot`, `mana-events`, `mana-geocoding` (3018), `mana-landing-builder`, `mana-ai` (3067, background AI Mission Runner — see [`services/mana-ai/CLAUDE.md`](services/mana-ai/CLAUDE.md)). Each non-trivial service has its own `CLAUDE.md`. +`mana-auth` (3001), `mana-sync` (3050), `mana-credits`, `mana-user`, `mana-subscriptions`, `mana-analytics`, `mana-search` (3021), `mana-crawler`, `mana-api-gateway`, `mana-notify`, `mana-media`, `mana-llm`, `mana-image-gen`, `mana-video-gen`, `mana-stt`, `mana-tts`, `mana-voice-bot`, `mana-events`, `mana-geocoding` (3018), `mana-landing-builder`, `mana-ai` (3067, background AI Mission Runner — see [`services/mana-ai/CLAUDE.md`](services/mana-ai/CLAUDE.md)), `mana-research` (3068, web research provider orchestration across 16+ providers — see [`services/mana-research/CLAUDE.md`](services/mana-research/CLAUDE.md) and [`docs/plans/mana-research-service.md`](docs/plans/mana-research-service.md)). Each non-trivial service has its own `CLAUDE.md`. ## Coding Guidelines diff --git a/docker-compose.macmini.yml b/docker-compose.macmini.yml index 0a7ae9ede..6cf2e7a25 100644 --- a/docker-compose.macmini.yml +++ b/docker-compose.macmini.yml @@ -381,6 +381,61 @@ services: - "traefik.http.routers.mana-credits.tls=true" - "traefik.http.services.mana-credits.loadbalancer.server.port=3002" + mana-research: + build: + context: . + dockerfile: services/mana-research/Dockerfile + image: mana-research:local + container_name: mana-research + restart: always + mem_limit: 256m + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_started + mana-credits: + condition: service_healthy + mana-search: + condition: service_started + environment: + TZ: Europe/Berlin + NODE_ENV: production + PORT: 3068 + DATABASE_URL: postgresql://postgres:${POSTGRES_PASSWORD:-mana123}@postgres:5432/mana_platform + REDIS_URL: redis://redis:6379 + MANA_AUTH_URL: http://mana-auth:3001 + MANA_LLM_URL: http://mana-llm:3025 + MANA_CREDITS_URL: http://mana-credits:3002 + MANA_SEARCH_URL: http://mana-search:3021 + MANA_SERVICE_KEY: ${MANA_SERVICE_KEY} + CACHE_TTL_SECONDS: 3600 + BRAVE_API_KEY: ${BRAVE_API_KEY:-} + TAVILY_API_KEY: ${TAVILY_API_KEY:-} + EXA_API_KEY: ${EXA_API_KEY:-} + SERPER_API_KEY: ${SERPER_API_KEY:-} + JINA_API_KEY: ${JINA_API_KEY:-} + FIRECRAWL_API_KEY: ${FIRECRAWL_API_KEY:-} + SCRAPINGBEE_API_KEY: ${SCRAPINGBEE_API_KEY:-} + PERPLEXITY_API_KEY: ${PERPLEXITY_API_KEY:-} + ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-} + OPENAI_API_KEY: ${OPENAI_API_KEY:-} + GOOGLE_GENAI_API_KEY: ${GOOGLE_GENAI_API_KEY:-} + CORS_ORIGINS: https://mana.how,https://chat.mana.how,https://research.mana.how + ports: + - "3068:3068" + healthcheck: + test: ["CMD", "bun", "-e", "fetch('http://127.0.0.1:3068/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))"] + interval: 120s + timeout: 10s + retries: 3 + start_period: 15s + labels: + - "traefik.enable=true" + - "traefik.http.routers.mana-research.rule=Host(`research.mana.how`)" + - "traefik.http.routers.mana-research.tls=true" + - "traefik.http.services.mana-research.loadbalancer.server.port=3068" + mana-events: build: context: services/mana-events diff --git a/docs/PORT_SCHEMA.md b/docs/PORT_SCHEMA.md index f43181414..1c8a8cb04 100644 --- a/docs/PORT_SCHEMA.md +++ b/docs/PORT_SCHEMA.md @@ -26,7 +26,8 @@ > - mana-mail `3042` > - mana-sync `3050` > - mana-credits `3061`, mana-user `3062`, mana-subscriptions `3063`, -> mana-analytics `3064`, mana-events `3065` +> mana-analytics `3064`, mana-events `3065`, mana-research `3068` +> (new 2026-04-17, Bun/Hono, public: `research.mana.how`) > > **Not deployed:** `mana-voice-bot` (default port `3024`, no scheduled > task, no cloudflared route, no launchd plist). diff --git a/docs/plans/mana-research-service.md b/docs/plans/mana-research-service.md new file mode 100644 index 000000000..bc717b238 --- /dev/null +++ b/docs/plans/mana-research-service.md @@ -0,0 +1,580 @@ +# Mana Research Service — Plan + +## Status (2026-04-17) + +Planung, noch kein Code. Neuer Service `services/mana-research` (Bun/Hono, Port 3068), der 16+ Web-Research-Provider hinter einer einheitlichen Schnittstelle bündelt. Ziel: Vergleichbarkeit zwischen Anbietern + beste verfügbare Research-Qualität je nach Query-Typ. + +**Vorausgegangene Analyse:** [`docs/reports/web-research-capabilities.md`](../reports/web-research-capabilities.md) +**Verwandtes Modul:** [`docs/plans/news-research-module.md`](./news-research-module.md) (das bestehende RSS-Modul, wird in Phase 3 auf den neuen Service migriert) + +## Ziel + +Eine zentrale Research-Schicht, die: + +1. **Viele Anbieter bündelt** hinter einer gemeinsamen Schnittstelle (Search, Extract, Agent). +2. **Side-by-Side-Vergleich** erlaubt (gleiche Query → N Provider parallel, normalisierte + rohe Ergebnisse in PostgreSQL). +3. **Auto-Routing** nach Query-Typ macht (News → Tavily, Paper → Exa, komplexe Frage → Perplexity, …). +4. **Pay-per-use-only**: keine Services mit Monats-Abo, nur API-Call-basierte Abrechnung. +5. **mana-credits** integriert ist — jeder Call kostet je nach Provider + Operation. +6. **Hybrid-Keys** unterstützt — Server-Keys als Default (charged via credits), BYO-Keys pro User optional (kein Credit-Verbrauch). +7. **Frontend Research Lab** liefert — UI zum Eingeben von Queries, Auswahl von Providern, Side-by-Side-Review, manuelles Rating. + +## Abgrenzung + +- **`mana-search`** (Go, Port 3021) bleibt als SearXNG+Readability-Primitive bestehen und wird *ein* Provider im neuen Service. Keine Ablösung, nur Umfassung. +- **`mana-crawler`** (Go, Port 3023) bleibt als Deep-Crawl-Tool separat. Wird nicht Teil der Research-Pipeline. +- **`news-research`-Modul** im Frontend und die `/api/v1/news-research/*`-Routen in mana-api bleiben zunächst parallel bestehen und werden in Phase 3 als dünner Adapter auf den neuen Service umgestellt. +- **`mana-ai`** behält seinen eigenen NewsResearchClient, wird in Phase 3 auf `mana-research` umgestellt. + +--- + +## 1. Architektur + +``` + apps/mana web / apps/api / mana-ai + │ + ▼ + mana-research (3068) + │ + ┌──────────────────┼──────────────────┐ + ▼ ▼ ▼ + SearchProvider ExtractProvider ResearchAgent + │ │ │ + ┌──────────┼──────────┐ │ ┌──────────┼──────────┐ + │ │ │ │ │ │ │ + SearXNG Brave Tavily Readab Perplex Claude OpenAI + DDG Exa Serper Jina Sonar Web-Search Responses + Mojeek SerpAPI Firecr Gemini DR (async) + ScrapBee Ground + + ┌──────────────────────────────────┐ + │ Redis Cache (query+provider) │ + │ Postgres research.{eval_runs, │ + │ eval_results, │ + │ provider_configs, │ + │ provider_stats} │ + │ mana-credits (cost tracking) │ + │ mana-llm (query classification) │ + └──────────────────────────────────┘ +``` + +### Service-Grenze + +- **Eigener Hono-Server** in `services/mana-research/src/server.ts`. +- **Auth:** `@mana/shared-hono` authMiddleware wie alle anderen Services. Interne Service-to-Service-Calls (mana-ai) nutzen weiterhin den vorhandenen Muster ohne User-Auth. +- **CORS-Origins:** `*.mana.how` + `localhost:5173` — wird in Phase 1 gesetzt. +- **Dependencies:** `mana-llm` (für Query-Klassifikation), `mana-credits` (für Cost-Tracking), `mana-search` (als Provider gewrappt), PostgreSQL (`mana_platform`), Redis (Cache). + +### Provider-Interfaces (TypeScript, exportiert aus `packages/shared-research`) + +Neues Shared-Package `@mana/shared-research` für Typen, die vom Frontend (research-lab Modul) und Backend (mana-research + mana-api) gemeinsam genutzt werden. + +```ts +// Gemeinsame Metadaten jedes Provider-Calls +export interface ProviderMeta { + provider: ProviderId; // 'brave' | 'tavily' | ... + category: 'search' | 'extract' | 'agent'; + latencyMs: number; + costCredits: number; // via creditsPerCall mapping + cacheHit: boolean; + billingMode: 'server-key' | 'byo-key'; + errorCode?: string; // wenn success=false +} + +export interface SearchHit { + url: string; + title: string; + snippet: string; + publishedAt?: string; + author?: string; + score?: number; // provider-eigenes Ranking + content?: string; // wenn Provider direkt Text liefert (Tavily) + providerRaw: unknown; // roh für Debug +} + +export interface ExtractedContent { + url: string; + title: string; + content: string; // Markdown oder plain text + excerpt?: string; + author?: string; + siteName?: string; + publishedAt?: string; + wordCount: number; + providerRaw: unknown; +} + +export interface AgentAnswer { + query: string; + answer: string; // synthetisierte Antwort, Markdown + citations: Array<{ url: string; title: string; snippet?: string }>; + followUpQueries?: string[]; + tokenUsage?: { input: number; output: number }; + providerRaw: unknown; +} + +export interface SearchProvider { + id: ProviderId; + capabilities: { + webSearch: boolean; + newsSearch: boolean; + scholarSearch: boolean; + semanticSearch: boolean; + contentInResults: boolean; // true wenn Tavily-style + }; + search(query: string, opts: SearchOptions): Promise<{ + results: SearchHit[]; + meta: ProviderMeta; + }>; +} + +export interface ExtractProvider { + id: ProviderId; + capabilities: { + jsRendering: boolean; + pdfSupport: boolean; + markdownOutput: boolean; + }; + extract(url: string, opts: ExtractOptions): Promise<{ + content: ExtractedContent; + meta: ProviderMeta; + }>; +} + +export interface ResearchAgent { + id: ProviderId; + capabilities: { + multiStep: boolean; + async: boolean; + withCitations: boolean; + }; + research(query: string, opts: AgentOptions): Promise<{ + answer: AgentAnswer; + meta: ProviderMeta; + }>; +} +``` + +### Service-Modi (HTTP-Endpoints) + +| Endpoint | Zweck | +|---|---| +| `POST /v1/search` | Single-Provider-Search. Body: `{ query, provider?, options }`. Wenn provider fehlt → Auto-Router. | +| `POST /v1/search/compare` | Fan-Out an N Provider, speichert Run in `research.eval_runs`. Body: `{ query, providers: [...], options }`. | +| `POST /v1/extract` | Single-Provider-Extract. Body: `{ url, provider? }`. | +| `POST /v1/extract/compare` | Fan-Out für Extract. | +| `POST /v1/research` | Agent-Mode. Query → synthesierte Antwort + Zitate. Body: `{ query, agent?, options }`. | +| `POST /v1/research/compare` | Fan-Out an N Agents. | +| `GET /v1/providers` | Liste aller registrierten Provider + Capabilities + aktuelle Kosten. | +| `GET /v1/providers/health` | Welche Provider sind aktuell erreichbar / haben gültige Keys? | +| `GET /v1/runs` | User's gespeicherte Eval-Runs (paginiert). | +| `GET /v1/runs/:id` | Einzelner Run mit allen Results. | +| `POST /v1/runs/:id/results/:resultId/rate` | User-Rating für Eval. | +| `GET /health`, `/metrics` | Standard | + +--- + +## 2. Provider-Inventar + +Alle Provider nur mit **Pay-per-use**-Abrechnung, kein fester Monatsbetrag nötig. + +### 2.1 Search Providers (6) + +| Provider | Stärke | Kosten-Modell | Key nötig? | `costCredits` (geschätzt) | +|---|---|---|---|---| +| `searxng` | Self-hosted, unabhängig | Infrastruktur only | Nein | 0 | +| `brave` | Unabhängiger Index, Privacy | $5/1k Queries (PAYG) | Ja | 5 (≈ $0.005) | +| `tavily` | Agent-optimiert, liefert Content + Answer | Credit-Packs, kein Abo-Zwang ($30 = 4k credits, persistent) | Ja | 8 | +| `exa` | Semantische/Embedding-basierte Suche, Papers | $0.001-0.01/Query PAYG | Ja | 3–10 je nach Modus | +| `serper` | Google-SERP als JSON (inkl. People-Also-Ask, Knowledge Panel) | $0.30-1/1k (PAYG) | Ja | 1 | +| `duckduckgo` | Free Instant-Answer API | Kostenlos, rate-limited | Nein | 0 | + +Optional später: `mojeek` (unabhängiger Index, PAYG), `serpapi` (Alternative zu Serper, höhere Coverage $5/1k). + +### 2.2 Extract Providers (5) + +| Provider | Stärke | Kosten-Modell | Key nötig? | `costCredits` | +|---|---|---|---|---| +| `readability` | Mozilla Readability via `mana-search` | Kostenlos (self-hosted) | Nein | 0 | +| `jina-reader` | `r.jina.ai`, extrem robust, Markdown out | Free-Tier 1M tokens/Monat, dann $0.02/1M tokens | Optional (höheres Rate-Limit mit Key) | 1 | +| `firecrawl` | JS-Rendering via Playwright, Batch-Crawls | PAYG Credits (oder self-host via Docker, dann 0) | Ja (bzw. intern für self-host) | 10 | +| `scrapingbee` | Proxy + JS-Render | PAYG Credits | Ja | 8 | +| `crawl4ai` | Self-hosted OSS (Python + Playwright) | Infrastruktur only | Nein | 0 | + +### 2.3 Research Agents (5) + +| Provider | Stärke | Kosten-Modell | Key nötig? | `costCredits` | +|---|---|---|---|---| +| `perplexity-sonar` | Beste Plug-and-Play-Research-API, 4 Modelle (`sonar`, `sonar-pro`, `sonar-reasoning`, `sonar-deep-research`) | Token-based PAYG ($1-15/1M input + search fees) | Ja | 50–500 je nach Modell | +| `claude-web-search` | Claude-API mit server-seitigem `web_search`-Tool | Token-based + $10/1k searches | Ja (Anthropic) | 100–300 | +| `openai-responses` | OpenAI Responses API mit `web_search_preview`-Tool | Token-based + per-tool-call | Ja (OpenAI) | 100–300 | +| `gemini-grounding` | Gemini + Google Search Grounding | Token-based + per-grounding-query | Ja (Google) | 80–200 | +| `openai-deep-research` | Async, autonomer Multi-Step-Agent | Pay-per-task (~$0.10-1/task) | Ja | 1000+ (async via Job-Queue) | + +**Preise sind Stand 2026-04-17** und werden zentral in `services/mana-research/src/providers/pricing.ts` gepflegt und im Service-Startup aus einer JSON-Datei gezogen (updatebar ohne Redeploy). + +--- + +## 3. Daten-Modell + +Neues PostgreSQL-Schema `research` in `mana_platform`. Drizzle-Schema in `services/mana-research/src/db/schema.ts`. + +```ts +export const researchSchema = pgSchema('research'); + +// Ein Run = eine Query, ein oder mehrere Provider +export const evalRuns = researchSchema.table('eval_runs', { + id: uuid('id').primaryKey().defaultRandom(), + userId: text('user_id'), // null für Service-to-Service + query: text('query').notNull(), + queryType: text('query_type'), // 'news' | 'general' | 'semantic' | 'academic' | 'code' + mode: text('mode').notNull(), // 'single' | 'compare' | 'auto' + category: text('category').notNull(),// 'search' | 'extract' | 'agent' + providersRequested: text('providers_requested').array().notNull(), + billingMode: text('billing_mode').notNull(), // 'server-key' | 'byo-key' | 'mixed' + totalCostCredits: integer('total_cost_credits').notNull().default(0), + createdAt: timestamp('created_at').notNull().defaultNow(), +}); + +// Ein Result = Antwort von genau einem Provider +export const evalResults = researchSchema.table('eval_results', { + id: uuid('id').primaryKey().defaultRandom(), + runId: uuid('run_id').notNull().references(() => evalRuns.id, { onDelete: 'cascade' }), + providerId: text('provider_id').notNull(), + success: boolean('success').notNull(), + latencyMs: integer('latency_ms').notNull(), + costCredits: integer('cost_credits').notNull().default(0), + billingMode: text('billing_mode').notNull(), + cacheHit: boolean('cache_hit').notNull().default(false), + rawResponse: jsonb('raw_response'), // Provider-spezifisch, für Debug + normalizedResult: jsonb('normalized_result'),// SearchHit[] | ExtractedContent | AgentAnswer + errorCode: text('error_code'), + errorMessage: text('error_message'), + userRating: integer('user_rating'), // 1-5, null wenn nicht bewertet + userNotes: text('user_notes'), + createdAt: timestamp('created_at').notNull().defaultNow(), +}); + +// Per-User Provider-Config (API-Keys, Limits) +export const providerConfigs = researchSchema.table('provider_configs', { + id: uuid('id').primaryKey().defaultRandom(), + userId: text('user_id'), // null = server-wide default + providerId: text('provider_id').notNull(), + apiKeyEncrypted: text('api_key_encrypted'),// verschlüsselt via shared-crypto (KEK-wrapped, AES-GCM) + enabled: boolean('enabled').notNull().default(true), + dailyBudgetCredits: integer('daily_budget_credits'), + monthlyBudgetCredits: integer('monthly_budget_credits'), + createdAt: timestamp('created_at').notNull().defaultNow(), + updatedAt: timestamp('updated_at').notNull().defaultNow(), +}, (t) => ({ + userProvider: uniqueIndex('user_provider_unique').on(t.userId, t.providerId), +})); + +// Aggregierte Stats für Admin-Dashboard & Auto-Router +export const providerStats = researchSchema.table('provider_stats', { + providerId: text('provider_id').notNull(), + day: date('day').notNull(), + totalCalls: integer('total_calls').notNull().default(0), + totalLatencyMs: bigint('total_latency_ms', { mode: 'number' }).notNull().default(0), + totalCostCredits:integer('total_cost_credits').notNull().default(0), + successCount: integer('success_count').notNull().default(0), + errorCount: integer('error_count').notNull().default(0), + avgUserRating: real('avg_user_rating'), + ratingCount: integer('rating_count').notNull().default(0), +}, (t) => ({ + pk: primaryKey({ columns: [t.providerId, t.day] }), +})); +``` + +**Encryption:** `providerConfigs.apiKeyEncrypted` wird via `shared-crypto` (existierendes AES-GCM + KEK-Pattern) verschlüsselt. Eintrag in `apps/mana/apps/web/src/lib/data/crypto/registry.ts` nötig, falls Client-Cache der Keys kommt — sonst serverseitig über `mana-auth` KEK. + +--- + +## 4. Credits-Integration + +**Prinzip:** Jeder Provider-Call hat festen `costCredits`-Preis (siehe §2). Bei Server-Key-Mode → Credits werden via `mana-credits` vom User abgezogen. Bei BYO-Key → 0 Credits, aber wir loggen den Call trotzdem (Usage-Transparenz). + +### Flow pro Call + +```ts +// services/mana-research/src/middleware/credits.ts +async function chargeForCall(userId, providerId, operation) { + const config = await loadUserConfig(userId, providerId); + const isByoKey = !!config?.apiKeyEncrypted; + + if (isByoKey) { + return { billingMode: 'byo-key', costCredits: 0, apiKey: decrypt(config.apiKeyEncrypted) }; + } + + // Server-Key-Mode: Credits prüfen + reservieren + const price = PROVIDER_PRICING[providerId][operation]; // aus pricing.ts + const balance = await manaCredits.getBalance(userId); + if (balance < price) throw new HTTPException(402, 'Insufficient credits'); + + // Soft-Reserve (atomic decrement) + await manaCredits.reserve(userId, price, `research:${providerId}:${operation}`); + + return { billingMode: 'server-key', costCredits: price, apiKey: SERVER_KEYS[providerId] }; +} + +// Nach erfolgreichem Call: commit (bei Fehler: refund) +async function finalizeCharge(userId, reservationId, success) { + if (success) await manaCredits.commit(reservationId); + else await manaCredits.refund(reservationId); +} +``` + +### mana-credits API (zu verifizieren / ggf. zu erweitern) + +Erwartete Endpoints: +- `GET /api/v1/credits/balance?userId=...` +- `POST /api/v1/credits/reserve` `{ userId, amount, reason }` → `{ reservationId }` +- `POST /api/v1/credits/commit` `{ reservationId }` +- `POST /api/v1/credits/refund` `{ reservationId }` +- `GET /api/v1/credits/usage?userId=...&since=...&filter=research:*` + +**Falls** `mana-credits` diese Granularität noch nicht hat → Task in Phase 1 ergänzen: Reserve/Commit/Refund-Pattern einführen. + +### Budget-Enforcement + +Pro User konfigurierbar in `providerConfigs.dailyBudgetCredits` / `monthlyBudgetCredits`. Default: unlimited (nur von `mana-credits`-Balance limitiert). UI in Settings → "Research Providers" (Phase 4). + +--- + +## 5. BYO-Keys (Hybrid-Modus) + +### UX + +Settings-Seite `/settings/research-providers` (Phase 4): +- Liste aller Provider mit Toggle "Server-Key nutzen" vs. "Eigener Key" +- Eingabefeld für API-Key (wird nicht angezeigt, nur `••••••`) +- Pro Provider optional tägliches/monatliches Budget setzen + +### Storage + +- Keys verschlüsselt via bestehender `shared-crypto`-Pipeline (AES-GCM-256, KEK aus `mana-auth`) +- Tabelle `research.provider_configs` mit `userId` + `providerId` unique constraint +- Beim Call: wenn User-Config existiert → verwende deren Key → `billingMode = 'byo-key'` → kein Credit-Verbrauch + +### Server-Keys + +- Zentral in Env-Vars: `BRAVE_API_KEY`, `TAVILY_API_KEY`, `EXA_API_KEY`, `SERPER_API_KEY`, `PERPLEXITY_API_KEY`, `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GOOGLE_GENAI_API_KEY`, `FIRECRAWL_API_KEY`, `SCRAPINGBEE_API_KEY`, `JINA_API_KEY` +- Einheitlich via `services/mana-research/src/config.ts` +- In `.env.development` dokumentiert + `scripts/generate-env.mjs` ergänzt + +--- + +## 6. Query-Klassifikation & Auto-Router + +Für `mode: 'auto'`: Query wird via `mana-llm` (gemma3:4b) klassifiziert → bester Provider je Typ. + +```ts +// services/mana-research/src/router/classify.ts +export type QueryType = 'news' | 'general' | 'semantic' | 'academic' | 'code' | 'conversational'; + +const ROUTE_MAP: Record = { + news: ['tavily', 'brave', 'searxng'], // Tavily first (News-strong) + general: ['brave', 'tavily', 'serper'], + semantic: ['exa', 'tavily'], // Exa für "find similar to" + academic: ['exa', 'searxng'], // Exa findet Papers am besten + code: ['exa', 'serper', 'searxng'], + conversational: ['perplexity-sonar', 'claude-web-search'],// agent mode für Frage-Antwort +}; + +// Fallback-Chain: wenn Primary fehlschlägt, nächster in Liste +``` + +Klassifikation ist optional und fällt bei LLM-Timeout auf `'general'` zurück. + +--- + +## 7. Phasen-Plan + +### Phase 1 — Foundation + Core Providers ✅ (2026-04-17) + +**Ziel:** Service läuft mit 4 Search-Providern + grundlegender Cache, Credits-Integration, `/search` + `/search/compare` Endpoints. + +- [ ] Service-Scaffold `services/mana-research/` (Bun/Hono + Drizzle + `@mana/shared-hono`) +- [ ] Shared-Package `packages/shared-research` für Provider-Typen +- [ ] DB-Migration: `research` schema (4 Tabellen) +- [ ] Provider-Adapter: + - `SearXNGProvider` (wrapt `mana-search`) + - `BraveSearchProvider` + - `TavilyProvider` + - `DuckDuckGoProvider` (als kostenloser Fallback) +- [ ] `POST /v1/search` + `POST /v1/search/compare` +- [ ] Redis-Cache (key: `research:${category}:${provider}:${sha256(query+opts)}`, TTL 1h) +- [ ] Credits-Middleware mit Reserve/Commit/Refund +- [ ] `pricing.ts` + `PROVIDER_PRICING`-Map +- [ ] Docker-Compose-Eintrag (`docker-compose.yml` + `docker-compose.macmini.yml`) +- [ ] Port-Eintrag in [`docs/PORT_SCHEMA.md`](../PORT_SCHEMA.md) +- [ ] Env-Vars: `BRAVE_API_KEY`, `TAVILY_API_KEY` in `.env.development` +- [ ] `services/mana-research/CLAUDE.md` +- [ ] Eintrag im Root-CLAUDE.md "Active services" +- [ ] **Falls nötig:** Erweiterung `mana-credits` um Reserve/Commit/Refund +- [ ] Integration-Tests: `tests/search-providers.spec.ts` (Mock-HTTP) + +### Phase 2 — Extraction + semantische Suche ✅ (2026-04-17) + +- [x] Provider-Adapter: + - `ReadabilityProvider` (wrapt `mana-search /extract`) + - `JinaReaderProvider` (zero-auth, optionaler Key für höheres Rate-Limit) + - `FirecrawlProvider` (PAYG + self-hostbar via `FIRECRAWL_API_URL`) + - ~~`ScrapingBeeProvider`~~ — deferred: liefert raw HTML, braucht zusätzlichen Readability-Pass. Wird als Phase-3-Addition behandelt. + - `ExaProvider` (semantische Suche in Phase 2 gelandet) + - `SerperProvider` (Google SERP als JSON) +- [x] `POST /v1/extract` + `POST /v1/extract/compare` +- [x] Query-Klassifikation: `classify.ts` mit Regex-Fast-Path + optionalem `mana-llm`-Call (`useLlmClassifier: true`) +- [x] Auto-Router in `POST /v1/search` (wenn `provider` nicht gesetzt) + Auto-Router für Extract +- [x] Provider-Health-Check-Endpoint `GET /v1/providers/health` +- [x] Run-Listen-Endpoints bereits in Phase 1 geliefert +- [x] ~~Nightly-Job~~: Live-Aggregation im `addResult()`-Pfad via `onConflictDoUpdate` genügt für Phase 2. + +### Phase 3 — Research Agents + mana-ai Migration (≈ 1–2 Wochen) + +- [ ] Provider-Adapter: + - `PerplexitySonarProvider` (4 Modelle: sonar, sonar-pro, sonar-reasoning, sonar-deep-research) + - `ClaudeWebSearchProvider` (via Anthropic SDK + tool-use) + - `OpenAIResponsesProvider` (via OpenAI SDK + `web_search_preview` tool) + - `GeminiGroundingProvider` (via google-genai SDK mit Search-Grounding) + - `OpenAIDeepResearchProvider` — **async**, via BullMQ/inline Job-Queue, Response-Endpoint `GET /v1/research/tasks/:id` +- [ ] `POST /v1/research` + `POST /v1/research/compare` +- [ ] Auto-Router für `conversational`-Queries → Agent-Mode +- [ ] `mana-llm` um Anthropic- und OpenAI-Provider erweitern (nur für Claude/OpenAI Agents; restlicher LLM-Workflow bleibt Ollama-first) +- [ ] **Migration:** `apps/api/src/modules/news-research/routes.ts` wird zum dünnen Adapter auf `mana-research` +- [ ] **Migration:** `services/mana-ai/src/planner/news-research-client.ts` ruft jetzt `mana-research` direkt statt `mana-api` +- [ ] **Migration:** `research_news`-Tool bekommt Option `depth: 'shallow' | 'deep'`; `deep` ruft Agent-Mode +- [ ] Altes `mana-api/news-research/*` bleibt als Backward-Compat, logged Deprecation-Warning + +### Phase 4 — Research Lab UI (≈ 1 Woche) + +**Neues Frontend-Modul** `apps/mana/apps/web/src/lib/modules/research-lab/` (tier: `beta`). + +- [ ] Routes: + - `/research-lab` — Main: Query-Input + Provider-Multi-Select + Compare-Button + - `/research-lab/runs` — Liste gespeicherter Runs mit Filter (query type, datum, provider) + - `/research-lab/runs/[id]` — Side-by-Side-View: Columns pro Provider, Ratings, Notes, Export +- [ ] Module-Store: `researchLab.svelte.ts` (Runs, Results, aktueller Vergleich) +- [ ] Component: `ProviderPicker.svelte` (gruppiert nach Kategorie, zeigt Kosten + Capabilities) +- [ ] Component: `ResultCard.svelte` (normalisierter Result + Rating-UI + "raw" Toggle) +- [ ] Component: `ComparisonGrid.svelte` +- [ ] Settings-Integration: `/settings/research-providers` für BYO-Keys + Budgets +- [ ] Tool-Schema erweitert: `research_news` mit neuen Options, neues `deep_research` Tool (propose-policy wegen höherer Kosten) +- [ ] Tier-Gate: `beta`+ für Research Lab, `research_news` bleibt `public`+ +- [ ] Registrierung in `apps/mana/apps/web/src/lib/app-registry/apps.ts` +- [ ] Seed-Handler für Demo-Runs in `apps/mana/apps/web/src/lib/data/seed-registry.ts` + +--- + +## 8. Offene Fragen + +- [ ] **`mana-credits` API-Shape**: hat der Service schon Reserve/Commit/Refund oder nur Debit? Muss geprüft werden in Phase 1 Task 0. +- [ ] **Rate-Limiting**: separate per-User Rate-Limits pro Provider (damit ein Power-User nicht das Quota für alle aufbraucht)? Meine Tendenz: via `provider_configs.dailyBudgetCredits` reicht als Soft-Limit. +- [ ] **Export-Format für Eval-Runs**: JSON-Download reicht, oder brauchen wir CSV/Markdown-Export für manuelle Analyse? +- [ ] **Ollama als Agent?** Wir könnten lokale LLMs mit Tool-Use-Capabilities (z.B. Llama 3.1 tool calling + eigener search-tool-loop) als "self-hosted agent" anbieten — wäre konsistent mit Self-Hosting-Positioning. Phase 5-Kandidat. +- [ ] **Caching-TTL pro Provider**: News sollten kürzer cachen als akademische Papers. Default 1h, aber konfigurierbar pro Provider-Config. +- [ ] **Kosten-Dashboard**: wann bauen wir ein Admin-Dashboard für `provider_stats`? Phase 4 oder später? + +--- + +## 9. Verzeichnisstruktur (Vorschlag) + +``` +services/mana-research/ +├── CLAUDE.md +├── package.json +├── tsconfig.json +├── Dockerfile +├── drizzle.config.ts +├── src/ +│ ├── server.ts # Hono bootstrap +│ ├── config.ts # Env-Vars +│ ├── db/ +│ │ ├── schema.ts +│ │ └── migrations/ +│ ├── providers/ +│ │ ├── index.ts # registerAll() +│ │ ├── types.ts # re-export @mana/shared-research +│ │ ├── pricing.ts # PROVIDER_PRICING map +│ │ ├── search/ +│ │ │ ├── searxng.ts +│ │ │ ├── brave.ts +│ │ │ ├── tavily.ts +│ │ │ ├── exa.ts +│ │ │ ├── serper.ts +│ │ │ └── duckduckgo.ts +│ │ ├── extract/ +│ │ │ ├── readability.ts +│ │ │ ├── jina-reader.ts +│ │ │ ├── firecrawl.ts +│ │ │ └── scrapingbee.ts +│ │ └── agent/ +│ │ ├── perplexity-sonar.ts +│ │ ├── claude-web-search.ts +│ │ ├── openai-responses.ts +│ │ ├── gemini-grounding.ts +│ │ └── openai-deep-research.ts +│ ├── routes/ +│ │ ├── search.ts +│ │ ├── extract.ts +│ │ ├── research.ts +│ │ ├── runs.ts +│ │ └── providers.ts +│ ├── middleware/ +│ │ ├── credits.ts # Reserve/Commit/Refund +│ │ ├── byo-keys.ts # Key-Resolution +│ │ └── cache.ts # Redis wrapper +│ ├── router/ +│ │ ├── classify.ts # Query → QueryType via mana-llm +│ │ └── auto-route.ts # QueryType → Provider-Chain +│ ├── storage/ +│ │ ├── runs.ts # persist + retrieve +│ │ └── stats.ts # aggregation job +│ └── clients/ +│ ├── mana-llm.ts +│ ├── mana-credits.ts +│ └── mana-auth.ts +└── tests/ + ├── providers/ + └── integration/ + +packages/shared-research/ +├── package.json +└── src/ + ├── index.ts + ├── types.ts # SearchHit, ExtractedContent, AgentAnswer, ProviderMeta + ├── options.ts # SearchOptions, ExtractOptions, AgentOptions + └── ids.ts # ProviderId union +``` + +--- + +## 10. Risiken + +| Risiko | Wahrscheinlichkeit | Impact | Mitigation | +|---|---|---|---| +| Provider-Pricing ändert sich | Hoch | Mittel | `pricing.ts` updatebar ohne Redeploy, Warnung wenn Kosten > X | +| Provider-Key wird gebannt (z.B. Brave bei zu viel Traffic) | Mittel | Hoch | Multi-Provider-Fallback in Auto-Router, Health-Check | +| `mana-credits` Reserve/Refund-Pattern nicht ready | Mittel | Hoch | Phase 1 Task 0: verifizieren + ggf. implementieren vor allem anderen | +| Kosten-Explosion durch Fan-Out-Compare | Mittel | Hoch | Hartes Limit auf `providers.length` (max 5), UI-Warnung mit Kosten-Preview | +| BYO-Key-Leak durch Logs | Niedrig | Sehr hoch | Keys nur verschlüsselt in DB, Logger-Filter für bekannte Key-Patterns | +| LLM-Klassifikation falsch → schlechter Auto-Route | Mittel | Niedrig | User kann immer explizit Provider wählen, Klassifikation ist nur Hint | + +--- + +## 11. Verwandte Dateien + +**Heute existierende Research-Pfade (werden in Phase 3 umgestellt):** +- `apps/api/src/lib/search.ts` — `webSearch()` Wrapper +- `apps/api/src/modules/news-research/routes.ts` — `/discover`, `/search`, `/extract` +- `services/mana-ai/src/planner/news-research-client.ts` +- `services/mana-ai/src/cron/tick.ts:282` — Pre-Planning Research Step +- `apps/mana/apps/web/src/lib/modules/news-research/tools.ts:48` — `research_news` Tool +- `services/mana-search/` — bleibt, wird SearXNG+Readability-Provider +- `packages/shared-rss/` — bleibt unverändert + +**Neu zu erstellen:** +- `services/mana-research/**` +- `packages/shared-research/**` +- `apps/mana/apps/web/src/lib/modules/research-lab/**` +- `apps/mana/apps/web/src/routes/(app)/research-lab/**` diff --git a/docs/reports/web-research-capabilities.md b/docs/reports/web-research-capabilities.md new file mode 100644 index 000000000..3d78c5658 --- /dev/null +++ b/docs/reports/web-research-capabilities.md @@ -0,0 +1,324 @@ +# Web-Recherche im Mana-System — Bestandsaufnahme & Vergleich + +**Datum:** 2026-04-17 +**Scope:** Wie recherchiert das System heute im Internet, wie gut ist das, und welche Alternativen gibt es? + +--- + +## 1. TL;DR + +Mana hat eine **vollständig selbstgehostete, RSS-zentrierte Recherche-Pipeline**: + +``` +User / AI-Mission + ↓ +research_news Tool (Frontend) bzw. NewsResearchClient (mana-ai Backend) + ↓ +mana-api /api/v1/news-research/{discover,search,extract} + ↓ +mana-search (Go, SearXNG-Proxy) + @mana/shared-rss (RSS + Readability) + ↓ +SearXNG (lokal, Port 8080) → Google / Bing / DDG / Brave / Wikipedia / arXiv / GitHub / ... +``` + +**Stärken:** Null API-Kosten, voller Datenschutz, kein Vendor-Lock-in, robuste Inhaltsextraktion (Mozilla Readability), RSS-First-Ansatz liefert strukturierte, zitierbare Quellen. + +**Schwächen:** Kein semantisches Ranking (nur Keyword-TF), keine agentische Multi-Step-Recherche, keine Paywall-/JS-Heavy-Site-Handling, keine Cross-Source-Synthese, SearXNG ist fragil gegen Provider-Blocks (Google blockt ihn regelmäßig). Ergebnisqualität liegt etwa bei **30–50 %** dessen, was Perplexity/Exa/OpenAI Deep Research heute liefern. + +**Empfehlung:** Hybrid-Modell. Self-Hosting als Default behalten, aber eine **optionale API-Bridge** (Tavily, Exa, oder Brave Search API) als Fallback bzw. "Premium-Recherche" einbauen. Siehe §6. + +--- + +## 2. Was das System HEUTE kann + +### 2.1 Service-Landschaft + +| Service | Port | Sprache | Rolle | +|---|---|---|---| +| `mana-search` | 3021 | Go | SearXNG-Proxy + Readability-Extract (zentral) | +| `mana-crawler` | 3023 | Go | Deep-Crawls mit Depth/Selector-Support (**wird nicht** von Research genutzt) | +| `mana-api` | 3060 | Bun/Hono | Orchestriert News-Research-Endpoints | +| `mana-ai` | 3067 | Bun | Background Mission Runner mit Pre-Planning Research Step | +| `mana-llm` | 3025 | Python | LLM-Gateway (Ollama-First, Gemini-Fallback) | +| `news-ingester` | 3066 | Bun | Cron-basiertes Vorab-Pooling kuratierter RSS-Quellen (passiv) | +| SearXNG | 8080 | — | Meta-Such-Frontend über ~15 Engines | + +### 2.2 Die vier Primitive + +**1. Web-Suche** (`mana-search` → SearXNG) +- File: `services/mana-search/internal/search/searxng.go` +- Kategorien: `general` (Google, Bing, DDG, Brave, Wikipedia), `news` (Google/Bing News), `science` (arXiv, Scholar, PubMed), `it` (GitHub, StackOverflow, NPM, MDN) +- Redis-Cache (TTL 1h), Prometheus-Metriken, Graceful-Degradation +- **Limitation:** SearXNG wird von Google regelmäßig per CAPTCHA geblockt; Fallback auf DDG/Bing + +**2. Inhaltsextraktion** (`mana-search` + `@mana/shared-rss`) +- go-shiori/go-readability (Go-Port von Mozilla Readability) im Service +- `@mozilla/readability` + jsdom in der shared-rss Lib +- Bulk-Extract bis 20 URLs parallel (`POST /api/v1/extract/bulk`) +- Redis-Cache 24h, Max-Length 50k chars +- **Limitation:** Kein JS-Rendering (Playwright fehlt in Research-Pfad), Paywalls werden nicht umgangen, keine PDF-Extraktion + +**3. RSS-Feed-Discovery** (`packages/shared-rss`) +- File: `packages/shared-rss/src/discover.ts` +- Strategy: `` scannen → Fallback auf Common-Paths (`/feed`, `/rss.xml`, `/atom.xml`, `/feeds/posts/default`, ...) +- `rss-parser` für RSS 1.0/2.0 und Atom +- **Stärke:** Trifft strukturierte Inhalte direkt von der Quelle (keine HTML-Extraktion nötig) + +**4. AI-Research-Tool** (`research_news`) +- File: `apps/mana/apps/web/src/lib/modules/news-research/tools.ts:48` +- Policy: `auto` (Tool läuft ohne User-Approval) +- Pipeline: Query → `discover` (Web-Search nach "query rss feed") → `search` (Top 10 Feeds parsen + rank) → formatter Markdown-Context +- Ranking: Keyword-Frequency (title +3, excerpt +2, content +1) + Recency-Boost (<24h: +2, <7d: +1) +- **Limitation:** Kein BM25, keine Embeddings, keine Query-Expansion, keine Cross-Source-Deduplication + +### 2.3 Integration in AI-Missions + +Der `mana-ai` Tick-Runner hat einen **Pre-Planning-Step** (`services/mana-ai/src/cron/tick.ts:282`): + +```typescript +if (RESEARCH_TRIGGER.test(m.objective) || RESEARCH_TRIGGER.test(m.conceptMarkdown)) { + const research = await nrc.research(m.objective, { language: 'de', limit: 8 }); + resolvedInputs.push({ + id: '__web-research__', + module: 'news-research', + table: 'web', + title: `Web-Research: "${m.objective.slice(0, 60)}"`, + content: research.contextMarkdown, + }); +} +``` + +Regex triggert auf: `recherchier|research|news|finde|suche|aktuelle|neueste|today|history|historisch|on this day`. Ergebnis wird als ResolvedInput in den Planner-Prompt injiziert → Planner sieht echte URLs + Excerpts, soll nur diese zitieren. + +### 2.4 LLM-Integration (mana-llm) + +Router unterstützt: Ollama (primary, lokal), Google Gemini (Fallback), OpenRouter, Groq, Together. + +**Wichtig:** Weder OpenAI noch Anthropic sind konfiguriert. Das heißt: +- **Kein** Claude `web_search` / `fetch_url` Tool-Use +- **Kein** OpenAI Browsing / Deep Research +- **Kein** Gemini Grounding mit Google Search +- Die gesamte Research-Logik ist **explizit in mana-api kodiert** — kein LLM macht hier eigenständig Tool-Calls ins Web. + +--- + +## 3. Qualitätseinschätzung + +### Bewertungsmatrix (1 = schwach, 5 = state-of-the-art) + +| Dimension | Score | Begründung | +|---|---|---| +| **Coverage** (wie viel vom Web) | 2/5 | Nur Quellen mit RSS + was SearXNG liefert (<30 % typisch für News, <5 % für Fachwissen) | +| **Recency** | 3/5 | RSS-Feeds meist ≤ 1h Verzögerung, SearXNG liefert tagesaktuell. Aber: Pre-Pool (`news-ingester`) ist 15-min-Cron | +| **Inhaltsqualität** | 4/5 | Mozilla Readability ist best-in-class OSS-Extraktion. Schwach bei Paywalls, JS-SPAs, PDFs | +| **Semantisches Ranking** | 1/5 | Nur TF + Recency-Boost. Keine Embeddings, kein BM25, keine Reranker | +| **Multi-Hop-Recherche** | 1/5 | Ein Shot: Query → Feeds → Artikel. Keine Iteration, kein "ich hab X gefunden, jetzt suche ich Y" | +| **Synthese & Zusammenfassung** | 2/5 | Wird dem LLM überlassen (gemma3:4b). Keine spezialisierten Research-Prompts, keine Quellen-Cross-Validierung | +| **Grounding / Zitierbarkeit** | 4/5 | URLs + Excerpts werden strukturiert übergeben; Planner-Prompt warnt explizit gegen URL-Halluzination | +| **Latenz** | 2/5 | Discover + Search + Extract in Serie: 3–15 s typisch. Bei Timeouts: deutlich langsamer | +| **Kosten** | 5/5 | Null API-Kosten, nur Infrastruktur | +| **Datenschutz** | 5/5 | Keine Queries verlassen die eigene Infra (bis auf den SearXNG → Google/Bing Roundtrip, der aber über SearXNG anonymisiert ist) | +| **Robustheit** | 2/5 | SearXNG wird von Google regelmäßig geblockt, Fallbacks sind vorhanden, aber Qualität fällt dann stark | + +**Gesamt-Score: ~2.8/5** — solide für das, was es ist (ein RSS-getriebener News-Aggregator mit AI-Integration), aber weit entfernt von dem, was moderne Research-Agents (Perplexity Sonar, Claude mit `web_search`, OpenAI Deep Research, Gemini Deep Research) leisten. + +### Konkrete Qualitäts-Gaps + +1. **Kein semantisches Matching.** Query "Auswirkungen von Zinssenkungen auf Immobilienmarkt" → findet nur Artikel, die genau diese Keywords haben, nicht solche mit "EZB senkt Leitzins, Hypotheken werden billiger". +2. **Kein Multi-Step.** "Was sagt Studie X zu Thema Y?" erfordert: 1) Studie finden, 2) Autoren identifizieren, 3) Original-Paper lesen, 4) Kritikpunkte recherchieren. System macht nur Schritt 1–2 rudimentär. +3. **Keine Inhalts-Synthese.** Wenn 8 Quellen dasselbe Ereignis berichten, werden 8 Texte ans LLM gegeben — keine automatische Dedup oder Consensus-Extraktion. +4. **Paywall-Problem.** FAZ, NYT, Spiegel+ etc. liefern nur Teaser. Kein Bypass (wäre rechtlich auch problematisch). +5. **Kein PDF/Paper-Access.** arXiv-Links werden gelistet, aber das PDF wird nicht geladen/extrahiert. +6. **Keine Wissensgraph-Anbindung.** Entities (Personen, Firmen, Orte) werden nicht extrahiert oder verknüpft. + +--- + +## 4. Marktübersicht: Alternativen & Ergänzungen + +### 4.1 Search APIs (ersetzen SearXNG) + +| API | Stärken | Schwächen | Preis (Stand 2026) | +|---|---|---|---| +| **Brave Search API** | Unabhängiger Index (kein Google-Relay), gute Privacy-Story, gutes Preis/Leistung | Kleinere Coverage als Google | $3–5 / 1k Queries | +| **Tavily** | Explizit für LLM-Agents gebaut, liefert extrahierten Content statt nur Links, eingebaute Answer-Synthese | Black-Box-Ranking | $0.008 / Query (Basic), ab $30/Monat | +| **Exa** (früher Metaphor) | Semantische Suche auf Embedding-Basis, "find me similar to this URL", beste Coverage für akademische/technische Quellen | Nicht optimal für News | $0.001–0.01 / Query | +| **Serper / SerpAPI** | Google-SERP als JSON (inkl. Knowledge Panels, People-Also-Ask, Shopping, Images) | Nur Google-Relay (nicht unabhängig) | $0.30–1 / 1k Queries | +| **You.com API** | Multi-Engine + integriertes LLM | Kleiner | $10–50/Monat | +| **Kagi Search API** | Bester Qualitäts-Index laut Reviews | Teuer, Wartelisten | $25/Monat Basic | +| **Bing Web Search API** | Große Coverage, stabil | Wird 2025 eingestellt (Microsoft deprecation) | — | +| **Google Custom Search** | Offizieller Google-Zugang | 100 Queries/Tag gratis, dann teuer; Custom-Engine-Setup nötig | $5 / 1k ab 101. Query | + +**Empfehlung:** **Tavily** für Agentic-Research (gibt bereits extrahierten Content + Answer), **Brave Search API** für Privacy-freundlichen Default, **Exa** für semantische / Paper-Recherche. + +### 4.2 Extraction & Scraping APIs (ersetzen/ergänzen Readability) + +| API | Stärken | Schwächen | Preis | +|---|---|---|---| +| **Firecrawl** | JS-Rendering via Playwright, LLM-ready Markdown, Crawl-Jobs, Sitemap, Schema-Extraktion | Selbst-hostbar (OSS), oder Cloud | $16–99/Monat, oder OSS gratis | +| **Jina Reader** (`r.jina.ai`) | Free-Tier großzügig, `https://r.jina.ai/` gibt Markdown | Rate-Limits ohne Key | Free + $0.02/1M tokens | +| **Diffbot** | KI-basierte Extraktion, strukturierte Entities, Knowledge-Graph | Teuer | $300+/Monat | +| **ScrapingBee / ScraperAPI** | Proxy + JS-Render | Generisch, nicht LLM-optimiert | $49–299/Monat | +| **Crawl4AI** (OSS) | Playwright + LLM-friendly Markdown, lokal self-hostbar, Python | Selbst betreiben | Free | +| **Trafilatura** (OSS Python lib) | Best-in-class Text-Extraktion (besser als Readability laut Benchmarks) | Nur Library, kein Service | Free | + +**Empfehlung für Mana:** **Jina Reader** als drop-in Replacement für Readability-Timeouts (1 Zeile HTTP-Call, extrem robust). Oder **Firecrawl self-hosted** für volle Kontrolle + JS-Support. + +### 4.3 All-in-One Research Agents (ersetzen die gesamte Pipeline) + +| Service | Was es tut | Preis | Hinweis | +|---|---|---|---| +| **Perplexity Sonar API** | Endpoint: "frage eine Frage, bekomm Antwort + Zitate". Multi-Step-Research eingebaut. `sonar-large-online`, `sonar-small-online`, `sonar-pro`, `sonar-reasoning` | $1–5 / 1M tokens + $5 / 1k searches | Beste "plug&play" Research-API am Markt | +| **OpenAI Deep Research API** | Async Jobs die 5–30 Min laufen, autonome Agentic-Research, Report als Ergebnis | $10+ / Task | Premium, nicht Echtzeit | +| **Gemini Deep Research** | Ähnlich OpenAI DR, in Gemini Advanced / Vertex AI | In Abos enthalten | Nur über Gemini-API | +| **Claude mit `web_search` Tool** | Claude-API hat server-seitiges Web-Search Tool seit 2025 | $10 / 1k searches + Token-Kosten | Integriert sich nahtlos in existierende Agents | +| **You.com Research API** | Ähnlich Perplexity | $ pro Query | Kleiner Anbieter | + +**Empfehlung:** **Perplexity Sonar** ist der direkteste Ersatz für das, was Mana heute tut — mit deutlich höherer Qualität. **Claude `web_search`** ist die natürlichste Integration, wenn Claude sowieso schon als LLM genutzt wird (aktuell nicht der Fall in Mana, siehe §2.4). + +### 4.4 Open-Source Research-Frameworks + +| Framework | Was es ist | Eignung für Mana | +|---|---|---| +| **Perplexica** | Self-hosted Perplexity-Clone (SearXNG + LLM + UI) | **Sehr hohe** Übereinstimmung mit Mana-Stack; könnte als Vorbild für eine verbesserte Research-UX dienen | +| **GPT Researcher** | LangChain-basiert, führt autonome Multi-Step-Research durch | Passt nicht direkt zu Mana's Tool-Architektur, aber Konzepte übertragbar | +| **SearXNG + LiteLLM + Self-Extract** | Was Mana im Wesentlichen schon hat | — | +| **llm.datasette.io + mwmbl** | Extrem minimalistisch, OSS-Index | Zu klein für Produktivnutzung | +| **Open WebUI + Tool-Servers** | UI-Layer für LLMs mit Tool-Ecosystem | Orthogonal | +| **crawl4ai + rag-stack** | Python-Pipelines für Deep-Research | Nur als inspirativer Input | + +--- + +## 5. Vergleichs-Matrix: Mana heute vs. Alternativen + +| Dimension | Mana (heute) | + Tavily | + Perplexity Sonar | Claude `web_search` | OpenAI DR | +|---|---|---|---|---|---| +| Coverage | 2/5 | 4/5 | 5/5 | 5/5 | 5/5 | +| Semantisches Ranking | 1/5 | 4/5 | 5/5 | 5/5 | 5/5 | +| Multi-Hop | 1/5 | 2/5 (mit Agent-Loop) | 4/5 | 5/5 | 5/5 | +| Synthese & Zitate | 2/5 | 4/5 | 5/5 | 5/5 | 5/5 | +| Latenz | 2/5 (3–15s) | 4/5 (<2s) | 4/5 (2–8s) | 3/5 (5–30s) | 1/5 (5–30 min) | +| Datenschutz | 5/5 | 3/5 | 2/5 | 3/5 | 2/5 | +| Kosten | 5/5 | 4/5 | 3/5 | 3/5 | 2/5 | +| Self-Hosting möglich | 5/5 | 0/5 | 0/5 | 0/5 | 0/5 | +| Produktionsreife | 3/5 | 5/5 | 5/5 | 5/5 | 5/5 | + +--- + +## 6. Empfehlungen — vom Minimum zum Maximum + +### 6.1 Quick-Wins ohne externe APIs (Null zusätzliche Kosten) + +1. **BM25-Ranking** statt Keyword-Frequency. 1 Tag Arbeit, 20–30 % Qualitätssprung. + - File: `apps/api/src/modules/news-research/routes.ts:160` (`scoreAndRank`) +2. **Query-Expansion via lokalem LLM.** Vor dem Search-Call: `gemma3:4b` generiert 2–3 Varianten der Query → mehr Coverage. ~0.5 s Latenz extra. +3. **Embedding-Dedup.** Vor dem Ranking: MiniLM-Embeddings (via `@mana/local-llm` oder mana-llm) für Cluster-Dedup gleicher News. Halbiert Redundanz. +4. **PDF-Extraktion** für arXiv & Paper-Links. `pdf-parse` npm-lib, ~2 h Arbeit. +5. **SearXNG hardenen.** `settings.yml` reviewen, Engines auf stabile (Brave, DDG, Qwant) fokussieren, Google deprioritisieren. +6. **Playwright-Fallback** in `mana-crawler` für JS-heavy Sites (ist da, aber nicht im Research-Pfad). Nutzen, wenn Readability leeren Text zurückgibt. + +**Aufwand gesamt:** ~3–5 Tage, Qualitätssprung von 2.8/5 auf ~3.5/5. + +### 6.2 Hybrid-Modell (empfohlen für Release) + +**Architektur:** `mana-search` behält SearXNG als Default, aber akzeptiert einen optionalen Provider-Switch: + +```typescript +// apps/api/src/lib/search.ts +async function webSearch(opts) { + const provider = opts.provider ?? env.DEFAULT_SEARCH_PROVIDER; // 'searxng' | 'brave' | 'tavily' + switch (provider) { + case 'tavily': return tavilySearch(opts); // agentic, gibt Content + Answer + case 'brave': return braveSearch(opts); // privacy-freundlich, stabil + case 'searxng': return searxngSearch(opts); // self-hosted, free + } +} +``` + +**Tier-Mapping** (nutzt existierendes `requiredTier`-System): +- `guest` / `public` → SearXNG (self-hosted, free) +- `beta` / `alpha` / `founder` → Tavily/Brave (Premium-Recherche) + +**Konkret:** +- Neues `research_news_deep` Tool mit `auto`-Policy für Premium-Tiers +- SearXNG bleibt für alles andere +- Env-Var: `TAVILY_API_KEY`, `BRAVE_SEARCH_API_KEY` (beide optional) + +**Kosten:** ~$30–100/Monat für moderate Nutzung (geschätzt 5k–20k Queries). + +**Qualitätssprung:** 2.8/5 → 4.2/5. + +### 6.3 Max-Quality-Setup (wenn Tier-Premium-Feature) + +Für zahlende Nutzer oder besonders wichtige Missionen: + +1. **Perplexity Sonar API** als Default-Agent (`sonar-pro` für Standard, `sonar-reasoning` für Deep-Dives). +2. **Jina Reader** als Extraction-Fallback für alles, was Readability nicht schafft. +3. **Exa** für akademische/technische Queries (detektiert über Query-Klassifikation). +4. **Langchain-style Multi-Step-Agent** in `mana-ai`: Query → initial search → identify gaps → follow-up searches → synthesis. Mit Token-Budget (bereits vorhanden). + +**Kosten:** ~$200–500/Monat für intensive Nutzung. + +**Qualitätssprung:** 4.2/5 → 4.7/5 (auf Augenhöhe mit Perplexity-UI). + +### 6.4 Was NICHT gemacht werden sollte + +- **Nicht** OpenAI Deep Research als Default einbauen — zu langsam (5–30 min), zu teuer, zu wenig Kontrolle. +- **Nicht** SearXNG komplett rauswerfen — bleibt der beste Free-Tier-Default und wichtig für Privacy-Positioning. +- **Nicht** `mana-crawler` zum News-Fetcher umfunktionieren — er ist für Tiefen-Scans gebaut, falsches Tool. +- **Nicht** Paywall-Umgehung einbauen — rechtlich riskant, schadet dem Ruf. + +--- + +## 7. Datenschutz & "Self-Hosting"-Positioning + +Mana positioniert sich als **Self-Hosted & Independent**. Eine Hybrid-Lösung ist damit vereinbar, wenn: + +1. Default-Verhalten = 100 % self-hosted (SearXNG). +2. Externe APIs sind **opt-in** pro User (Settings → "Premium-Recherche aktivieren"). +3. API-Keys sind **pro User** oder **pro Tier** — nicht geshared. +4. Transparente Anzeige im UI: *"Diese Recherche wurde via Tavily durchgeführt (externer Dienst)"*. +5. Dokumentation in `docs/TECH_STACK_INDEPENDENCE.md` entsprechend erweitern. + +Alternativ: **Nur Brave Search API** (unabhängiger Index, Privacy-Fokus) als einzige externe Option — behält die "unabhängig vom Big-Tech-Stack"-Story. + +--- + +## 8. Nächste Schritte (konkret) + +**Phase 1 — Quick-Wins** (1 Woche) +- [ ] BM25 in `apps/api/src/modules/news-research/routes.ts:160` +- [ ] Query-Expansion via `gemma3:4b` im `research_news`-Pfad +- [ ] SearXNG-`settings.yml` reviewen + dokumentieren +- [ ] Playwright-Fallback in Extraction-Chain (nur wenn Readability leer) + +**Phase 2 — Hybrid-Provider** (1–2 Wochen) +- [ ] `webSearch()` Abstraktion mit Provider-Switch in `apps/api/src/lib/search.ts` +- [ ] Brave Search API als Fallback-Provider +- [ ] Tier-Gating in `packages/shared-branding/src/mana-apps.ts` +- [ ] UI: Settings → "Premium-Recherche" + +**Phase 3 — Agentic Research** (3–4 Wochen) +- [ ] Multi-Step-Loop in `mana-ai` mit Follow-Up-Queries +- [ ] Embedding-Dedup + semantisches Clustering +- [ ] Perplexity Sonar als optionaler `research_deep` Tool +- [ ] Zitations-UI im Frontend (Inline-Footnotes in AI-Antworten) + +--- + +## 9. Referenzen im Code + +| Funktion | Datei | Zeile | +|---|---|---| +| Web-Search Client | `apps/api/src/lib/search.ts` | `webSearch()` | +| News-Research Routes | `apps/api/src/modules/news-research/routes.ts` | 1–220 | +| SearXNG Integration | `services/mana-search/internal/search/searxng.go` | — | +| Readability Extract | `services/mana-search/internal/extract/extractor.go` | — | +| RSS-Discovery | `packages/shared-rss/src/discover.ts` | 1–129 | +| Research-Tool (Frontend) | `apps/mana/apps/web/src/lib/modules/news-research/tools.ts` | 48–102 | +| Pre-Planning Research Step | `services/mana-ai/src/cron/tick.ts` | 282–297 | +| LLM Router | `services/mana-llm/src/providers/router.py` | — | +| News Ingester | `services/news-ingester/src/sources.ts` | — | + +--- + +**Zusammenfassung in einem Satz:** Mana hat eine solide, self-hosted RSS+SearXNG+Readability-Pipeline, die bei ~60 % der Use-Cases ausreicht — für echten agentischen Research-Qualitätsanspruch braucht es entweder Phase-1-Optimierungen (BM25, Query-Expansion, Embedding-Dedup) oder eine optionale Brücke zu Brave/Tavily/Perplexity Sonar als Premium-Feature. diff --git a/packages/shared-research/package.json b/packages/shared-research/package.json new file mode 100644 index 000000000..ce4cf5cdb --- /dev/null +++ b/packages/shared-research/package.json @@ -0,0 +1,27 @@ +{ + "name": "@mana/shared-research", + "version": "0.1.0", + "private": true, + "sideEffects": false, + "description": "Shared types and provider interfaces for the Mana research service.", + "main": "./src/index.ts", + "types": "./src/index.ts", + "exports": { + ".": "./src/index.ts", + "./types": "./src/types.ts", + "./providers": "./src/providers.ts", + "./options": "./src/options.ts", + "./ids": "./src/ids.ts" + }, + "scripts": { + "type-check": "tsc --noEmit", + "clean": "rm -rf dist" + }, + "dependencies": { + "zod": "^3.24.0" + }, + "devDependencies": { + "@types/node": "^24.10.1", + "typescript": "^5.9.3" + } +} diff --git a/packages/shared-research/src/ids.ts b/packages/shared-research/src/ids.ts new file mode 100644 index 000000000..45cc1fbf2 --- /dev/null +++ b/packages/shared-research/src/ids.ts @@ -0,0 +1,38 @@ +export const SEARCH_PROVIDER_IDS = [ + 'searxng', + 'brave', + 'tavily', + 'exa', + 'serper', + 'duckduckgo', +] as const; + +export const EXTRACT_PROVIDER_IDS = [ + 'readability', + 'jina-reader', + 'firecrawl', + 'scrapingbee', +] as const; + +export const AGENT_PROVIDER_IDS = [ + 'perplexity-sonar', + 'claude-web-search', + 'openai-responses', + 'gemini-grounding', + 'openai-deep-research', +] as const; + +export type SearchProviderId = (typeof SEARCH_PROVIDER_IDS)[number]; +export type ExtractProviderId = (typeof EXTRACT_PROVIDER_IDS)[number]; +export type AgentProviderId = (typeof AGENT_PROVIDER_IDS)[number]; +export type ProviderId = SearchProviderId | ExtractProviderId | AgentProviderId; + +export type ProviderCategory = 'search' | 'extract' | 'agent'; + +export type BillingMode = 'server-key' | 'byo-key' | 'free'; + +export function providerCategory(id: ProviderId): ProviderCategory { + if ((SEARCH_PROVIDER_IDS as readonly string[]).includes(id)) return 'search'; + if ((EXTRACT_PROVIDER_IDS as readonly string[]).includes(id)) return 'extract'; + return 'agent'; +} diff --git a/packages/shared-research/src/index.ts b/packages/shared-research/src/index.ts new file mode 100644 index 000000000..1a7e879ba --- /dev/null +++ b/packages/shared-research/src/index.ts @@ -0,0 +1,38 @@ +export type { + ProviderId, + SearchProviderId, + ExtractProviderId, + AgentProviderId, + ProviderCategory, + BillingMode, +} from './ids'; +export { + SEARCH_PROVIDER_IDS, + EXTRACT_PROVIDER_IDS, + AGENT_PROVIDER_IDS, + providerCategory, +} from './ids'; + +export type { + ProviderMeta, + SearchHit, + ExtractedContent, + Citation, + AgentAnswer, + SearchResponse, + ExtractResponse, + AgentResponse, + CompareResponse, +} from './types'; +export { searchHitSchema, citationSchema } from './types'; + +export type { SearchOptions, ExtractOptions, AgentOptions } from './options'; +export { searchOptionsSchema, extractOptionsSchema, agentOptionsSchema } from './options'; + +export type { + ProviderCapabilities, + ProviderCallContext, + SearchProvider, + ExtractProvider, + ResearchAgent, +} from './providers'; diff --git a/packages/shared-research/src/options.ts b/packages/shared-research/src/options.ts new file mode 100644 index 000000000..41e63f373 --- /dev/null +++ b/packages/shared-research/src/options.ts @@ -0,0 +1,27 @@ +import { z } from 'zod'; + +export const searchOptionsSchema = z.object({ + limit: z.number().int().min(1).max(50).optional(), + language: z.string().optional(), + categories: z.array(z.enum(['general', 'news', 'science', 'it'])).optional(), + timeRange: z.enum(['day', 'week', 'month', 'year']).optional(), + safeSearch: z.number().int().min(0).max(2).optional(), +}); + +export const extractOptionsSchema = z.object({ + maxLength: z.number().int().positive().optional(), + includeHtml: z.boolean().optional(), + includeMarkdown: z.boolean().optional(), + timeoutMs: z.number().int().positive().optional(), +}); + +export const agentOptionsSchema = z.object({ + model: z.string().optional(), + temperature: z.number().min(0).max(2).optional(), + maxTokens: z.number().int().positive().optional(), + systemPrompt: z.string().optional(), +}); + +export type SearchOptions = z.infer; +export type ExtractOptions = z.infer; +export type AgentOptions = z.infer; diff --git a/packages/shared-research/src/providers.ts b/packages/shared-research/src/providers.ts new file mode 100644 index 000000000..9fb78a927 --- /dev/null +++ b/packages/shared-research/src/providers.ts @@ -0,0 +1,61 @@ +import type { SearchProviderId, ExtractProviderId, AgentProviderId } from './ids'; +import type { AgentOptions, ExtractOptions, SearchOptions } from './options'; +import type { AgentResponse, ExtractResponse, SearchResponse } from './types'; + +export interface ProviderCapabilities { + webSearch?: boolean; + newsSearch?: boolean; + scholarSearch?: boolean; + semanticSearch?: boolean; + contentInResults?: boolean; + jsRendering?: boolean; + pdfSupport?: boolean; + markdownOutput?: boolean; + multiStep?: boolean; + async?: boolean; + withCitations?: boolean; +} + +export interface ProviderCallContext { + apiKey: string | null; + userId?: string; + signal?: AbortSignal; +} + +export interface SearchProvider { + id: SearchProviderId; + capabilities: ProviderCapabilities; + requiresApiKey: boolean; + search( + query: string, + options: SearchOptions, + ctx: ProviderCallContext + ): Promise & { rawLatencyMs: number }>; +} + +export interface ExtractProvider { + id: ExtractProviderId; + capabilities: ProviderCapabilities; + requiresApiKey: boolean; + extract( + url: string, + options: ExtractOptions, + ctx: ProviderCallContext + ): Promise & { rawLatencyMs: number }>; +} + +export interface ResearchAgent { + id: AgentProviderId; + capabilities: ProviderCapabilities; + requiresApiKey: boolean; + research( + query: string, + options: AgentOptions, + ctx: ProviderCallContext + ): Promise< + Omit & { + rawLatencyMs: number; + tokenUsage?: { input: number; output: number }; + } + >; +} diff --git a/packages/shared-research/src/types.ts b/packages/shared-research/src/types.ts new file mode 100644 index 000000000..4ead32756 --- /dev/null +++ b/packages/shared-research/src/types.ts @@ -0,0 +1,93 @@ +import { z } from 'zod'; +import type { BillingMode, ProviderCategory, ProviderId } from './ids'; + +export interface ProviderMeta { + provider: ProviderId; + category: ProviderCategory; + latencyMs: number; + costCredits: number; + cacheHit: boolean; + billingMode: BillingMode; + errorCode?: string; +} + +export interface SearchHit { + url: string; + title: string; + snippet: string; + publishedAt?: string; + author?: string; + score?: number; + content?: string; + providerRaw?: unknown; +} + +export interface ExtractedContent { + url: string; + title: string; + content: string; + excerpt?: string; + author?: string; + siteName?: string; + publishedAt?: string; + wordCount: number; + providerRaw?: unknown; +} + +export interface Citation { + url: string; + title: string; + snippet?: string; +} + +export interface AgentAnswer { + query: string; + answer: string; + citations: Citation[]; + followUpQueries?: string[]; + tokenUsage?: { input: number; output: number }; + providerRaw?: unknown; +} + +export interface SearchResponse { + results: SearchHit[]; + meta: ProviderMeta; +} + +export interface ExtractResponse { + content: ExtractedContent; + meta: ProviderMeta; +} + +export interface AgentResponse { + answer: AgentAnswer; + meta: ProviderMeta; +} + +export interface CompareResponse { + runId: string; + query: string; + results: Array<{ + provider: ProviderId; + success: boolean; + data?: T; + meta: ProviderMeta; + }>; +} + +export const searchHitSchema = z.object({ + url: z.string().url(), + title: z.string(), + snippet: z.string(), + publishedAt: z.string().optional(), + author: z.string().optional(), + score: z.number().optional(), + content: z.string().optional(), + providerRaw: z.unknown().optional(), +}); + +export const citationSchema = z.object({ + url: z.string().url(), + title: z.string(), + snippet: z.string().optional(), +}); diff --git a/packages/shared-research/tsconfig.json b/packages/shared-research/tsconfig.json new file mode 100644 index 000000000..0e5738959 --- /dev/null +++ b/packages/shared-research/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "ESNext", + "module": "ESNext", + "moduleResolution": "bundler", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "declaration": true, + "noEmit": true, + "types": ["node"] + }, + "include": ["src/**/*.ts"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fe36e190d..c128a0fcf 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -193,7 +193,7 @@ importers: version: link:../../../../packages/shared-landing-ui astro: specifier: ^5.16.0 - version: 5.18.1(@azure/storage-blob@12.31.0)(@types/node@24.12.2)(ioredis@5.10.1)(jiti@1.21.7)(lightningcss@1.32.0)(rollup@4.60.1)(terser@5.46.1)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.3) + version: 5.18.1(@azure/storage-blob@12.31.0)(@types/node@24.12.2)(ioredis@5.10.1)(jiti@2.6.1)(lightningcss@1.32.0)(rollup@4.60.1)(terser@5.46.1)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.3) astro-icon: specifier: ^1.1.5 version: 1.1.5 @@ -203,7 +203,7 @@ importers: devDependencies: '@astrojs/tailwind': specifier: ^6.0.0 - version: 6.0.2(astro@5.18.1(@azure/storage-blob@12.31.0)(@types/node@24.12.2)(ioredis@5.10.1)(jiti@1.21.7)(lightningcss@1.32.0)(rollup@4.60.1)(terser@5.46.1)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.3))(tailwindcss@3.4.19(tsx@4.21.0)(yaml@2.8.3)) + version: 6.0.2(astro@5.18.1(@azure/storage-blob@12.31.0)(@types/node@24.12.2)(ioredis@5.10.1)(jiti@2.6.1)(lightningcss@1.32.0)(rollup@4.60.1)(terser@5.46.1)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.3))(tailwindcss@3.4.19(tsx@4.21.0)(yaml@2.8.3)) '@tailwindcss/typography': specifier: ^0.5.16 version: 0.5.19(tailwindcss@3.4.19(tsx@4.21.0)(yaml@2.8.3)) @@ -484,19 +484,19 @@ importers: version: 19.1.17 '@typescript-eslint/eslint-plugin': specifier: ^7.7.0 - version: 7.18.0(@typescript-eslint/parser@7.18.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.3.3))(eslint@9.39.4(jiti@2.6.1))(typescript@5.3.3) + version: 7.18.0(@typescript-eslint/parser@7.18.0(eslint@9.39.4(jiti@1.21.7))(typescript@5.3.3))(eslint@9.39.4(jiti@1.21.7))(typescript@5.3.3) '@typescript-eslint/parser': specifier: ^7.7.0 - version: 7.18.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.3.3) + version: 7.18.0(eslint@9.39.4(jiti@1.21.7))(typescript@5.3.3) dotenv: specifier: ^16.4.7 version: 16.6.1 eslint: specifier: ^9.39.1 - version: 9.39.4(jiti@2.6.1) + version: 9.39.4(jiti@1.21.7) eslint-config-universe: specifier: ^12.0.1 - version: 12.1.0(@types/eslint@9.6.1)(eslint@9.39.4(jiti@2.6.1))(prettier@3.8.1)(typescript@5.3.3) + version: 12.1.0(@types/eslint@9.6.1)(eslint@9.39.4(jiti@1.21.7))(prettier@3.8.1)(typescript@5.3.3) prettier: specifier: ^3.2.5 version: 3.8.1 @@ -2990,6 +2990,19 @@ importers: specifier: ^7.0.0 version: 7.4.0(@types/babel__core@7.20.5) + packages/shared-research: + dependencies: + zod: + specifier: ^3.24.0 + version: 3.25.76 + devDependencies: + '@types/node': + specifier: ^24.10.1 + version: 24.12.2 + typescript: + specifier: ^5.9.3 + version: 5.9.3 + packages/shared-rss: dependencies: '@mozilla/readability': @@ -3616,6 +3629,40 @@ importers: services/mana-notify: {} + services/mana-research: + dependencies: + '@mana/shared-hono': + specifier: workspace:* + version: link:../../packages/shared-hono + '@mana/shared-research': + specifier: workspace:* + version: link:../../packages/shared-research + drizzle-orm: + specifier: ^0.38.3 + version: 0.38.4(@opentelemetry/api@1.9.1)(@types/pg@8.6.1)(@types/react@19.2.14)(bun-types@1.3.12)(kysely@0.28.15)(postgres@3.4.9)(react@19.2.0) + hono: + specifier: ^4.7.0 + version: 4.12.12 + ioredis: + specifier: ^5.4.1 + version: 5.10.1 + jose: + specifier: ^6.1.2 + version: 6.2.2 + postgres: + specifier: ^3.4.5 + version: 3.4.9 + zod: + specifier: ^3.24.0 + version: 3.25.76 + devDependencies: + drizzle-kit: + specifier: ^0.30.4 + version: 0.30.6 + typescript: + specifier: ^5.9.3 + version: 5.9.3 + services/mana-search: {} services/mana-subscriptions: @@ -18202,16 +18249,6 @@ snapshots: transitivePeerDependencies: - ts-node - '@astrojs/tailwind@6.0.2(astro@5.18.1(@azure/storage-blob@12.31.0)(@types/node@24.12.2)(ioredis@5.10.1)(jiti@1.21.7)(lightningcss@1.32.0)(rollup@4.60.1)(terser@5.46.1)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.3))(tailwindcss@3.4.19(tsx@4.21.0)(yaml@2.8.3))': - dependencies: - astro: 5.18.1(@azure/storage-blob@12.31.0)(@types/node@24.12.2)(ioredis@5.10.1)(jiti@1.21.7)(lightningcss@1.32.0)(rollup@4.60.1)(terser@5.46.1)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.3) - autoprefixer: 10.4.27(postcss@8.5.8) - postcss: 8.5.8 - postcss-load-config: 4.0.2(postcss@8.5.8) - tailwindcss: 3.4.19(tsx@4.21.0)(yaml@2.8.3) - transitivePeerDependencies: - - ts-node - '@astrojs/tailwind@6.0.2(astro@5.18.1(@azure/storage-blob@12.31.0)(@types/node@24.12.2)(ioredis@5.10.1)(jiti@2.6.1)(lightningcss@1.32.0)(rollup@4.60.1)(terser@5.46.1)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.3))(tailwindcss@3.4.19(tsx@4.21.0)(yaml@2.8.3))': dependencies: astro: 5.18.1(@azure/storage-blob@12.31.0)(@types/node@24.12.2)(ioredis@5.10.1)(jiti@2.6.1)(lightningcss@1.32.0)(rollup@4.60.1)(terser@5.46.1)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.3) @@ -20392,6 +20429,11 @@ snapshots: eslint: 8.57.1 eslint-visitor-keys: 3.4.3 + '@eslint-community/eslint-utils@4.9.1(eslint@9.39.4(jiti@1.21.7))': + dependencies: + eslint: 9.39.4(jiti@1.21.7) + eslint-visitor-keys: 3.4.3 + '@eslint-community/eslint-utils@4.9.1(eslint@9.39.4(jiti@2.6.1))': dependencies: eslint: 9.39.4(jiti@2.6.1) @@ -20535,82 +20577,6 @@ snapshots: - supports-color - utf-8-validate - '@expo/cli@55.0.22(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-constants@55.0.12)(expo-font@55.0.6(expo@55.0.12)(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0))(expo-router@55.0.11)(expo@55.0.12)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.9.3)': - dependencies: - '@expo/code-signing-certificates': 0.0.6 - '@expo/config': 55.0.13(typescript@5.9.3) - '@expo/config-plugins': 55.0.8 - '@expo/devcert': 1.2.1 - '@expo/env': 2.1.1 - '@expo/image-utils': 0.8.12 - '@expo/json-file': 10.0.13 - '@expo/log-box': 55.0.10(@expo/dom-webview@55.0.5)(expo@55.0.12)(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0) - '@expo/metro': 55.0.0 - '@expo/metro-config': 55.0.14(expo@55.0.12)(typescript@5.9.3) - '@expo/osascript': 2.4.2 - '@expo/package-manager': 1.10.4 - '@expo/plist': 0.5.2 - '@expo/prebuild-config': 55.0.13(expo@55.0.12)(typescript@5.9.3) - '@expo/require-utils': 55.0.3(typescript@5.9.3) - '@expo/router-server': 55.0.13(@expo/metro-runtime@55.0.9)(expo-constants@55.0.12)(expo-font@55.0.6(expo@55.0.12)(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0))(expo-router@55.0.11)(expo-server@55.0.7)(expo@55.0.12)(react-dom@19.2.0(react@19.2.0))(react@19.2.0) - '@expo/schema-utils': 55.0.3 - '@expo/spawn-async': 1.7.2 - '@expo/ws-tunnel': 1.0.6 - '@expo/xcpretty': 4.4.1 - '@react-native/dev-middleware': 0.83.4 - accepts: 1.3.8 - arg: 5.0.2 - better-opn: 3.0.2 - bplist-creator: 0.1.0 - bplist-parser: 0.3.2 - chalk: 4.1.2 - ci-info: 3.9.0 - compression: 1.8.1 - connect: 3.7.0 - debug: 4.4.3 - dnssd-advertise: 1.1.4 - expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.9.3) - expo-server: 55.0.7 - fetch-nodeshim: 0.4.10 - getenv: 2.0.0 - glob: 13.0.6 - lan-network: 0.2.1 - multitars: 0.2.4 - node-forge: 1.4.0 - npm-package-arg: 11.0.3 - ora: 3.4.0 - picomatch: 4.0.4 - pretty-format: 29.7.0 - progress: 2.0.3 - prompts: 2.4.2 - resolve-from: 5.0.0 - semver: 7.7.4 - send: 0.19.2 - slugify: 1.6.9 - source-map-support: 0.5.21 - stacktrace-parser: 0.1.11 - structured-headers: 0.4.1 - terminal-link: 2.1.1 - toqr: 0.1.1 - wrap-ansi: 7.0.0 - ws: 8.20.0 - zod: 3.25.76 - optionalDependencies: - expo-router: 55.0.11(pdj77jcnw7u2taeri5ruj5t4mi) - react-native: 0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0) - transitivePeerDependencies: - - '@expo/dom-webview' - - '@expo/metro-runtime' - - bufferutil - - expo-constants - - expo-font - - react - - react-dom - - react-server-dom-webpack - - supports-color - - typescript - - utf-8-validate - '@expo/cli@55.0.22(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-constants@55.0.12)(expo-font@55.0.6)(expo-router@55.0.11)(expo@55.0.12)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.3.3)': dependencies: '@expo/code-signing-certificates': 0.0.6 @@ -20763,6 +20729,82 @@ snapshots: - typescript - utf-8-validate + '@expo/cli@55.0.22(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-constants@55.0.12)(expo-font@55.0.6)(expo-router@55.0.11)(expo@55.0.12)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.9.3)': + dependencies: + '@expo/code-signing-certificates': 0.0.6 + '@expo/config': 55.0.13(typescript@5.9.3) + '@expo/config-plugins': 55.0.8 + '@expo/devcert': 1.2.1 + '@expo/env': 2.1.1 + '@expo/image-utils': 0.8.12 + '@expo/json-file': 10.0.13 + '@expo/log-box': 55.0.10(@expo/dom-webview@55.0.5)(expo@55.0.12)(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0) + '@expo/metro': 55.0.0 + '@expo/metro-config': 55.0.14(expo@55.0.12)(typescript@5.9.3) + '@expo/osascript': 2.4.2 + '@expo/package-manager': 1.10.4 + '@expo/plist': 0.5.2 + '@expo/prebuild-config': 55.0.13(expo@55.0.12)(typescript@5.9.3) + '@expo/require-utils': 55.0.3(typescript@5.9.3) + '@expo/router-server': 55.0.13(@expo/metro-runtime@55.0.9)(expo-constants@55.0.12)(expo-font@55.0.6)(expo-router@55.0.11)(expo-server@55.0.7)(expo@55.0.12)(react-dom@19.2.0(react@19.2.0))(react@19.2.0) + '@expo/schema-utils': 55.0.3 + '@expo/spawn-async': 1.7.2 + '@expo/ws-tunnel': 1.0.6 + '@expo/xcpretty': 4.4.1 + '@react-native/dev-middleware': 0.83.4 + accepts: 1.3.8 + arg: 5.0.2 + better-opn: 3.0.2 + bplist-creator: 0.1.0 + bplist-parser: 0.3.2 + chalk: 4.1.2 + ci-info: 3.9.0 + compression: 1.8.1 + connect: 3.7.0 + debug: 4.4.3 + dnssd-advertise: 1.1.4 + expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.9.3) + expo-server: 55.0.7 + fetch-nodeshim: 0.4.10 + getenv: 2.0.0 + glob: 13.0.6 + lan-network: 0.2.1 + multitars: 0.2.4 + node-forge: 1.4.0 + npm-package-arg: 11.0.3 + ora: 3.4.0 + picomatch: 4.0.4 + pretty-format: 29.7.0 + progress: 2.0.3 + prompts: 2.4.2 + resolve-from: 5.0.0 + semver: 7.7.4 + send: 0.19.2 + slugify: 1.6.9 + source-map-support: 0.5.21 + stacktrace-parser: 0.1.11 + structured-headers: 0.4.1 + terminal-link: 2.1.1 + toqr: 0.1.1 + wrap-ansi: 7.0.0 + ws: 8.20.0 + zod: 3.25.76 + optionalDependencies: + expo-router: 55.0.11(eq6waxwbjtmotercssqwfwfcim) + react-native: 0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0) + transitivePeerDependencies: + - '@expo/dom-webview' + - '@expo/metro-runtime' + - bufferutil + - expo-constants + - expo-font + - react + - react-dom + - react-server-dom-webpack + - supports-color + - typescript + - utf-8-validate + '@expo/cli@55.0.22(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-constants@55.0.12)(expo-font@55.0.6)(expo-router@55.0.11)(expo@55.0.12)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0)(typescript@5.9.3)': dependencies: '@expo/code-signing-certificates': 0.0.6 @@ -20990,7 +21032,7 @@ snapshots: '@expo/dom-webview@55.0.5(expo@55.0.12)(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)': dependencies: - expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.9.3) + expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.3.3) react: 19.2.0 react-native: 0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0) @@ -21103,7 +21145,7 @@ snapshots: dependencies: '@expo/dom-webview': 55.0.5(expo@55.0.12)(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0) anser: 1.4.10 - expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.9.3) + expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.3.3) react: 19.2.0 react-native: 0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0) stacktrace-parser: 0.1.11 @@ -21227,7 +21269,7 @@ snapshots: postcss: 8.4.49 resolve-from: 5.0.0 optionalDependencies: - expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.9.3) + expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0)(typescript@5.9.3) transitivePeerDependencies: - bufferutil - supports-color @@ -21254,7 +21296,7 @@ snapshots: dependencies: '@expo/log-box': 55.0.10(@expo/dom-webview@55.0.5)(expo@55.0.12)(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0) anser: 1.4.10 - expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.9.3) + expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.3.3) pretty-format: 29.7.0 react: 19.2.0 react-native: 0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0) @@ -21406,7 +21448,7 @@ snapshots: '@expo/json-file': 10.0.13 '@react-native/normalize-colors': 0.83.4 debug: 4.4.3 - expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.9.3) + expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0)(typescript@5.9.3) resolve-from: 5.0.0 semver: 7.7.4 xml2js: 0.6.0 @@ -21452,21 +21494,6 @@ snapshots: transitivePeerDependencies: - supports-color - '@expo/router-server@55.0.13(@expo/metro-runtime@55.0.9)(expo-constants@55.0.12)(expo-font@55.0.6(expo@55.0.12)(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0))(expo-router@55.0.11)(expo-server@55.0.7)(expo@55.0.12)(react-dom@19.2.0(react@19.2.0))(react@19.2.0)': - dependencies: - debug: 4.4.3 - expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.9.3) - expo-constants: 55.0.12(expo@55.0.12)(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(typescript@5.9.3) - expo-font: 55.0.6(expo@55.0.12)(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0) - expo-server: 55.0.7 - react: 19.2.0 - optionalDependencies: - '@expo/metro-runtime': 55.0.9(@expo/dom-webview@55.0.5)(expo@55.0.12)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0) - expo-router: 55.0.11(pdj77jcnw7u2taeri5ruj5t4mi) - react-dom: 19.2.0(react@19.2.0) - transitivePeerDependencies: - - supports-color - '@expo/router-server@55.0.13(@expo/metro-runtime@55.0.9)(expo-constants@55.0.12)(expo-font@55.0.6)(expo-router@55.0.11)(expo-server@55.0.7)(expo@55.0.12)(react-dom@19.2.0(react@19.2.0))(react@19.2.0)': dependencies: debug: 4.4.3 @@ -25368,16 +25395,16 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.3.3))(eslint@9.39.4(jiti@2.6.1))(typescript@5.3.3)': + '@typescript-eslint/eslint-plugin@6.21.0(@typescript-eslint/parser@6.21.0(eslint@9.39.4(jiti@1.21.7))(typescript@5.3.3))(eslint@9.39.4(jiti@1.21.7))(typescript@5.3.3)': dependencies: '@eslint-community/regexpp': 4.12.2 - '@typescript-eslint/parser': 6.21.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.3.3) + '@typescript-eslint/parser': 6.21.0(eslint@9.39.4(jiti@1.21.7))(typescript@5.3.3) '@typescript-eslint/scope-manager': 6.21.0 - '@typescript-eslint/type-utils': 6.21.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.3.3) - '@typescript-eslint/utils': 6.21.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.3.3) + '@typescript-eslint/type-utils': 6.21.0(eslint@9.39.4(jiti@1.21.7))(typescript@5.3.3) + '@typescript-eslint/utils': 6.21.0(eslint@9.39.4(jiti@1.21.7))(typescript@5.3.3) '@typescript-eslint/visitor-keys': 6.21.0 debug: 4.4.3 - eslint: 9.39.4(jiti@2.6.1) + eslint: 9.39.4(jiti@1.21.7) graphemer: 1.4.0 ignore: 5.3.2 natural-compare: 1.4.0 @@ -25426,15 +25453,15 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/eslint-plugin@7.18.0(@typescript-eslint/parser@7.18.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.3.3))(eslint@9.39.4(jiti@2.6.1))(typescript@5.3.3)': + '@typescript-eslint/eslint-plugin@7.18.0(@typescript-eslint/parser@7.18.0(eslint@9.39.4(jiti@1.21.7))(typescript@5.3.3))(eslint@9.39.4(jiti@1.21.7))(typescript@5.3.3)': dependencies: '@eslint-community/regexpp': 4.12.2 - '@typescript-eslint/parser': 7.18.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.3.3) + '@typescript-eslint/parser': 7.18.0(eslint@9.39.4(jiti@1.21.7))(typescript@5.3.3) '@typescript-eslint/scope-manager': 7.18.0 - '@typescript-eslint/type-utils': 7.18.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.3.3) - '@typescript-eslint/utils': 7.18.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.3.3) + '@typescript-eslint/type-utils': 7.18.0(eslint@9.39.4(jiti@1.21.7))(typescript@5.3.3) + '@typescript-eslint/utils': 7.18.0(eslint@9.39.4(jiti@1.21.7))(typescript@5.3.3) '@typescript-eslint/visitor-keys': 7.18.0 - eslint: 9.39.4(jiti@2.6.1) + eslint: 9.39.4(jiti@1.21.7) graphemer: 1.4.0 ignore: 5.3.2 natural-compare: 1.4.0 @@ -25507,14 +25534,14 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/parser@6.21.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.3.3)': + '@typescript-eslint/parser@6.21.0(eslint@9.39.4(jiti@1.21.7))(typescript@5.3.3)': dependencies: '@typescript-eslint/scope-manager': 6.21.0 '@typescript-eslint/types': 6.21.0 '@typescript-eslint/typescript-estree': 6.21.0(typescript@5.3.3) '@typescript-eslint/visitor-keys': 6.21.0 debug: 4.4.3 - eslint: 9.39.4(jiti@2.6.1) + eslint: 9.39.4(jiti@1.21.7) optionalDependencies: typescript: 5.3.3 transitivePeerDependencies: @@ -25546,14 +25573,14 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/parser@7.18.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.3.3)': + '@typescript-eslint/parser@7.18.0(eslint@9.39.4(jiti@1.21.7))(typescript@5.3.3)': dependencies: '@typescript-eslint/scope-manager': 7.18.0 '@typescript-eslint/types': 7.18.0 '@typescript-eslint/typescript-estree': 7.18.0(typescript@5.3.3) '@typescript-eslint/visitor-keys': 7.18.0 debug: 4.4.3 - eslint: 9.39.4(jiti@2.6.1) + eslint: 9.39.4(jiti@1.21.7) optionalDependencies: typescript: 5.3.3 transitivePeerDependencies: @@ -25649,12 +25676,12 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/type-utils@6.21.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.3.3)': + '@typescript-eslint/type-utils@6.21.0(eslint@9.39.4(jiti@1.21.7))(typescript@5.3.3)': dependencies: '@typescript-eslint/typescript-estree': 6.21.0(typescript@5.3.3) - '@typescript-eslint/utils': 6.21.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.3.3) + '@typescript-eslint/utils': 6.21.0(eslint@9.39.4(jiti@1.21.7))(typescript@5.3.3) debug: 4.4.3 - eslint: 9.39.4(jiti@2.6.1) + eslint: 9.39.4(jiti@1.21.7) ts-api-utils: 1.4.3(typescript@5.3.3) optionalDependencies: typescript: 5.3.3 @@ -25685,12 +25712,12 @@ snapshots: transitivePeerDependencies: - supports-color - '@typescript-eslint/type-utils@7.18.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.3.3)': + '@typescript-eslint/type-utils@7.18.0(eslint@9.39.4(jiti@1.21.7))(typescript@5.3.3)': dependencies: '@typescript-eslint/typescript-estree': 7.18.0(typescript@5.3.3) - '@typescript-eslint/utils': 7.18.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.3.3) + '@typescript-eslint/utils': 7.18.0(eslint@9.39.4(jiti@1.21.7))(typescript@5.3.3) debug: 4.4.3 - eslint: 9.39.4(jiti@2.6.1) + eslint: 9.39.4(jiti@1.21.7) ts-api-utils: 1.4.3(typescript@5.3.3) optionalDependencies: typescript: 5.3.3 @@ -25843,15 +25870,15 @@ snapshots: - supports-color - typescript - '@typescript-eslint/utils@6.21.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.3.3)': + '@typescript-eslint/utils@6.21.0(eslint@9.39.4(jiti@1.21.7))(typescript@5.3.3)': dependencies: - '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.4(jiti@2.6.1)) + '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.4(jiti@1.21.7)) '@types/json-schema': 7.0.15 '@types/semver': 7.7.1 '@typescript-eslint/scope-manager': 6.21.0 '@typescript-eslint/types': 6.21.0 '@typescript-eslint/typescript-estree': 6.21.0(typescript@5.3.3) - eslint: 9.39.4(jiti@2.6.1) + eslint: 9.39.4(jiti@1.21.7) semver: 7.7.4 transitivePeerDependencies: - supports-color @@ -25882,13 +25909,13 @@ snapshots: - supports-color - typescript - '@typescript-eslint/utils@7.18.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.3.3)': + '@typescript-eslint/utils@7.18.0(eslint@9.39.4(jiti@1.21.7))(typescript@5.3.3)': dependencies: - '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.4(jiti@2.6.1)) + '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.4(jiti@1.21.7)) '@typescript-eslint/scope-manager': 7.18.0 '@typescript-eslint/types': 7.18.0 '@typescript-eslint/typescript-estree': 7.18.0(typescript@5.3.3) - eslint: 9.39.4(jiti@2.6.1) + eslint: 9.39.4(jiti@1.21.7) transitivePeerDependencies: - supports-color - typescript @@ -26780,108 +26807,6 @@ snapshots: - uploadthing - yaml - astro@5.18.1(@azure/storage-blob@12.31.0)(@types/node@24.12.2)(ioredis@5.10.1)(jiti@1.21.7)(lightningcss@1.32.0)(rollup@4.60.1)(terser@5.46.1)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.3): - dependencies: - '@astrojs/compiler': 2.13.1 - '@astrojs/internal-helpers': 0.7.6 - '@astrojs/markdown-remark': 6.3.11 - '@astrojs/telemetry': 3.3.0 - '@capsizecss/unpack': 4.0.0 - '@oslojs/encoding': 1.1.0 - '@rollup/pluginutils': 5.3.0(rollup@4.60.1) - acorn: 8.16.0 - aria-query: 5.3.2 - axobject-query: 4.1.0 - boxen: 8.0.1 - ci-info: 4.4.0 - clsx: 2.1.1 - common-ancestor-path: 1.0.1 - cookie: 1.1.1 - cssesc: 3.0.0 - debug: 4.4.3 - deterministic-object-hash: 2.0.2 - devalue: 5.7.0 - diff: 8.0.4 - dlv: 1.1.3 - dset: 3.1.4 - es-module-lexer: 1.7.0 - esbuild: 0.27.7 - estree-walker: 3.0.3 - flattie: 1.1.1 - fontace: 0.4.1 - github-slugger: 2.0.0 - html-escaper: 3.0.3 - http-cache-semantics: 4.2.0 - import-meta-resolve: 4.2.0 - js-yaml: 4.1.1 - magic-string: 0.30.21 - magicast: 0.5.2 - mrmime: 2.0.1 - neotraverse: 0.6.18 - p-limit: 6.2.0 - p-queue: 8.1.1 - package-manager-detector: 1.6.0 - piccolore: 0.1.3 - picomatch: 4.0.4 - prompts: 2.4.2 - rehype: 13.0.2 - semver: 7.7.4 - shiki: 3.23.0 - smol-toml: 1.6.1 - svgo: 4.0.1 - tinyexec: 1.0.4 - tinyglobby: 0.2.15 - tsconfck: 3.1.6(typescript@5.9.3) - ultrahtml: 1.6.0 - unifont: 0.7.4 - unist-util-visit: 5.1.0 - unstorage: 1.17.5(@azure/storage-blob@12.31.0)(ioredis@5.10.1) - vfile: 6.0.3 - vite: 6.4.2(@types/node@24.12.2)(jiti@1.21.7)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3) - vitefu: 1.1.3(vite@6.4.2(@types/node@24.12.2)(jiti@1.21.7)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3)) - xxhash-wasm: 1.1.0 - yargs-parser: 21.1.1 - yocto-spinner: 0.2.3 - zod: 3.25.76 - zod-to-json-schema: 3.25.2(zod@3.25.76) - zod-to-ts: 1.2.0(typescript@5.9.3)(zod@3.25.76) - optionalDependencies: - sharp: 0.34.5 - transitivePeerDependencies: - - '@azure/app-configuration' - - '@azure/cosmos' - - '@azure/data-tables' - - '@azure/identity' - - '@azure/keyvault-secrets' - - '@azure/storage-blob' - - '@capacitor/preferences' - - '@deno/kv' - - '@netlify/blobs' - - '@planetscale/database' - - '@types/node' - - '@upstash/redis' - - '@vercel/blob' - - '@vercel/functions' - - '@vercel/kv' - - aws4fetch - - db0 - - idb-keyval - - ioredis - - jiti - - less - - lightningcss - - rollup - - sass - - sass-embedded - - stylus - - sugarss - - supports-color - - terser - - tsx - - typescript - - uploadthing - - yaml - astro@5.18.1(@azure/storage-blob@12.31.0)(@types/node@24.12.2)(ioredis@5.10.1)(jiti@2.6.1)(lightningcss@1.32.0)(rollup@4.60.1)(terser@5.46.1)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.3): dependencies: '@astrojs/compiler': 2.13.1 @@ -27191,7 +27116,7 @@ snapshots: resolve-from: 5.0.0 optionalDependencies: '@babel/runtime': 7.29.2 - expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.9.3) + expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.3.3) transitivePeerDependencies: - '@babel/core' - supports-color @@ -28758,6 +28683,10 @@ snapshots: dependencies: eslint: 8.57.1 + eslint-config-prettier@8.10.2(eslint@9.39.4(jiti@1.21.7)): + dependencies: + eslint: 9.39.4(jiti@1.21.7) + eslint-config-prettier@8.10.2(eslint@9.39.4(jiti@2.6.1)): dependencies: eslint: 9.39.4(jiti@2.6.1) @@ -28786,17 +28715,17 @@ snapshots: - supports-color - typescript - eslint-config-universe@12.1.0(@types/eslint@9.6.1)(eslint@9.39.4(jiti@2.6.1))(prettier@3.8.1)(typescript@5.3.3): + eslint-config-universe@12.1.0(@types/eslint@9.6.1)(eslint@9.39.4(jiti@1.21.7))(prettier@3.8.1)(typescript@5.3.3): dependencies: - '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.3.3))(eslint@9.39.4(jiti@2.6.1))(typescript@5.3.3) - '@typescript-eslint/parser': 6.21.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.3.3) - eslint: 9.39.4(jiti@2.6.1) - eslint-config-prettier: 8.10.2(eslint@9.39.4(jiti@2.6.1)) - eslint-plugin-import: 2.32.0(@typescript-eslint/parser@6.21.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.3.3))(eslint@9.39.4(jiti@2.6.1)) - eslint-plugin-node: 11.1.0(eslint@9.39.4(jiti@2.6.1)) - eslint-plugin-prettier: 5.5.5(@types/eslint@9.6.1)(eslint-config-prettier@8.10.2(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1))(prettier@3.8.1) - eslint-plugin-react: 7.37.5(eslint@9.39.4(jiti@2.6.1)) - eslint-plugin-react-hooks: 4.6.2(eslint@9.39.4(jiti@2.6.1)) + '@typescript-eslint/eslint-plugin': 6.21.0(@typescript-eslint/parser@6.21.0(eslint@9.39.4(jiti@1.21.7))(typescript@5.3.3))(eslint@9.39.4(jiti@1.21.7))(typescript@5.3.3) + '@typescript-eslint/parser': 6.21.0(eslint@9.39.4(jiti@1.21.7))(typescript@5.3.3) + eslint: 9.39.4(jiti@1.21.7) + eslint-config-prettier: 8.10.2(eslint@9.39.4(jiti@1.21.7)) + eslint-plugin-import: 2.32.0(@typescript-eslint/parser@6.21.0(eslint@9.39.4(jiti@1.21.7))(typescript@5.3.3))(eslint@9.39.4(jiti@1.21.7)) + eslint-plugin-node: 11.1.0(eslint@9.39.4(jiti@1.21.7)) + eslint-plugin-prettier: 5.5.5(@types/eslint@9.6.1)(eslint-config-prettier@8.10.2(eslint@9.39.4(jiti@1.21.7)))(eslint@9.39.4(jiti@1.21.7))(prettier@3.8.1) + eslint-plugin-react: 7.37.5(eslint@9.39.4(jiti@1.21.7)) + eslint-plugin-react-hooks: 4.6.2(eslint@9.39.4(jiti@1.21.7)) optionalDependencies: prettier: 3.8.1 transitivePeerDependencies: @@ -28903,12 +28832,12 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-module-utils@2.12.1(@typescript-eslint/parser@6.21.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.3.3))(eslint-import-resolver-node@0.3.10)(eslint@9.39.4(jiti@2.6.1)): + eslint-module-utils@2.12.1(@typescript-eslint/parser@6.21.0(eslint@9.39.4(jiti@1.21.7))(typescript@5.3.3))(eslint-import-resolver-node@0.3.10)(eslint@9.39.4(jiti@1.21.7)): dependencies: debug: 3.2.7 optionalDependencies: - '@typescript-eslint/parser': 6.21.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.3.3) - eslint: 9.39.4(jiti@2.6.1) + '@typescript-eslint/parser': 6.21.0(eslint@9.39.4(jiti@1.21.7))(typescript@5.3.3) + eslint: 9.39.4(jiti@1.21.7) eslint-import-resolver-node: 0.3.10 transitivePeerDependencies: - supports-color @@ -28972,6 +28901,12 @@ snapshots: eslint-utils: 2.1.0 regexpp: 3.2.0 + eslint-plugin-es@3.0.1(eslint@9.39.4(jiti@1.21.7)): + dependencies: + eslint: 9.39.4(jiti@1.21.7) + eslint-utils: 2.1.0 + regexpp: 3.2.0 + eslint-plugin-es@3.0.1(eslint@9.39.4(jiti@2.6.1)): dependencies: eslint: 9.39.4(jiti@2.6.1) @@ -29025,7 +28960,7 @@ snapshots: - eslint-import-resolver-webpack - supports-color - eslint-plugin-import@2.32.0(@typescript-eslint/parser@6.21.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.3.3))(eslint@9.39.4(jiti@2.6.1)): + eslint-plugin-import@2.32.0(@typescript-eslint/parser@6.21.0(eslint@9.39.4(jiti@1.21.7))(typescript@5.3.3))(eslint@9.39.4(jiti@1.21.7)): dependencies: '@rtsao/scc': 1.1.0 array-includes: 3.1.9 @@ -29034,9 +28969,9 @@ snapshots: array.prototype.flatmap: 1.3.3 debug: 3.2.7 doctrine: 2.1.0 - eslint: 9.39.4(jiti@2.6.1) + eslint: 9.39.4(jiti@1.21.7) eslint-import-resolver-node: 0.3.10 - eslint-module-utils: 2.12.1(@typescript-eslint/parser@6.21.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.3.3))(eslint-import-resolver-node@0.3.10)(eslint@9.39.4(jiti@2.6.1)) + eslint-module-utils: 2.12.1(@typescript-eslint/parser@6.21.0(eslint@9.39.4(jiti@1.21.7))(typescript@5.3.3))(eslint-import-resolver-node@0.3.10)(eslint@9.39.4(jiti@1.21.7)) hasown: 2.0.2 is-core-module: 2.16.1 is-glob: 4.0.3 @@ -29048,7 +28983,7 @@ snapshots: string.prototype.trimend: 1.0.9 tsconfig-paths: 3.15.0 optionalDependencies: - '@typescript-eslint/parser': 6.21.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.3.3) + '@typescript-eslint/parser': 6.21.0(eslint@9.39.4(jiti@1.21.7))(typescript@5.3.3) transitivePeerDependencies: - eslint-import-resolver-typescript - eslint-import-resolver-webpack @@ -29166,6 +29101,16 @@ snapshots: resolve: 1.22.11 semver: 6.3.1 + eslint-plugin-node@11.1.0(eslint@9.39.4(jiti@1.21.7)): + dependencies: + eslint: 9.39.4(jiti@1.21.7) + eslint-plugin-es: 3.0.1(eslint@9.39.4(jiti@1.21.7)) + eslint-utils: 2.1.0 + ignore: 5.3.2 + minimatch: 3.1.5 + resolve: 1.22.11 + semver: 6.3.1 + eslint-plugin-node@11.1.0(eslint@9.39.4(jiti@2.6.1)): dependencies: eslint: 9.39.4(jiti@2.6.1) @@ -29196,6 +29141,16 @@ snapshots: '@types/eslint': 9.6.1 eslint-config-prettier: 8.10.2(eslint@8.57.1) + eslint-plugin-prettier@5.5.5(@types/eslint@9.6.1)(eslint-config-prettier@8.10.2(eslint@9.39.4(jiti@1.21.7)))(eslint@9.39.4(jiti@1.21.7))(prettier@3.8.1): + dependencies: + eslint: 9.39.4(jiti@1.21.7) + prettier: 3.8.1 + prettier-linter-helpers: 1.0.1 + synckit: 0.11.12 + optionalDependencies: + '@types/eslint': 9.6.1 + eslint-config-prettier: 8.10.2(eslint@9.39.4(jiti@1.21.7)) + eslint-plugin-prettier@5.5.5(@types/eslint@9.6.1)(eslint-config-prettier@8.10.2(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1))(prettier@3.8.1): dependencies: eslint: 9.39.4(jiti@2.6.1) @@ -29220,6 +29175,10 @@ snapshots: dependencies: eslint: 8.57.1 + eslint-plugin-react-hooks@4.6.2(eslint@9.39.4(jiti@1.21.7)): + dependencies: + eslint: 9.39.4(jiti@1.21.7) + eslint-plugin-react-hooks@4.6.2(eslint@9.39.4(jiti@2.6.1)): dependencies: eslint: 9.39.4(jiti@2.6.1) @@ -29250,6 +29209,28 @@ snapshots: string.prototype.matchall: 4.0.12 string.prototype.repeat: 1.0.0 + eslint-plugin-react@7.37.5(eslint@9.39.4(jiti@1.21.7)): + dependencies: + array-includes: 3.1.9 + array.prototype.findlast: 1.2.5 + array.prototype.flatmap: 1.3.3 + array.prototype.tosorted: 1.1.4 + doctrine: 2.1.0 + es-iterator-helpers: 1.3.1 + eslint: 9.39.4(jiti@1.21.7) + estraverse: 5.3.0 + hasown: 2.0.2 + jsx-ast-utils: 3.3.5 + minimatch: 3.1.5 + object.entries: 1.1.9 + object.fromentries: 2.0.8 + object.values: 1.2.1 + prop-types: 15.8.1 + resolve: 2.0.0-next.6 + semver: 6.3.1 + string.prototype.matchall: 4.0.12 + string.prototype.repeat: 1.0.0 + eslint-plugin-react@7.37.5(eslint@9.39.4(jiti@2.6.1)): dependencies: array-includes: 3.1.9 @@ -29366,6 +29347,47 @@ snapshots: transitivePeerDependencies: - supports-color + eslint@9.39.4(jiti@1.21.7): + dependencies: + '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.4(jiti@1.21.7)) + '@eslint-community/regexpp': 4.12.2 + '@eslint/config-array': 0.21.2 + '@eslint/config-helpers': 0.4.2 + '@eslint/core': 0.17.0 + '@eslint/eslintrc': 3.3.5 + '@eslint/js': 9.39.4 + '@eslint/plugin-kit': 0.4.1 + '@humanfs/node': 0.16.7 + '@humanwhocodes/module-importer': 1.0.1 + '@humanwhocodes/retry': 0.4.3 + '@types/estree': 1.0.8 + ajv: 6.14.0 + chalk: 4.1.2 + cross-spawn: 7.0.6 + debug: 4.4.3 + escape-string-regexp: 4.0.0 + eslint-scope: 8.4.0 + eslint-visitor-keys: 4.2.1 + espree: 10.4.0 + esquery: 1.7.0 + esutils: 2.0.3 + fast-deep-equal: 3.1.3 + file-entry-cache: 8.0.0 + find-up: 5.0.0 + glob-parent: 6.0.2 + ignore: 5.3.2 + imurmurhash: 0.1.4 + is-glob: 4.0.3 + json-stable-stringify-without-jsonify: 1.0.1 + lodash.merge: 4.6.2 + minimatch: 3.1.5 + natural-compare: 1.4.0 + optionator: 0.9.4 + optionalDependencies: + jiti: 1.21.7 + transitivePeerDependencies: + - supports-color + eslint@9.39.4(jiti@2.6.1): dependencies: '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.4(jiti@2.6.1)) @@ -29716,7 +29738,7 @@ snapshots: expo-dev-client@6.0.20(expo@55.0.12): dependencies: - expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.9.3) + expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.3.3) expo-dev-launcher: 6.0.20(expo@55.0.12) expo-dev-menu: 7.0.18(expo@55.0.12) expo-dev-menu-interface: 2.0.0(expo@55.0.12) @@ -29738,7 +29760,7 @@ snapshots: expo-dev-launcher@6.0.20(expo@55.0.12): dependencies: ajv: 8.18.0 - expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.9.3) + expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.3.3) expo-dev-menu: 7.0.18(expo@55.0.12) expo-manifests: 1.0.10(expo@55.0.12) transitivePeerDependencies: @@ -29746,7 +29768,7 @@ snapshots: expo-dev-menu-interface@2.0.0(expo@55.0.12): dependencies: - expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.9.3) + expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.3.3) expo-dev-menu-interface@55.0.2(expo@55.0.12): dependencies: @@ -29759,7 +29781,7 @@ snapshots: expo-dev-menu@7.0.18(expo@55.0.12): dependencies: - expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.9.3) + expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.3.3) expo-dev-menu-interface: 2.0.0(expo@55.0.12) expo-device@55.0.13(expo@55.0.12): @@ -29785,7 +29807,7 @@ snapshots: expo-file-system@55.0.15(expo@55.0.12)(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0)): dependencies: - expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.9.3) + expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.3.3) react-native: 0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0) expo-file-system@55.0.15(expo@55.0.12)(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0)): @@ -29809,7 +29831,7 @@ snapshots: expo-font@55.0.6(expo@55.0.12)(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0): dependencies: - expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.9.3) + expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.3.3) fontfaceobserver: 2.3.0 react: 19.2.0 react-native: 0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0) @@ -29830,7 +29852,7 @@ snapshots: expo-glass-effect@55.0.10(expo@55.0.12)(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0): dependencies: - expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.9.3) + expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.3.3) react: 19.2.0 react-native: 0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0) @@ -29846,11 +29868,11 @@ snapshots: expo-image-loader@55.0.0(expo@55.0.12): dependencies: - expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.9.3) + expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0)(typescript@5.9.3) expo-image-picker@55.0.17(expo@55.0.12): dependencies: - expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.9.3) + expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0)(typescript@5.9.3) expo-image-loader: 55.0.0(expo@55.0.12) expo-image@55.0.8(expo@54.0.33)(react-native-web@0.21.2(react-dom@19.2.0(react@19.1.0))(react@19.1.0))(react-native@0.81.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.1.0))(react@19.1.0): @@ -29865,7 +29887,7 @@ snapshots: expo-image@55.0.8(expo@55.0.12)(react-native-web@0.21.2(react-dom@19.2.0(react@19.2.0))(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0): dependencies: - expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.9.3) + expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.3.3) react: 19.2.0 react-native: 0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0) sf-symbols-typescript: 2.2.0 @@ -29892,7 +29914,7 @@ snapshots: expo-keep-awake@55.0.6(expo@55.0.12)(react@19.2.0): dependencies: - expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.9.3) + expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.3.3) react: 19.2.0 expo-linear-gradient@15.0.8(expo@55.0.12)(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0): @@ -29988,7 +30010,7 @@ snapshots: expo-manifests@1.0.10(expo@55.0.12): dependencies: '@expo/config': 12.0.13 - expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.9.3) + expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.3.3) expo-json-utils: 0.15.0 transitivePeerDependencies: - supports-color @@ -30360,7 +30382,7 @@ snapshots: expo-secure-store@55.0.12(expo@55.0.12): dependencies: - expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.9.3) + expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.2.0))(react@19.2.0)(typescript@5.9.3) expo-server@1.0.5: {} @@ -30455,7 +30477,7 @@ snapshots: dependencies: '@react-native/normalize-colors': 0.83.4 debug: 4.4.3 - expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.9.3) + expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.3.3) react-native: 0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0) optionalDependencies: react-native-web: 0.21.2(react-dom@19.2.0(react@19.2.0))(react@19.2.0) @@ -30481,7 +30503,7 @@ snapshots: expo-updates-interface@2.0.0(expo@55.0.12): dependencies: - expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.9.3) + expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.3.3) expo-updates-interface@55.1.5(expo@55.0.12): dependencies: @@ -30511,7 +30533,7 @@ snapshots: expo-web-browser@55.0.13(expo@55.0.12)(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0)): dependencies: - expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.9.3) + expo: 55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.3.3) react-native: 0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0) expo@54.0.33(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-native@0.81.4(@babel/core@7.29.0)(@types/react@19.2.14)(react@19.1.0))(react@19.1.0): @@ -30635,7 +30657,7 @@ snapshots: expo@55.0.12(@babel/core@7.29.0)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-router@55.0.11)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.9.3): dependencies: '@babel/runtime': 7.29.2 - '@expo/cli': 55.0.22(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-constants@55.0.12)(expo-font@55.0.6(expo@55.0.12)(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0))(expo-router@55.0.11)(expo@55.0.12)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.9.3) + '@expo/cli': 55.0.22(@expo/dom-webview@55.0.5)(@expo/metro-runtime@55.0.9)(expo-constants@55.0.12)(expo-font@55.0.6)(expo-router@55.0.11)(expo@55.0.12)(react-dom@19.2.0(react@19.2.0))(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0)(typescript@5.9.3) '@expo/config': 55.0.13(typescript@5.9.3) '@expo/config-plugins': 55.0.8 '@expo/devtools': 55.0.2(react-native@0.83.2(@babel/core@7.29.0)(@types/react@19.1.17)(react@19.2.0))(react@19.2.0) @@ -37578,23 +37600,6 @@ snapshots: tsx: 4.21.0 yaml: 2.8.3 - vite@6.4.2(@types/node@24.12.2)(jiti@1.21.7)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3): - dependencies: - esbuild: 0.25.12 - fdir: 6.5.0(picomatch@4.0.4) - picomatch: 4.0.4 - postcss: 8.5.8 - rollup: 4.60.1 - tinyglobby: 0.2.15 - optionalDependencies: - '@types/node': 24.12.2 - fsevents: 2.3.3 - jiti: 1.21.7 - lightningcss: 1.32.0 - terser: 5.46.1 - tsx: 4.21.0 - yaml: 2.8.3 - vite@6.4.2(@types/node@24.12.2)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3): dependencies: esbuild: 0.25.12 @@ -37620,10 +37625,6 @@ snapshots: optionalDependencies: vite: 6.4.2(@types/node@22.19.17)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3) - vitefu@1.1.3(vite@6.4.2(@types/node@24.12.2)(jiti@1.21.7)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3)): - optionalDependencies: - vite: 6.4.2(@types/node@24.12.2)(jiti@1.21.7)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3) - vitefu@1.1.3(vite@6.4.2(@types/node@24.12.2)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3)): optionalDependencies: vite: 6.4.2(@types/node@24.12.2)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3) diff --git a/scripts/generate-env.mjs b/scripts/generate-env.mjs index 5f4b3e5e5..2c1f03b5c 100644 --- a/scripts/generate-env.mjs +++ b/scripts/generate-env.mjs @@ -75,6 +75,37 @@ const APP_CONFIGS = [ }, }, + // Mana Research Service (Hono + Bun, Port 3068) + { + path: 'services/mana-research/.env', + vars: { + NODE_ENV: () => 'development', + PORT: (env) => env.MANA_RESEARCH_PORT || '3068', + DATABASE_URL: (env) => + env.MANA_RESEARCH_DATABASE_URL || + 'postgresql://mana:devpassword@localhost:5432/mana_platform', + REDIS_URL: (env) => env.REDIS_URL || 'redis://localhost:6379', + MANA_AUTH_URL: (env) => env.MANA_AUTH_URL || 'http://localhost:3001', + MANA_LLM_URL: (env) => env.MANA_LLM_URL || 'http://localhost:3025', + MANA_CREDITS_URL: (env) => env.MANA_CREDITS_URL || 'http://localhost:3061', + MANA_SEARCH_URL: (env) => env.MANA_SEARCH_URL || 'http://localhost:3021', + MANA_SERVICE_KEY: (env) => env.MANA_SERVICE_KEY || 'dev-service-key', + CACHE_TTL_SECONDS: (env) => env.MANA_RESEARCH_CACHE_TTL_SECONDS || '3600', + CORS_ORIGINS: (env) => env.CORS_ORIGINS || 'http://localhost:5173', + BRAVE_API_KEY: (env) => env.BRAVE_API_KEY || '', + TAVILY_API_KEY: (env) => env.TAVILY_API_KEY || '', + EXA_API_KEY: (env) => env.EXA_API_KEY || '', + SERPER_API_KEY: (env) => env.SERPER_API_KEY || '', + JINA_API_KEY: (env) => env.JINA_API_KEY || '', + FIRECRAWL_API_KEY: (env) => env.FIRECRAWL_API_KEY || '', + SCRAPINGBEE_API_KEY: (env) => env.SCRAPINGBEE_API_KEY || '', + PERPLEXITY_API_KEY: (env) => env.PERPLEXITY_API_KEY || '', + ANTHROPIC_API_KEY: (env) => env.ANTHROPIC_API_KEY || '', + OPENAI_API_KEY: (env) => env.OPENAI_API_KEY || '', + GOOGLE_GENAI_API_KEY: (env) => env.GOOGLE_GENAI_API_KEY || '', + }, + }, + // Chat Server (Hono/Bun) { path: 'apps/chat/apps/server/.env', diff --git a/services/mana-research/CLAUDE.md b/services/mana-research/CLAUDE.md new file mode 100644 index 000000000..18a128c47 --- /dev/null +++ b/services/mana-research/CLAUDE.md @@ -0,0 +1,149 @@ +# mana-research + +Web research orchestration service. Bundles 16+ providers (search, extract, agent) behind one interface. Pay-per-use APIs only, integrated with `mana-credits` 2-phase debit. + +**Plan:** [`docs/plans/mana-research-service.md`](../../docs/plans/mana-research-service.md) +**Related analysis:** [`docs/reports/web-research-capabilities.md`](../../docs/reports/web-research-capabilities.md) + +## Tech Stack + +| Layer | Technology | +|-------|------------| +| **Runtime** | Bun | +| **Framework** | Hono | +| **Database** | PostgreSQL + Drizzle ORM (`research.*` schema in `mana_platform`) | +| **Cache** | Redis (ioredis, graceful degradation) | +| **Auth** | JWT via JWKS from mana-auth, plus `X-Service-Key` for service-to-service | + +## Quick Start + +```bash +# From repo root: ensure postgres + redis are up, then run +pnpm --filter @mana/research-service dev + +# Database schema (creates research.* tables) +cd services/mana-research +bun run db:push +bun run db:studio +``` + +## Port: 3068 + +## Phases + +- **Phase 1** ✅ — 4 search providers (`searxng`, `duckduckgo`, `brave`, `tavily`), `/v1/search`, `/v1/search/compare`, `/v1/runs`, `/v1/providers`, `mana-credits` reserve/commit/refund. +- **Phase 2 (current)** ✅ — +2 search providers (`exa`, `serper`), 3 extract providers (`readability`, `jina-reader`, `firecrawl`), `/v1/extract`, `/v1/extract/compare`, query classifier + auto-router, `/v1/providers/health`. +- **Phase 3** — Research agents (`perplexity-sonar`, `claude-web-search`, `openai-responses`, `gemini-grounding`, `openai-deep-research`). mana-ai migration to use this service. +- **Phase 4** — Research Lab UI + Settings for BYO-keys. + +## API Endpoints + +### User-facing (JWT auth) + +| Method | Path | Description | +|---|---|---| +| POST | `/api/v1/search` | Single-provider search, or auto-routed if `provider` omitted. Body: `{ query, provider?, options?, useLlmClassifier? }`. | +| POST | `/api/v1/search/compare` | Fan-out to N providers (max 5), persist eval_run. Body: `{ query, providers[], options? }`. | +| POST | `/api/v1/extract` | Single-provider extract, auto-routed if `provider` omitted. Body: `{ url, provider?, options? }`. | +| POST | `/api/v1/extract/compare` | Fan-out to N extract providers (max 4). Body: `{ url, providers[], options? }`. | +| GET | `/api/v1/runs` | List user's eval runs. Query: `?limit=50&offset=0`. | +| GET | `/api/v1/runs/:id` | Run + all results. | +| POST | `/api/v1/runs/:runId/results/:resultId/rate` | Body: `{ rating: 1-5, notes? }`. | + +### Public + +| Method | Path | Description | +|---|---|---| +| GET | `/health` | Health check. | +| GET | `/metrics` | Prometheus stub (wired up later). | +| GET | `/api/v1/providers` | List registered providers + capabilities + pricing. | +| GET | `/api/v1/providers/health` | Per-provider readiness check (`free` / `ready` / `needs-key`). | + +### Service-to-service (X-Service-Key) + +Reserved for Phase 3 when `mana-ai` migrates to call this service directly. `/api/v1/internal/health` exists as a placeholder. + +## Providers + +### Search (6) + +| Provider | Key | Cost | Notes | +|---|---|---|---| +| `searxng` | — | 0 | Wraps `mana-search` (SearXNG). Self-hosted. | +| `duckduckgo` | — | 0 | Instant Answer API. Rate-limited. | +| `brave` | `BRAVE_API_KEY` | 5 | $5/1k PAYG. Independent index. | +| `tavily` | `TAVILY_API_KEY` | 8 | Agent-optimized, returns content. | +| `exa` | `EXA_API_KEY` | 6 | Semantic/neural, best for papers + semantic similarity. | +| `serper` | `SERPER_API_KEY` | 1 | Google SERP as JSON. $0.30–1/1k. | + +### Extract (3) + +| Provider | Key | Cost | Notes | +|---|---|---|---| +| `readability` | — | 0 | Wraps `mana-search /extract` (go-readability). | +| `jina-reader` | optional `JINA_API_KEY` | 1 | `r.jina.ai`, JS-rendering + PDF, Markdown out. | +| `firecrawl` | `FIRECRAWL_API_KEY` | 10 | Playwright-based, best for JS-heavy sites. Self-hostable. | + +## Auto-routing + +When `provider` is omitted from `POST /v1/search`, the service classifies the query via regex (fast path, ~0ms) and optionally the LLM (`useLlmClassifier: true`), then picks the first available provider from `SEARCH_ROUTE_MAP[type]`: + +- `news` → tavily, brave, serper, searxng, duckduckgo +- `general` → brave, tavily, serper, searxng +- `semantic` → exa, tavily, brave +- `academic` → exa, searxng, brave +- `code` → exa, serper, brave +- `conversational` → tavily, brave, serper + +Extract auto-routing prefers `firecrawl` (best quality) → `jina-reader` → `readability`. + +## Credits Integration + +Server-key mode uses `mana-credits` 2-phase debit: + +``` +reserve → provider call → (commit on success | refund on error) +``` + +BYO-key mode bypasses credits entirely (user brings their own API key, Phase 4 UI). + +Pricing map: `src/lib/pricing.ts`. + +## Database + +Schema `research` in `mana_platform`: + +- `eval_runs` — one per request (`single`/`compare`/`auto` mode). +- `eval_results` — one per provider response. Raw + normalized output, latency, cost, optional user rating. +- `provider_configs` — per-user BYO-key + budget. `userId=null` reserved for server defaults. +- `provider_stats` — rolled-up daily metrics for admin dashboard + auto-router. + +All eval runs are **permanent** by design — this is the comparison engine's point. + +## Environment Variables + +```env +PORT=3068 +DATABASE_URL=postgresql://mana:devpassword@localhost:5432/mana_platform +REDIS_URL=redis://localhost:6379 +MANA_AUTH_URL=http://localhost:3001 +MANA_LLM_URL=http://localhost:3025 +MANA_CREDITS_URL=http://localhost:3061 +MANA_SEARCH_URL=http://localhost:3021 +MANA_SERVICE_KEY=dev-service-key +CACHE_TTL_SECONDS=3600 +CORS_ORIGINS=http://localhost:5173 + +# Provider keys (optional in dev — providers without keys are unavailable) +BRAVE_API_KEY= +TAVILY_API_KEY= +EXA_API_KEY= +SERPER_API_KEY= +JINA_API_KEY= +FIRECRAWL_API_KEY= +SCRAPINGBEE_API_KEY= +PERPLEXITY_API_KEY= +ANTHROPIC_API_KEY= +OPENAI_API_KEY= +GOOGLE_GENAI_API_KEY= +``` diff --git a/services/mana-research/Dockerfile b/services/mana-research/Dockerfile new file mode 100644 index 000000000..9770383bb --- /dev/null +++ b/services/mana-research/Dockerfile @@ -0,0 +1,37 @@ +# Install stage: use node + pnpm to resolve workspace dependencies +FROM node:22-alpine AS installer + +RUN corepack enable && corepack prepare pnpm@9.15.0 --activate + +WORKDIR /app + +COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ +COPY services/mana-research/package.json ./services/mana-research/ +COPY packages/shared-hono ./packages/shared-hono +COPY packages/shared-logger ./packages/shared-logger +COPY packages/shared-types ./packages/shared-types +COPY packages/shared-research ./packages/shared-research + +RUN pnpm install --filter @mana/research-service... --no-frozen-lockfile --ignore-scripts + +# Runtime stage: bun +FROM oven/bun:1 AS production + +WORKDIR /app + +COPY --from=installer /app/node_modules ./node_modules +COPY --from=installer /app/services/mana-research/node_modules ./services/mana-research/node_modules +COPY --from=installer /app/packages ./packages + +COPY services/mana-research/package.json ./services/mana-research/ +COPY services/mana-research/src ./services/mana-research/src +COPY services/mana-research/tsconfig.json services/mana-research/drizzle.config.ts ./services/mana-research/ + +WORKDIR /app/services/mana-research + +EXPOSE 3068 + +HEALTHCHECK --interval=30s --timeout=10s --start-period=10s --retries=3 \ + CMD bun -e "fetch('http://localhost:3068/health').then(r => process.exit(r.ok ? 0 : 1)).catch(() => process.exit(1))" + +CMD ["bun", "run", "src/index.ts"] diff --git a/services/mana-research/drizzle.config.ts b/services/mana-research/drizzle.config.ts new file mode 100644 index 000000000..6fd0a699f --- /dev/null +++ b/services/mana-research/drizzle.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from 'drizzle-kit'; + +export default defineConfig({ + schema: './src/db/schema/*.ts', + out: './drizzle', + dialect: 'postgresql', + dbCredentials: { + url: process.env.DATABASE_URL || 'postgresql://mana:devpassword@localhost:5432/mana_platform', + }, + schemaFilter: ['research'], +}); diff --git a/services/mana-research/package.json b/services/mana-research/package.json new file mode 100644 index 000000000..8d5515a6f --- /dev/null +++ b/services/mana-research/package.json @@ -0,0 +1,28 @@ +{ + "name": "@mana/research-service", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "dev": "bun run --watch src/index.ts", + "start": "bun run src/index.ts", + "db:push": "drizzle-kit push", + "db:generate": "drizzle-kit generate", + "db:studio": "drizzle-kit studio", + "type-check": "tsc --noEmit" + }, + "dependencies": { + "@mana/shared-hono": "workspace:*", + "@mana/shared-research": "workspace:*", + "hono": "^4.7.0", + "drizzle-orm": "^0.38.3", + "postgres": "^3.4.5", + "ioredis": "^5.4.1", + "jose": "^6.1.2", + "zod": "^3.24.0" + }, + "devDependencies": { + "drizzle-kit": "^0.30.4", + "typescript": "^5.9.3" + } +} diff --git a/services/mana-research/src/clients/mana-credits.ts b/services/mana-research/src/clients/mana-credits.ts new file mode 100644 index 000000000..802009a90 --- /dev/null +++ b/services/mana-research/src/clients/mana-credits.ts @@ -0,0 +1,68 @@ +/** + * HTTP client for mana-credits. Uses the internal Reserve/Commit/Refund endpoints + * (added in this phase — see services/mana-credits/src/routes/internal.ts). + */ + +export interface CreditsClientConfig { + baseUrl: string; + serviceKey: string; +} + +export interface ReservationResponse { + reservationId: string; + balance: number; +} + +export class CreditsClient { + constructor(private config: CreditsClientConfig) {} + + private headers() { + return { + 'Content-Type': 'application/json', + 'X-Service-Key': this.config.serviceKey, + 'X-App-Id': 'mana-research', + }; + } + + async getBalance(userId: string): Promise<{ balance: number }> { + const res = await fetch( + `${this.config.baseUrl}/api/v1/internal/credits/balance/${encodeURIComponent(userId)}`, + { headers: this.headers() } + ); + if (!res.ok) throw new Error(`credits.balance failed: ${res.status}`); + return res.json() as Promise<{ balance: number }>; + } + + async reserve(userId: string, amount: number, reason: string): Promise { + const res = await fetch(`${this.config.baseUrl}/api/v1/internal/credits/reserve`, { + method: 'POST', + headers: this.headers(), + body: JSON.stringify({ userId, amount, reason }), + }); + if (!res.ok) { + const body = await res.text(); + throw new Error(`credits.reserve failed: ${res.status} ${body}`); + } + return res.json() as Promise; + } + + async commit(reservationId: string, description?: string): Promise<{ success: boolean }> { + const res = await fetch(`${this.config.baseUrl}/api/v1/internal/credits/commit`, { + method: 'POST', + headers: this.headers(), + body: JSON.stringify({ reservationId, description }), + }); + if (!res.ok) throw new Error(`credits.commit failed: ${res.status}`); + return res.json() as Promise<{ success: boolean }>; + } + + async refund(reservationId: string): Promise<{ success: boolean }> { + const res = await fetch(`${this.config.baseUrl}/api/v1/internal/credits/refund-reservation`, { + method: 'POST', + headers: this.headers(), + body: JSON.stringify({ reservationId }), + }); + if (!res.ok) throw new Error(`credits.refund failed: ${res.status}`); + return res.json() as Promise<{ success: boolean }>; + } +} diff --git a/services/mana-research/src/clients/mana-llm.ts b/services/mana-research/src/clients/mana-llm.ts new file mode 100644 index 000000000..d3252b852 --- /dev/null +++ b/services/mana-research/src/clients/mana-llm.ts @@ -0,0 +1,46 @@ +/** + * HTTP client for mana-llm — OpenAI-compatible chat completions endpoint. + * Used by the query classifier (short prompt → JSON tag). + */ + +export interface ChatMessage { + role: 'system' | 'user' | 'assistant'; + content: string; +} + +export class ManaLlmClient { + constructor(private baseUrl: string) {} + + async chat( + messages: ChatMessage[], + opts: { model?: string; maxTokens?: number; temperature?: number; signal?: AbortSignal } = {} + ): Promise<{ content: string; tokenUsage?: { input: number; output: number } }> { + const res = await fetch(`${this.baseUrl}/v1/chat/completions`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + model: opts.model ?? 'ollama/gemma3:4b', + messages, + max_tokens: opts.maxTokens ?? 256, + temperature: opts.temperature ?? 0.2, + }), + signal: opts.signal, + }); + if (!res.ok) { + throw new Error(`mana-llm returned ${res.status}`); + } + type ChatResponse = { + choices?: Array<{ message?: { content?: string } }>; + usage?: { prompt_tokens?: number; completion_tokens?: number }; + }; + const data = (await res.json()) as ChatResponse; + const content = data.choices?.[0]?.message?.content ?? ''; + const tokenUsage = data.usage + ? { + input: data.usage.prompt_tokens ?? 0, + output: data.usage.completion_tokens ?? 0, + } + : undefined; + return { content, tokenUsage }; + } +} diff --git a/services/mana-research/src/clients/mana-search.ts b/services/mana-research/src/clients/mana-search.ts new file mode 100644 index 000000000..360e688e5 --- /dev/null +++ b/services/mana-research/src/clients/mana-search.ts @@ -0,0 +1,91 @@ +/** + * HTTP client for mana-search (Go service on port 3021). + * Used by the SearXNGProvider and ReadabilityProvider wrappers. + */ + +import { ProviderError } from '../lib/errors'; + +export interface ManaSearchHit { + url: string; + title: string; + snippet: string; + engine?: string; + score?: number; + publishedDate?: string; + category?: string; +} + +export interface ManaSearchResponse { + results: ManaSearchHit[]; + meta: { + query: string; + totalResults: number; + engines: string[]; + cached: boolean; + duration: number; + }; +} + +export interface ManaExtractResponse { + success: boolean; + content?: { + title: string; + text: string; + markdown?: string; + author?: string; + publishedDate?: string; + siteName?: string; + wordCount: number; + }; +} + +export class ManaSearchClient { + constructor(private baseUrl: string) {} + + async search( + query: string, + options: { + limit?: number; + categories?: string[]; + language?: string; + signal?: AbortSignal; + } = {} + ): Promise { + const res = await fetch(`${this.baseUrl}/api/v1/search`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + query, + options: { + limit: options.limit ?? 10, + categories: options.categories, + language: options.language ?? 'de-DE', + }, + }), + signal: options.signal, + }); + if (!res.ok) { + throw new ProviderError('searxng', `mana-search returned ${res.status}`); + } + return res.json() as Promise; + } + + async extract( + url: string, + options: { maxLength?: number; signal?: AbortSignal } = {} + ): Promise { + const res = await fetch(`${this.baseUrl}/api/v1/extract`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + url, + options: { maxLength: options.maxLength ?? 50000, includeMarkdown: true }, + }), + signal: options.signal, + }); + if (!res.ok) { + throw new ProviderError('readability', `mana-search extract returned ${res.status}`); + } + return res.json() as Promise; + } +} diff --git a/services/mana-research/src/config.ts b/services/mana-research/src/config.ts new file mode 100644 index 000000000..1bf7492a5 --- /dev/null +++ b/services/mana-research/src/config.ts @@ -0,0 +1,68 @@ +/** + * Application configuration loaded from environment variables. + */ + +export interface Config { + port: number; + databaseUrl: string; + redisUrl: string; + manaAuthUrl: string; + manaLlmUrl: string; + manaCreditsUrl: string; + manaSearchUrl: string; + serviceKey: string; + cors: { origins: string[] }; + cacheTtlSeconds: number; + providerKeys: { + brave?: string; + tavily?: string; + exa?: string; + serper?: string; + perplexity?: string; + anthropic?: string; + openai?: string; + googleGenai?: string; + jina?: string; + firecrawl?: string; + scrapingbee?: string; + }; +} + +export function loadConfig(): Config { + const requiredEnv = (key: string, fallback?: string): string => { + const value = process.env[key] || fallback; + if (!value) throw new Error(`Missing required env var: ${key}`); + return value; + }; + + return { + port: parseInt(process.env.PORT || '3068', 10), + databaseUrl: requiredEnv( + 'DATABASE_URL', + 'postgresql://mana:devpassword@localhost:5432/mana_platform' + ), + redisUrl: process.env.REDIS_URL || 'redis://localhost:6379', + manaAuthUrl: requiredEnv('MANA_AUTH_URL', 'http://localhost:3001'), + manaLlmUrl: requiredEnv('MANA_LLM_URL', 'http://localhost:3025'), + manaCreditsUrl: requiredEnv('MANA_CREDITS_URL', 'http://localhost:3061'), + manaSearchUrl: requiredEnv('MANA_SEARCH_URL', 'http://localhost:3021'), + serviceKey: requiredEnv('MANA_SERVICE_KEY', 'dev-service-key'), + cors: { + origins: (process.env.CORS_ORIGINS || 'http://localhost:5173').split(','), + }, + cacheTtlSeconds: parseInt(process.env.CACHE_TTL_SECONDS || '3600', 10), + providerKeys: { + brave: process.env.BRAVE_API_KEY, + tavily: process.env.TAVILY_API_KEY, + exa: process.env.EXA_API_KEY, + serper: process.env.SERPER_API_KEY, + perplexity: process.env.PERPLEXITY_API_KEY, + anthropic: process.env.ANTHROPIC_API_KEY, + openai: process.env.OPENAI_API_KEY, + googleGenai: process.env.GOOGLE_GENAI_API_KEY, + jina: process.env.JINA_API_KEY, + firecrawl: process.env.FIRECRAWL_API_KEY, + scrapingbee: process.env.SCRAPINGBEE_API_KEY, + }, + }; +} diff --git a/services/mana-research/src/db/connection.ts b/services/mana-research/src/db/connection.ts new file mode 100644 index 000000000..89a121b3c --- /dev/null +++ b/services/mana-research/src/db/connection.ts @@ -0,0 +1,15 @@ +import { drizzle } from 'drizzle-orm/postgres-js'; +import postgres from 'postgres'; +import * as schema from './schema/index'; + +let db: ReturnType> | null = null; + +export function getDb(databaseUrl: string) { + if (!db) { + const client = postgres(databaseUrl, { max: 10 }); + db = drizzle(client, { schema }); + } + return db; +} + +export type Database = ReturnType; diff --git a/services/mana-research/src/db/schema/index.ts b/services/mana-research/src/db/schema/index.ts new file mode 100644 index 000000000..4404aa484 --- /dev/null +++ b/services/mana-research/src/db/schema/index.ts @@ -0,0 +1 @@ +export * from './research'; diff --git a/services/mana-research/src/db/schema/research.ts b/services/mana-research/src/db/schema/research.ts new file mode 100644 index 000000000..3e9d210cf --- /dev/null +++ b/services/mana-research/src/db/schema/research.ts @@ -0,0 +1,131 @@ +/** + * Research Schema — provider configs, eval runs, results, and stats. + * + * Lives in mana_platform DB under the `research` pgSchema. + * All userId columns are text without FK (separate ownership from auth.users). + */ + +import { + pgSchema, + uuid, + integer, + text, + timestamp, + jsonb, + boolean, + real, + index, + uniqueIndex, + primaryKey, + pgEnum, +} from 'drizzle-orm/pg-core'; + +export const researchSchema = pgSchema('research'); + +export const billingModeEnum = pgEnum('research_billing_mode', [ + 'server-key', + 'byo-key', + 'free', + 'mixed', +]); + +export const runCategoryEnum = pgEnum('research_run_category', ['search', 'extract', 'agent']); + +export const runModeEnum = pgEnum('research_run_mode', ['single', 'compare', 'auto']); + +/** A single research request: one query, one or more providers. */ +export const evalRuns = researchSchema.table( + 'eval_runs', + { + id: uuid('id').primaryKey().defaultRandom(), + userId: text('user_id'), + query: text('query').notNull(), + queryType: text('query_type'), + mode: runModeEnum('mode').notNull(), + category: runCategoryEnum('category').notNull(), + providersRequested: text('providers_requested').array().notNull(), + billingMode: billingModeEnum('billing_mode').notNull(), + totalCostCredits: integer('total_cost_credits').notNull().default(0), + createdAt: timestamp('created_at', { withTimezone: true }).defaultNow().notNull(), + }, + (t) => ({ + userIdx: index('eval_runs_user_idx').on(t.userId, t.createdAt), + queryIdx: index('eval_runs_query_idx').on(t.query), + }) +); + +/** One provider response per run. */ +export const evalResults = researchSchema.table( + 'eval_results', + { + id: uuid('id').primaryKey().defaultRandom(), + runId: uuid('run_id') + .notNull() + .references(() => evalRuns.id, { onDelete: 'cascade' }), + providerId: text('provider_id').notNull(), + success: boolean('success').notNull(), + latencyMs: integer('latency_ms').notNull(), + costCredits: integer('cost_credits').notNull().default(0), + billingMode: billingModeEnum('billing_mode').notNull(), + cacheHit: boolean('cache_hit').notNull().default(false), + rawResponse: jsonb('raw_response'), + normalizedResult: jsonb('normalized_result'), + errorCode: text('error_code'), + errorMessage: text('error_message'), + userRating: integer('user_rating'), + userNotes: text('user_notes'), + createdAt: timestamp('created_at', { withTimezone: true }).defaultNow().notNull(), + }, + (t) => ({ + runIdx: index('eval_results_run_idx').on(t.runId), + providerIdx: index('eval_results_provider_idx').on(t.providerId, t.createdAt), + }) +); + +/** Per-user BYO-key config + budgets. `userId=null` reserved for server-default row. */ +export const providerConfigs = researchSchema.table( + 'provider_configs', + { + id: uuid('id').primaryKey().defaultRandom(), + userId: text('user_id'), + providerId: text('provider_id').notNull(), + apiKeyEncrypted: text('api_key_encrypted'), + enabled: boolean('enabled').notNull().default(true), + dailyBudgetCredits: integer('daily_budget_credits'), + monthlyBudgetCredits: integer('monthly_budget_credits'), + createdAt: timestamp('created_at', { withTimezone: true }).defaultNow().notNull(), + updatedAt: timestamp('updated_at', { withTimezone: true }).defaultNow().notNull(), + }, + (t) => ({ + userProviderUnique: uniqueIndex('provider_configs_user_provider_unique').on( + t.userId, + t.providerId + ), + }) +); + +/** Aggregated per-day stats for Admin dashboard + auto-router. */ +export const providerStats = researchSchema.table( + 'provider_stats', + { + providerId: text('provider_id').notNull(), + day: text('day').notNull(), + totalCalls: integer('total_calls').notNull().default(0), + totalLatencyMs: integer('total_latency_ms').notNull().default(0), + totalCostCredits: integer('total_cost_credits').notNull().default(0), + successCount: integer('success_count').notNull().default(0), + errorCount: integer('error_count').notNull().default(0), + avgUserRating: real('avg_user_rating'), + ratingCount: integer('rating_count').notNull().default(0), + }, + (t) => ({ + pk: primaryKey({ columns: [t.providerId, t.day] }), + }) +); + +export type EvalRun = typeof evalRuns.$inferSelect; +export type NewEvalRun = typeof evalRuns.$inferInsert; +export type EvalResult = typeof evalResults.$inferSelect; +export type NewEvalResult = typeof evalResults.$inferInsert; +export type ProviderConfig = typeof providerConfigs.$inferSelect; +export type ProviderStat = typeof providerStats.$inferSelect; diff --git a/services/mana-research/src/executor/env-map.ts b/services/mana-research/src/executor/env-map.ts new file mode 100644 index 000000000..d9822edfb --- /dev/null +++ b/services/mana-research/src/executor/env-map.ts @@ -0,0 +1,25 @@ +import type { ProviderId } from '@mana/shared-research'; +import type { Config } from '../config'; + +/** + * Maps a ProviderId to the corresponding env-key slot on Config.providerKeys. + * Extract/agent providers that share a key with search (e.g. openai agents) + * reuse the same slot. + */ +export function mapEnvKey(providerId: ProviderId): keyof Config['providerKeys'] { + const map: Partial> = { + brave: 'brave', + tavily: 'tavily', + exa: 'exa', + serper: 'serper', + 'perplexity-sonar': 'perplexity', + 'claude-web-search': 'anthropic', + 'openai-responses': 'openai', + 'openai-deep-research': 'openai', + 'gemini-grounding': 'googleGenai', + 'jina-reader': 'jina', + firecrawl: 'firecrawl', + scrapingbee: 'scrapingbee', + }; + return map[providerId] ?? 'brave'; +} diff --git a/services/mana-research/src/executor/execute-extract.ts b/services/mana-research/src/executor/execute-extract.ts new file mode 100644 index 000000000..e6010b115 --- /dev/null +++ b/services/mana-research/src/executor/execute-extract.ts @@ -0,0 +1,150 @@ +/** + * Extract-side executor. Same shape as executeSearch but for URL extraction. + */ + +import type { + BillingMode, + ExtractedContent, + ExtractOptions, + ExtractProvider, + ProviderId, + ProviderMeta, +} from '@mana/shared-research'; +import type { CreditsClient } from '../clients/mana-credits'; +import type { Config } from '../config'; +import { ProviderNotConfiguredError } from '../lib/errors'; +import { priceFor } from '../lib/pricing'; +import type { ConfigStorage } from '../storage/configs'; +import { cacheGet, cacheKey, cacheSet } from '../lib/cache'; +import { mapEnvKey } from './env-map'; + +export interface ExecuteExtractInput { + provider: ExtractProvider; + url: string; + options: ExtractOptions; + userId: string; + signal?: AbortSignal; +} + +export interface ExecuteExtractOutput { + success: boolean; + data?: { content: ExtractedContent }; + meta: ProviderMeta; +} + +export interface ExecutorDeps { + credits: CreditsClient; + configs: ConfigStorage; + config: Config; +} + +export async function executeExtract( + input: ExecuteExtractInput, + deps: ExecutorDeps +): Promise { + const { provider, url, options, userId, signal } = input; + const providerId = provider.id; + const t0 = performance.now(); + + // Resolve API key (BYO → server → none) + let apiKey: string | null = null; + let billingMode: BillingMode = 'free'; + + if (provider.requiresApiKey) { + const userConfig = await deps.configs.getForUser(userId, providerId); + if (userConfig?.enabled && userConfig.apiKeyEncrypted) { + apiKey = await deps.configs.decryptKey(userConfig); + if (apiKey) billingMode = 'byo-key'; + } + if (!apiKey) { + apiKey = deps.config.providerKeys[mapEnvKey(providerId)] ?? null; + if (apiKey) billingMode = 'server-key'; + } + if (!apiKey) { + return makeError(providerId, t0, new ProviderNotConfiguredError(providerId)); + } + } else if (providerId === 'jina-reader' && deps.config.providerKeys.jina) { + // jina-reader is zero-auth but a key lifts the rate limit + apiKey = deps.config.providerKeys.jina; + } + + const price = billingMode === 'server-key' ? priceFor(providerId, 'extract') : 0; + + const ckey = cacheKey('extract', providerId, url, options); + const cached = await cacheGet<{ content: ExtractedContent }>(ckey); + if (cached) { + return { + success: true, + data: cached, + meta: { + provider: providerId, + category: 'extract', + latencyMs: Math.round(performance.now() - t0), + costCredits: 0, + cacheHit: true, + billingMode, + }, + }; + } + + let reservationId: string | null = null; + if (price > 0 && billingMode === 'server-key') { + try { + const reservation = await deps.credits.reserve( + userId, + price, + `research:${providerId}:extract` + ); + reservationId = reservation.reservationId; + } catch (err) { + return makeError(providerId, t0, err as Error); + } + } + + try { + const res = await provider.extract(url, options, { apiKey, userId, signal }); + await cacheSet(ckey, { content: res.content }, deps.config.cacheTtlSeconds * 24); + + if (reservationId) { + await deps.credits + .commit(reservationId, `extract ${providerId}`) + .catch((err) => console.warn('[executor] commit failed:', err)); + } + + return { + success: true, + data: { content: res.content }, + meta: { + provider: providerId, + category: 'extract', + latencyMs: Math.round(performance.now() - t0), + costCredits: price, + cacheHit: false, + billingMode, + }, + }; + } catch (err) { + if (reservationId) { + await deps.credits + .refund(reservationId) + .catch((refundErr) => console.warn('[executor] refund failed:', refundErr)); + } + return makeError(providerId, t0, err as Error); + } +} + +function makeError(providerId: ProviderId, t0: number, err: Error): ExecuteExtractOutput { + const code = (err as { code?: string }).code ?? err.name ?? 'ERROR'; + return { + success: false, + meta: { + provider: providerId, + category: 'extract', + latencyMs: Math.round(performance.now() - t0), + costCredits: 0, + cacheHit: false, + billingMode: 'free', + errorCode: code, + }, + }; +} diff --git a/services/mana-research/src/executor/execute.ts b/services/mana-research/src/executor/execute.ts new file mode 100644 index 000000000..00cac659c --- /dev/null +++ b/services/mana-research/src/executor/execute.ts @@ -0,0 +1,153 @@ +/** + * Core execution path: resolve key → reserve credits → call provider → + * commit/refund → persist result. + * + * Used by both /v1/search (single) and /v1/search/compare (fan-out). + */ + +import type { + BillingMode, + ProviderId, + ProviderMeta, + SearchHit, + SearchOptions, + SearchProvider, +} from '@mana/shared-research'; +import type { CreditsClient } from '../clients/mana-credits'; +import type { Config } from '../config'; +import { ProviderNotConfiguredError } from '../lib/errors'; +import { priceFor } from '../lib/pricing'; +import type { ConfigStorage } from '../storage/configs'; +import { cacheGet, cacheKey, cacheSet } from '../lib/cache'; +import { mapEnvKey } from './env-map'; + +export interface ExecuteSearchInput { + provider: SearchProvider; + query: string; + options: SearchOptions; + userId: string; + signal?: AbortSignal; +} + +export interface ExecuteSearchOutput { + success: boolean; + data?: { results: SearchHit[] }; + meta: ProviderMeta; +} + +export interface ExecutorDeps { + credits: CreditsClient; + configs: ConfigStorage; + config: Config; +} + +export async function executeSearch( + input: ExecuteSearchInput, + deps: ExecutorDeps +): Promise { + const { provider, query, options, userId, signal } = input; + const providerId = provider.id; + const t0 = performance.now(); + + // Resolve API key (BYO first, then server) + let apiKey: string | null = null; + let billingMode: BillingMode = 'free'; + + if (provider.requiresApiKey) { + const userConfig = await deps.configs.getForUser(userId, providerId); + if (userConfig?.enabled && userConfig.apiKeyEncrypted) { + apiKey = await deps.configs.decryptKey(userConfig); + if (apiKey) billingMode = 'byo-key'; + } + if (!apiKey) { + apiKey = deps.config.providerKeys[mapEnvKey(providerId)] ?? null; + if (apiKey) billingMode = 'server-key'; + } + if (!apiKey) { + return makeError(providerId, t0, new ProviderNotConfiguredError(providerId)); + } + } + + const price = billingMode === 'server-key' ? priceFor(providerId, 'search') : 0; + + // Cache check + const ckey = cacheKey('search', providerId, query, options); + const cached = await cacheGet<{ results: SearchHit[] }>(ckey); + if (cached) { + return { + success: true, + data: cached, + meta: { + provider: providerId, + category: 'search', + latencyMs: Math.round(performance.now() - t0), + costCredits: 0, + cacheHit: true, + billingMode, + }, + }; + } + + // Reserve credits for paid server-key calls + let reservationId: string | null = null; + if (price > 0 && billingMode === 'server-key') { + try { + const reservation = await deps.credits.reserve( + userId, + price, + `research:${providerId}:search` + ); + reservationId = reservation.reservationId; + } catch (err) { + return makeError(providerId, t0, err as Error); + } + } + + // Execute provider + try { + const res = await provider.search(query, options, { apiKey, userId, signal }); + await cacheSet(ckey, { results: res.results }, deps.config.cacheTtlSeconds); + + if (reservationId) { + await deps.credits + .commit(reservationId, `search ${providerId}`) + .catch((err) => console.warn('[executor] commit failed:', err)); + } + + return { + success: true, + data: { results: res.results }, + meta: { + provider: providerId, + category: 'search', + latencyMs: Math.round(performance.now() - t0), + costCredits: price, + cacheHit: false, + billingMode, + }, + }; + } catch (err) { + if (reservationId) { + await deps.credits + .refund(reservationId) + .catch((refundErr) => console.warn('[executor] refund failed:', refundErr)); + } + return makeError(providerId, t0, err as Error); + } +} + +function makeError(providerId: ProviderId, t0: number, err: Error): ExecuteSearchOutput { + const code = (err as { code?: string }).code ?? err.name ?? 'ERROR'; + return { + success: false, + meta: { + provider: providerId, + category: 'search', + latencyMs: Math.round(performance.now() - t0), + costCredits: 0, + cacheHit: false, + billingMode: 'free', + errorCode: code, + }, + }; +} diff --git a/services/mana-research/src/index.ts b/services/mana-research/src/index.ts new file mode 100644 index 000000000..d41b3944c --- /dev/null +++ b/services/mana-research/src/index.ts @@ -0,0 +1,101 @@ +/** + * mana-research — Web Research Provider Orchestration + * + * Bundles search/extract/agent providers behind a unified interface. + * Phase 1: 4 search providers (SearXNG, DuckDuckGo, Brave, Tavily) with + * credits + cache + eval-run persistence. + * + * Port: 3068. See docs/plans/mana-research-service.md. + */ + +import { Hono } from 'hono'; +import { cors } from 'hono/cors'; +import { loadConfig } from './config'; +import { getDb } from './db/connection'; +import { serviceErrorHandler } from '@mana/shared-hono'; +import { jwtAuth } from './middleware/jwt-auth'; +import { serviceAuth } from './middleware/service-auth'; +import { healthRoutes } from './routes/health'; +import { createSearchRoutes } from './routes/search'; +import { createExtractRoutes } from './routes/extract'; +import { createProvidersRoutes } from './routes/providers'; +import { createRunsRoutes } from './routes/runs'; +import { buildRegistry } from './providers/registry'; +import { RunStorage } from './storage/runs'; +import { ConfigStorage } from './storage/configs'; +import { CreditsClient } from './clients/mana-credits'; +import { ManaSearchClient } from './clients/mana-search'; +import { ManaLlmClient } from './clients/mana-llm'; +import { initCache } from './lib/cache'; + +// ─── Bootstrap ────────────────────────────────────────────── + +const config = loadConfig(); +const db = getDb(config.databaseUrl); + +initCache(config.redisUrl); + +const manaSearch = new ManaSearchClient(config.manaSearchUrl); +const manaLlm = new ManaLlmClient(config.manaLlmUrl); +const credits = new CreditsClient({ + baseUrl: config.manaCreditsUrl, + serviceKey: config.serviceKey, +}); + +const runStorage = new RunStorage(db); +const configStorage = new ConfigStorage(db); +const registry = buildRegistry({ manaSearch }); + +const executorDeps = { + credits, + configs: configStorage, + config, +}; + +// ─── App ──────────────────────────────────────────────────── + +const app = new Hono(); + +app.onError(serviceErrorHandler); +app.use( + '*', + cors({ + origin: config.cors.origins, + credentials: true, + }) +); + +// Health (no auth) +app.route('/health', healthRoutes); + +// Metrics stub (no auth) — will be populated in Phase 2 with prometheus-style output +app.get('/metrics', (c) => c.text('# mana-research metrics stub\n')); + +// Providers catalog (no auth — callers often query this to build UIs) +app.route('/api/v1/providers', createProvidersRoutes(registry, config)); + +// User-facing research (JWT auth) +app.use('/api/v1/search/*', jwtAuth(config.manaAuthUrl)); +app.route( + '/api/v1/search', + createSearchRoutes(registry, runStorage, executorDeps, config, manaLlm) +); + +app.use('/api/v1/extract/*', jwtAuth(config.manaAuthUrl)); +app.route('/api/v1/extract', createExtractRoutes(registry, runStorage, executorDeps, config)); + +app.use('/api/v1/runs/*', jwtAuth(config.manaAuthUrl)); +app.route('/api/v1/runs', createRunsRoutes(runStorage)); + +// Service-to-service (X-Service-Key auth) — wired up in Phase 3 when mana-ai migrates +app.use('/api/v1/internal/*', serviceAuth(config.serviceKey)); +app.get('/api/v1/internal/health', (c) => c.json({ ok: true })); + +// ─── Start ────────────────────────────────────────────────── + +console.log(`mana-research starting on port ${config.port}...`); + +export default { + port: config.port, + fetch: app.fetch, +}; diff --git a/services/mana-research/src/lib/cache.ts b/services/mana-research/src/lib/cache.ts new file mode 100644 index 000000000..3d162b418 --- /dev/null +++ b/services/mana-research/src/lib/cache.ts @@ -0,0 +1,59 @@ +/** + * Redis cache wrapper. Graceful degradation — if Redis is down, cache methods + * return null (miss) and set() is a no-op so the service still works. + */ + +import Redis from 'ioredis'; +import { createHash } from 'node:crypto'; + +let redis: Redis | null = null; + +export function initCache(redisUrl: string) { + if (redis) return redis; + redis = new Redis(redisUrl, { + lazyConnect: true, + maxRetriesPerRequest: 2, + enableOfflineQueue: false, + }); + redis.on('error', (err) => { + console.warn('[cache] redis error:', err.message); + }); + redis.connect().catch((err) => { + console.warn('[cache] connect failed, running without cache:', err.message); + }); + return redis; +} + +export function cacheKey( + category: string, + providerId: string, + query: string, + opts: unknown +): string { + const h = createHash('sha256'); + h.update(providerId); + h.update('\0'); + h.update(query); + h.update('\0'); + h.update(JSON.stringify(opts ?? {})); + return `research:${category}:${providerId}:${h.digest('hex').slice(0, 32)}`; +} + +export async function cacheGet(key: string): Promise { + if (!redis || redis.status !== 'ready') return null; + try { + const raw = await redis.get(key); + return raw ? (JSON.parse(raw) as T) : null; + } catch { + return null; + } +} + +export async function cacheSet(key: string, value: T, ttlSeconds: number): Promise { + if (!redis || redis.status !== 'ready') return; + try { + await redis.setex(key, ttlSeconds, JSON.stringify(value)); + } catch { + /* ignore */ + } +} diff --git a/services/mana-research/src/lib/errors.ts b/services/mana-research/src/lib/errors.ts new file mode 100644 index 000000000..a0bc987db --- /dev/null +++ b/services/mana-research/src/lib/errors.ts @@ -0,0 +1,54 @@ +/** + * Custom errors for mana-research. All extend HTTPException so `serviceErrorHandler` + * from @mana/shared-hono renders them with status + legacy `{ statusCode, message, details }`. + */ + +import { HTTPException } from 'hono/http-exception'; + +export class BadRequestError extends HTTPException { + constructor(message: string, cause?: Record) { + super(400, { message, cause }); + } +} + +export class UnauthorizedError extends HTTPException { + constructor(message = 'Unauthorized') { + super(401, { message }); + } +} + +export class NotFoundError extends HTTPException { + constructor(message = 'Not found') { + super(404, { message }); + } +} + +export class InsufficientCreditsError extends HTTPException { + constructor( + public readonly required: number, + public readonly available: number + ) { + super(402, { + message: 'Insufficient credits', + cause: { required, available }, + }); + } +} + +export class ProviderError extends HTTPException { + constructor(providerId: string, message: string, status: 500 | 502 | 503 = 502) { + super(status, { + message: `Provider "${providerId}" error: ${message}`, + cause: { providerId }, + }); + } +} + +export class ProviderNotConfiguredError extends HTTPException { + constructor(providerId: string) { + super(501, { + message: `Provider "${providerId}" is not configured — no API key available`, + cause: { providerId }, + }); + } +} diff --git a/services/mana-research/src/lib/hono-env.ts b/services/mana-research/src/lib/hono-env.ts new file mode 100644 index 000000000..9eebd6db4 --- /dev/null +++ b/services/mana-research/src/lib/hono-env.ts @@ -0,0 +1,16 @@ +/** + * Hono Context Variables typing for mana-research. + * + * Allows `c.get('user')` / `c.set('user', ...)` to be typed throughout the + * service. Used via `new Hono<{ Variables: HonoVariables }>()`. + */ + +import type { AuthUser } from '../middleware/jwt-auth'; + +export interface HonoVariables { + user: AuthUser; + appId?: string; + service?: boolean; +} + +export type HonoEnv = { Variables: HonoVariables }; diff --git a/services/mana-research/src/lib/pricing.ts b/services/mana-research/src/lib/pricing.ts new file mode 100644 index 000000000..4096ef27a --- /dev/null +++ b/services/mana-research/src/lib/pricing.ts @@ -0,0 +1,42 @@ +/** + * Provider pricing in credits. 1 credit ≈ 1 cent EUR (matches mana-credits). + * + * Keep in sync with docs/plans/mana-research-service.md §2. Review quarterly. + * Prices as of 2026-04-17. + */ + +import type { ProviderId } from '@mana/shared-research'; + +export const PROVIDER_PRICING: Record< + ProviderId, + { search?: number; extract?: number; research?: number } +> = { + // Search providers + searxng: { search: 0 }, + duckduckgo: { search: 0 }, + brave: { search: 5 }, + tavily: { search: 8 }, + exa: { search: 6 }, + serper: { search: 1 }, + + // Extract providers + readability: { extract: 0 }, + 'jina-reader': { extract: 1 }, + firecrawl: { extract: 10 }, + scrapingbee: { extract: 8 }, + + // Research agents + 'perplexity-sonar': { research: 50 }, + 'claude-web-search': { research: 200 }, + 'openai-responses': { research: 200 }, + 'gemini-grounding': { research: 100 }, + 'openai-deep-research': { research: 1000 }, +}; + +export function priceFor( + providerId: ProviderId, + operation: 'search' | 'extract' | 'research' +): number { + const entry = PROVIDER_PRICING[providerId]; + return entry?.[operation] ?? 0; +} diff --git a/services/mana-research/src/middleware/jwt-auth.ts b/services/mana-research/src/middleware/jwt-auth.ts new file mode 100644 index 000000000..859a8c1d0 --- /dev/null +++ b/services/mana-research/src/middleware/jwt-auth.ts @@ -0,0 +1,54 @@ +/** + * JWT Authentication Middleware + * + * Validates Bearer tokens via JWKS from mana-auth. Mirrors mana-credits pattern. + */ + +import type { MiddlewareHandler } from 'hono'; +import { createRemoteJWKSet, jwtVerify } from 'jose'; +import { UnauthorizedError } from '../lib/errors'; + +let jwks: ReturnType | null = null; + +function getJwks(authUrl: string) { + if (!jwks) { + jwks = createRemoteJWKSet(new URL('/api/auth/jwks', authUrl)); + } + return jwks; +} + +export interface AuthUser { + userId: string; + email: string; + role: string; + tier?: string; +} + +export function jwtAuth(authUrl: string): MiddlewareHandler { + return async (c, next) => { + const authHeader = c.req.header('Authorization'); + if (!authHeader?.startsWith('Bearer ')) { + throw new UnauthorizedError('Missing or invalid Authorization header'); + } + + const token = authHeader.slice(7); + try { + const { payload } = await jwtVerify(token, getJwks(authUrl), { + issuer: authUrl, + audience: 'mana', + }); + + const user: AuthUser = { + userId: payload.sub || '', + email: (payload.email as string) || '', + role: (payload.role as string) || 'user', + tier: payload.tier as string | undefined, + }; + + c.set('user', user); + await next(); + } catch { + throw new UnauthorizedError('Invalid or expired token'); + } + }; +} diff --git a/services/mana-research/src/middleware/service-auth.ts b/services/mana-research/src/middleware/service-auth.ts new file mode 100644 index 000000000..c9e8b7948 --- /dev/null +++ b/services/mana-research/src/middleware/service-auth.ts @@ -0,0 +1,22 @@ +/** + * Service-to-Service Authentication Middleware + * + * Validates X-Service-Key header for backend-to-backend calls. + * Used by /internal/* routes and for calls from mana-ai. + */ + +import type { MiddlewareHandler } from 'hono'; +import { UnauthorizedError } from '../lib/errors'; + +export function serviceAuth(serviceKey: string): MiddlewareHandler { + return async (c, next) => { + const key = c.req.header('X-Service-Key'); + if (!key || key !== serviceKey) { + throw new UnauthorizedError('Invalid or missing service key'); + } + const appId = c.req.header('X-App-Id') || 'unknown'; + c.set('appId', appId); + c.set('service', true); + await next(); + }; +} diff --git a/services/mana-research/src/providers/extract/firecrawl.ts b/services/mana-research/src/providers/extract/firecrawl.ts new file mode 100644 index 000000000..ac91735c7 --- /dev/null +++ b/services/mana-research/src/providers/extract/firecrawl.ts @@ -0,0 +1,86 @@ +/** + * Firecrawl — Playwright-based JS rendering + LLM-friendly Markdown output. + * Docs: https://docs.firecrawl.dev/api-reference/endpoint/scrape + * + * Pay-per-use credits. Self-hostable via Docker (then set FIRECRAWL_API_URL to + * your own instance and any non-empty key works). + */ + +import type { ExtractProvider } from '@mana/shared-research'; +import { ProviderError, ProviderNotConfiguredError } from '../../lib/errors'; + +interface FirecrawlApiResponse { + success: boolean; + data?: { + markdown?: string; + html?: string; + metadata?: { + title?: string; + description?: string; + language?: string; + sourceURL?: string; + author?: string; + publishedTime?: string; + ogSiteName?: string; + }; + }; + error?: string; +} + +export function createFirecrawlProvider(apiUrl = 'https://api.firecrawl.dev'): ExtractProvider { + return { + id: 'firecrawl', + requiresApiKey: true, + capabilities: { + jsRendering: true, + pdfSupport: true, + markdownOutput: true, + }, + async extract(url, options, ctx) { + if (!ctx.apiKey) throw new ProviderNotConfiguredError('firecrawl'); + const t0 = performance.now(); + + const res = await fetch(`${apiUrl}/v1/scrape`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${ctx.apiKey}`, + }, + body: JSON.stringify({ + url, + formats: ['markdown'], + onlyMainContent: true, + timeout: options.timeoutMs ?? 30000, + }), + signal: ctx.signal, + }); + + if (!res.ok) { + const body = await res.text().catch(() => ''); + throw new ProviderError('firecrawl', `HTTP ${res.status} ${body.slice(0, 200)}`); + } + + const data = (await res.json()) as FirecrawlApiResponse; + if (!data.success || !data.data) { + throw new ProviderError('firecrawl', data.error ?? 'extraction failed'); + } + + const md = (data.data.markdown ?? '').slice(0, options.maxLength ?? 50000); + const meta = data.data.metadata ?? {}; + return { + content: { + url: meta.sourceURL ?? url, + title: meta.title ?? '', + content: md, + excerpt: meta.description, + author: meta.author, + siteName: meta.ogSiteName, + publishedAt: meta.publishedTime, + wordCount: md.split(/\s+/).filter(Boolean).length, + providerRaw: data, + }, + rawLatencyMs: Math.round(performance.now() - t0), + }; + }, + }; +} diff --git a/services/mana-research/src/providers/extract/jina-reader.ts b/services/mana-research/src/providers/extract/jina-reader.ts new file mode 100644 index 000000000..07e333832 --- /dev/null +++ b/services/mana-research/src/providers/extract/jina-reader.ts @@ -0,0 +1,66 @@ +/** + * Jina Reader — `https://r.jina.ai/` returns extracted Markdown. + * Free tier: 1M tokens/month without key. Paid tier via `JINA_API_KEY` lifts rate limit. + * + * The service is markedly better than plain Readability on JS-heavy sites. + */ + +import type { ExtractProvider } from '@mana/shared-research'; +import { ProviderError } from '../../lib/errors'; + +export function createJinaReaderProvider(): ExtractProvider { + return { + id: 'jina-reader', + requiresApiKey: false, + capabilities: { + jsRendering: true, + pdfSupport: true, + markdownOutput: true, + }, + async extract(url, options, ctx) { + const t0 = performance.now(); + const readerUrl = `https://r.jina.ai/${url}`; + + const headers: Record = { + Accept: 'application/json', + 'X-Return-Format': 'markdown', + }; + if (ctx.apiKey) headers.Authorization = `Bearer ${ctx.apiKey}`; + if (options.timeoutMs) headers['X-Timeout'] = String(Math.round(options.timeoutMs / 1000)); + + const res = await fetch(readerUrl, { headers, signal: ctx.signal }); + + if (!res.ok) { + const body = await res.text().catch(() => ''); + throw new ProviderError('jina-reader', `HTTP ${res.status} ${body.slice(0, 200)}`); + } + + type JinaResponse = { + data?: { + title?: string; + url?: string; + content?: string; + description?: string; + publishedTime?: string; + }; + }; + const data = (await res.json()) as JinaResponse; + const d = data.data ?? {}; + const content = (d.content ?? '').slice(0, options.maxLength ?? 50000); + const wordCount = content.split(/\s+/).filter(Boolean).length; + + return { + content: { + url: d.url ?? url, + title: d.title ?? '', + content, + excerpt: d.description, + publishedAt: d.publishedTime, + wordCount, + providerRaw: data, + }, + rawLatencyMs: Math.round(performance.now() - t0), + }; + }, + }; +} diff --git a/services/mana-research/src/providers/extract/readability.ts b/services/mana-research/src/providers/extract/readability.ts new file mode 100644 index 000000000..e077bee93 --- /dev/null +++ b/services/mana-research/src/providers/extract/readability.ts @@ -0,0 +1,43 @@ +/** + * Readability Extract Provider — wraps mana-search /api/v1/extract (go-readability). + * Free (self-hosted), no JS rendering. Good baseline for simple HTML. + */ + +import type { ExtractProvider } from '@mana/shared-research'; +import { ProviderError } from '../../lib/errors'; +import type { ManaSearchClient } from '../../clients/mana-search'; + +export function createReadabilityProvider(client: ManaSearchClient): ExtractProvider { + return { + id: 'readability', + requiresApiKey: false, + capabilities: { + jsRendering: false, + pdfSupport: false, + markdownOutput: true, + }, + async extract(url, options, ctx) { + const t0 = performance.now(); + const res = await client.extract(url, { maxLength: options.maxLength, signal: ctx.signal }); + + if (!res.success || !res.content) { + throw new ProviderError('readability', 'extraction failed'); + } + + const c = res.content; + return { + content: { + url, + title: c.title, + content: c.markdown ?? c.text, + author: c.author, + siteName: c.siteName, + publishedAt: c.publishedDate, + wordCount: c.wordCount, + providerRaw: res, + }, + rawLatencyMs: Math.round(performance.now() - t0), + }; + }, + }; +} diff --git a/services/mana-research/src/providers/registry.ts b/services/mana-research/src/providers/registry.ts new file mode 100644 index 000000000..8cbc79422 --- /dev/null +++ b/services/mana-research/src/providers/registry.ts @@ -0,0 +1,85 @@ +/** + * Provider registry — maps provider IDs to their instances + metadata. + */ + +import type { + AgentProviderId, + ExtractProvider, + ExtractProviderId, + ProviderId, + SearchProvider, + SearchProviderId, +} from '@mana/shared-research'; +import { BadRequestError } from '../lib/errors'; +import type { ManaSearchClient } from '../clients/mana-search'; +import { createBraveProvider } from './search/brave'; +import { createDuckDuckGoProvider } from './search/duckduckgo'; +import { createExaProvider } from './search/exa'; +import { createSearxngProvider } from './search/searxng'; +import { createSerperProvider } from './search/serper'; +import { createTavilyProvider } from './search/tavily'; +import { createFirecrawlProvider } from './extract/firecrawl'; +import { createJinaReaderProvider } from './extract/jina-reader'; +import { createReadabilityProvider } from './extract/readability'; + +export interface ProviderRegistry { + search: Map; + extract: Map; +} + +export function buildRegistry(deps: { manaSearch: ManaSearchClient }): ProviderRegistry { + const search = new Map(); + search.set('searxng', createSearxngProvider(deps.manaSearch)); + search.set('duckduckgo', createDuckDuckGoProvider()); + search.set('brave', createBraveProvider()); + search.set('tavily', createTavilyProvider()); + search.set('exa', createExaProvider()); + search.set('serper', createSerperProvider()); + + const extract = new Map(); + extract.set('readability', createReadabilityProvider(deps.manaSearch)); + extract.set('jina-reader', createJinaReaderProvider()); + extract.set('firecrawl', createFirecrawlProvider()); + + return { search, extract }; +} + +export function getSearchProvider(registry: ProviderRegistry, id: string): SearchProvider { + const provider = registry.search.get(id as SearchProviderId); + if (!provider) { + throw new BadRequestError(`Unknown search provider: ${id}`, { + available: [...registry.search.keys()], + }); + } + return provider; +} + +export function getExtractProvider(registry: ProviderRegistry, id: string): ExtractProvider { + const provider = registry.extract.get(id as ExtractProviderId); + if (!provider) { + throw new BadRequestError(`Unknown extract provider: ${id}`, { + available: [...registry.extract.keys()], + }); + } + return provider; +} + +export function listProviders(registry: ProviderRegistry) { + return { + search: [...registry.search.values()].map((p) => ({ + id: p.id, + category: 'search' as const, + requiresApiKey: p.requiresApiKey, + capabilities: p.capabilities, + })), + extract: [...registry.extract.values()].map((p) => ({ + id: p.id, + category: 'extract' as const, + requiresApiKey: p.requiresApiKey, + capabilities: p.capabilities, + })), + agent: [] as Array<{ id: AgentProviderId; category: 'agent'; requiresApiKey: boolean }>, + }; +} + +export type { ProviderId }; diff --git a/services/mana-research/src/providers/search/brave.ts b/services/mana-research/src/providers/search/brave.ts new file mode 100644 index 000000000..a5dbb9681 --- /dev/null +++ b/services/mana-research/src/providers/search/brave.ts @@ -0,0 +1,74 @@ +/** + * Brave Search API provider. + * Docs: https://api.search.brave.com/app/documentation/web-search/get-started + * + * Pay-per-use (Data for Search plan, $5 / 1k queries). Requires X-Subscription-Token header. + */ + +import type { SearchProvider } from '@mana/shared-research'; +import { ProviderError, ProviderNotConfiguredError } from '../../lib/errors'; + +interface BraveApiResponse { + web?: { + results?: Array<{ + url: string; + title: string; + description: string; + age?: string; + page_age?: string; + profile?: { name?: string }; + }>; + }; +} + +export function createBraveProvider(): SearchProvider { + return { + id: 'brave', + requiresApiKey: true, + capabilities: { + webSearch: true, + newsSearch: true, + }, + async search(query, options, ctx) { + if (!ctx.apiKey) throw new ProviderNotConfiguredError('brave'); + const t0 = performance.now(); + + const params = new URLSearchParams({ + q: query, + count: String(options.limit ?? 10), + }); + if (options.language) params.set('search_lang', options.language.split('-')[0]); + if (options.safeSearch !== undefined) { + params.set('safesearch', ['off', 'moderate', 'strict'][options.safeSearch] || 'moderate'); + } + + const res = await fetch(`https://api.search.brave.com/res/v1/web/search?${params}`, { + headers: { + Accept: 'application/json', + 'X-Subscription-Token': ctx.apiKey, + }, + signal: ctx.signal, + }); + + if (!res.ok) { + const body = await res.text().catch(() => ''); + throw new ProviderError('brave', `HTTP ${res.status} ${body.slice(0, 200)}`); + } + + const data = (await res.json()) as BraveApiResponse; + const webResults = data.web?.results ?? []; + + return { + results: webResults.map((r) => ({ + url: r.url, + title: r.title, + snippet: r.description, + publishedAt: r.page_age ?? r.age, + author: r.profile?.name, + providerRaw: r, + })), + rawLatencyMs: Math.round(performance.now() - t0), + }; + }, + }; +} diff --git a/services/mana-research/src/providers/search/duckduckgo.ts b/services/mana-research/src/providers/search/duckduckgo.ts new file mode 100644 index 000000000..dac20dbf7 --- /dev/null +++ b/services/mana-research/src/providers/search/duckduckgo.ts @@ -0,0 +1,68 @@ +/** + * DuckDuckGo provider — uses DDG HTML search via the unofficial "html.duckduckgo.com" + * endpoint. Zero-auth, zero-cost, but heavily rate-limited in practice. + * + * For Phase 1 we keep this minimal: good as a free fallback / sanity check. + */ + +import type { SearchProvider, SearchHit } from '@mana/shared-research'; +import { ProviderError } from '../../lib/errors'; + +interface DDGInstantAnswer { + Heading?: string; + Abstract?: string; + AbstractURL?: string; + RelatedTopics?: Array<{ + Text?: string; + FirstURL?: string; + Icon?: unknown; + }>; +} + +export function createDuckDuckGoProvider(): SearchProvider { + return { + id: 'duckduckgo', + requiresApiKey: false, + capabilities: { + webSearch: true, + }, + async search(query, options, ctx) { + const t0 = performance.now(); + const url = `https://api.duckduckgo.com/?q=${encodeURIComponent(query)}&format=json&no_html=1&no_redirect=1`; + const res = await fetch(url, { + signal: ctx.signal, + headers: { 'User-Agent': 'Mozilla/5.0 (mana-research)' }, + }); + if (!res.ok) { + throw new ProviderError('duckduckgo', `HTTP ${res.status}`); + } + const data = (await res.json()) as DDGInstantAnswer; + const hits: SearchHit[] = []; + + if (data.AbstractURL && data.Abstract) { + hits.push({ + url: data.AbstractURL, + title: data.Heading ?? query, + snippet: data.Abstract, + providerRaw: data, + }); + } + + for (const topic of data.RelatedTopics ?? []) { + if (!topic.FirstURL || !topic.Text) continue; + hits.push({ + url: topic.FirstURL, + title: topic.Text.slice(0, 80), + snippet: topic.Text, + providerRaw: topic, + }); + if (hits.length >= (options.limit ?? 10)) break; + } + + return { + results: hits, + rawLatencyMs: Math.round(performance.now() - t0), + }; + }, + }; +} diff --git a/services/mana-research/src/providers/search/exa.ts b/services/mana-research/src/providers/search/exa.ts new file mode 100644 index 000000000..cba40cbfd --- /dev/null +++ b/services/mana-research/src/providers/search/exa.ts @@ -0,0 +1,76 @@ +/** + * Exa (formerly Metaphor) — semantic/neural search. + * Docs: https://docs.exa.ai/reference/search + * + * Best for "find similar to this", academic papers, long-tail technical queries. + * Pay-per-use, ~$0.001–0.01/query depending on options. + */ + +import type { SearchProvider } from '@mana/shared-research'; +import { ProviderError, ProviderNotConfiguredError } from '../../lib/errors'; + +interface ExaApiResponse { + results: Array<{ + id: string; + url: string; + title: string; + publishedDate?: string; + author?: string; + score?: number; + text?: string; + }>; +} + +export function createExaProvider(): SearchProvider { + return { + id: 'exa', + requiresApiKey: true, + capabilities: { + webSearch: true, + semanticSearch: true, + scholarSearch: true, + contentInResults: true, + }, + async search(query, options, ctx) { + if (!ctx.apiKey) throw new ProviderNotConfiguredError('exa'); + const t0 = performance.now(); + + const res = await fetch('https://api.exa.ai/search', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'x-api-key': ctx.apiKey, + }, + body: JSON.stringify({ + query, + numResults: Math.min(options.limit ?? 10, 25), + type: 'neural', + useAutoprompt: true, + contents: { text: { maxCharacters: 2000 } }, + }), + signal: ctx.signal, + }); + + if (!res.ok) { + const body = await res.text().catch(() => ''); + throw new ProviderError('exa', `HTTP ${res.status} ${body.slice(0, 200)}`); + } + + const data = (await res.json()) as ExaApiResponse; + + return { + results: data.results.map((r) => ({ + url: r.url, + title: r.title, + snippet: r.text?.slice(0, 300) ?? '', + content: r.text, + publishedAt: r.publishedDate, + author: r.author, + score: r.score, + providerRaw: r, + })), + rawLatencyMs: Math.round(performance.now() - t0), + }; + }, + }; +} diff --git a/services/mana-research/src/providers/search/searxng.ts b/services/mana-research/src/providers/search/searxng.ts new file mode 100644 index 000000000..2bdf05745 --- /dev/null +++ b/services/mana-research/src/providers/search/searxng.ts @@ -0,0 +1,48 @@ +/** + * SearXNG provider — wraps mana-search (Go service on port 3021). + * Free, self-hosted, no API key. Always available. + */ + +import type { SearchProvider } from '@mana/shared-research'; +import type { ManaSearchClient } from '../../clients/mana-search'; + +export function createSearxngProvider(client: ManaSearchClient): SearchProvider { + return { + id: 'searxng', + requiresApiKey: false, + capabilities: { + webSearch: true, + newsSearch: true, + scholarSearch: true, + }, + async search(query, options, ctx) { + const t0 = performance.now(); + const categoryMap: Record = { + general: 'general', + news: 'news', + science: 'science', + it: 'it', + }; + const categories = options.categories?.map((c) => categoryMap[c]).filter(Boolean); + + const res = await client.search(query, { + limit: options.limit, + categories, + language: options.language, + signal: ctx.signal, + }); + + return { + results: res.results.map((r) => ({ + url: r.url, + title: r.title, + snippet: r.snippet, + publishedAt: r.publishedDate, + score: r.score, + providerRaw: r, + })), + rawLatencyMs: Math.round(performance.now() - t0), + }; + }, + }; +} diff --git a/services/mana-research/src/providers/search/serper.ts b/services/mana-research/src/providers/search/serper.ts new file mode 100644 index 000000000..66bae8f25 --- /dev/null +++ b/services/mana-research/src/providers/search/serper.ts @@ -0,0 +1,78 @@ +/** + * Serper — Google SERP as JSON. + * Docs: https://serper.dev/ + * + * Good for classic Google search coverage (incl. People Also Ask, Knowledge Panel). + * $0.30–1 / 1k queries. Pay-per-use. + */ + +import type { SearchProvider } from '@mana/shared-research'; +import { ProviderError, ProviderNotConfiguredError } from '../../lib/errors'; + +interface SerperApiResponse { + organic?: Array<{ + title: string; + link: string; + snippet: string; + date?: string; + position?: number; + }>; + answerBox?: { + title?: string; + answer?: string; + snippet?: string; + link?: string; + }; +} + +export function createSerperProvider(): SearchProvider { + return { + id: 'serper', + requiresApiKey: true, + capabilities: { + webSearch: true, + newsSearch: true, + }, + async search(query, options, ctx) { + if (!ctx.apiKey) throw new ProviderNotConfiguredError('serper'); + const t0 = performance.now(); + + const [gl, hl] = (options.language ?? 'de-DE').toLowerCase().split('-'); + + const res = await fetch('https://google.serper.dev/search', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-API-KEY': ctx.apiKey, + }, + body: JSON.stringify({ + q: query, + num: Math.min(options.limit ?? 10, 20), + gl: hl || gl, + hl: gl, + }), + signal: ctx.signal, + }); + + if (!res.ok) { + const body = await res.text().catch(() => ''); + throw new ProviderError('serper', `HTTP ${res.status} ${body.slice(0, 200)}`); + } + + const data = (await res.json()) as SerperApiResponse; + const organic = data.organic ?? []; + + return { + results: organic.map((r) => ({ + url: r.link, + title: r.title, + snippet: r.snippet, + publishedAt: r.date, + score: r.position ? 1 - r.position / 100 : undefined, + providerRaw: r, + })), + rawLatencyMs: Math.round(performance.now() - t0), + }; + }, + }; +} diff --git a/services/mana-research/src/providers/search/tavily.ts b/services/mana-research/src/providers/search/tavily.ts new file mode 100644 index 000000000..f345c5858 --- /dev/null +++ b/services/mana-research/src/providers/search/tavily.ts @@ -0,0 +1,75 @@ +/** + * Tavily Search API — optimized for LLM agents. Returns extracted content + * alongside links, which is why we map it to `content` on SearchHit. + * + * Docs: https://docs.tavily.com/docs/rest-api/api-reference + * Billing: credit-packs (pay-per-use, no subscription lock-in). + */ + +import type { SearchProvider } from '@mana/shared-research'; +import { ProviderError, ProviderNotConfiguredError } from '../../lib/errors'; + +interface TavilyApiResponse { + query: string; + answer?: string; + results: Array<{ + url: string; + title: string; + content: string; + score: number; + published_date?: string; + }>; +} + +export function createTavilyProvider(): SearchProvider { + return { + id: 'tavily', + requiresApiKey: true, + capabilities: { + webSearch: true, + newsSearch: true, + contentInResults: true, + }, + async search(query, options, ctx) { + if (!ctx.apiKey) throw new ProviderNotConfiguredError('tavily'); + const t0 = performance.now(); + + const topic = options.categories?.includes('news') ? 'news' : 'general'; + const maxResults = Math.min(options.limit ?? 10, 20); + + const res = await fetch('https://api.tavily.com/search', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + api_key: ctx.apiKey, + query, + topic, + max_results: maxResults, + include_answer: false, + search_depth: 'basic', + }), + signal: ctx.signal, + }); + + if (!res.ok) { + const body = await res.text().catch(() => ''); + throw new ProviderError('tavily', `HTTP ${res.status} ${body.slice(0, 200)}`); + } + + const data = (await res.json()) as TavilyApiResponse; + + return { + results: data.results.map((r) => ({ + url: r.url, + title: r.title, + snippet: r.content.slice(0, 300), + content: r.content, + publishedAt: r.published_date, + score: r.score, + providerRaw: r, + })), + rawLatencyMs: Math.round(performance.now() - t0), + }; + }, + }; +} diff --git a/services/mana-research/src/router/auto-route.ts b/services/mana-research/src/router/auto-route.ts new file mode 100644 index 000000000..8d52c3974 --- /dev/null +++ b/services/mana-research/src/router/auto-route.ts @@ -0,0 +1,70 @@ +/** + * Auto-router — maps QueryType + available providers to an ordered preference + * list. The first provider in the returned list that has a valid API key wins. + */ + +import type { SearchProviderId, ExtractProviderId } from '@mana/shared-research'; +import type { Config } from '../config'; +import type { QueryType } from './classify'; + +export const SEARCH_ROUTE_MAP: Record = { + news: ['tavily', 'brave', 'serper', 'searxng', 'duckduckgo'], + general: ['brave', 'tavily', 'serper', 'searxng', 'duckduckgo'], + semantic: ['exa', 'tavily', 'brave', 'searxng'], + academic: ['exa', 'searxng', 'brave', 'tavily'], + code: ['exa', 'serper', 'brave', 'searxng'], + conversational: ['tavily', 'brave', 'serper', 'searxng'], +}; + +export const EXTRACT_ROUTE_DEFAULT: ExtractProviderId[] = [ + 'firecrawl', + 'jina-reader', + 'readability', +]; + +/** + * Pick the first provider in `preferences` that has a configured key (or + * doesn't need one). Returns `null` if no provider is usable — caller should + * fall back to free providers. + */ +export function pickSearchProvider( + preferences: SearchProviderId[], + config: Config, + alwaysAvailable: Set = new Set(['searxng', 'duckduckgo']) +): SearchProviderId | null { + const envMap: Record = { + searxng: null, + duckduckgo: null, + brave: 'brave', + tavily: 'tavily', + exa: 'exa', + serper: 'serper', + }; + + for (const id of preferences) { + if (alwaysAvailable.has(id)) return id; + const envKey = envMap[id]; + if (envKey && config.providerKeys[envKey]) return id; + } + return null; +} + +export function pickExtractProvider( + preferences: ExtractProviderId[], + config: Config, + alwaysAvailable: Set = new Set(['readability', 'jina-reader']) +): ExtractProviderId | null { + const envMap: Record = { + readability: null, + 'jina-reader': 'jina', + firecrawl: 'firecrawl', + scrapingbee: 'scrapingbee', + }; + + for (const id of preferences) { + if (alwaysAvailable.has(id)) return id; + const envKey = envMap[id]; + if (envKey && config.providerKeys[envKey]) return id; + } + return null; +} diff --git a/services/mana-research/src/router/classify.ts b/services/mana-research/src/router/classify.ts new file mode 100644 index 000000000..2fa188433 --- /dev/null +++ b/services/mana-research/src/router/classify.ts @@ -0,0 +1,107 @@ +/** + * Query classifier — maps a free-text query to a QueryType. + * + * Hybrid strategy: + * 1. Regex fast-path (no network, ~0ms) catches the obvious cases + * (URLs, "paper"/"arxiv", "news"/"latest", "github"/"code", etc.) + * 2. Optional mana-llm fallback for ambiguous queries. Off by default + * — callers opt-in via `useLlm: true` when latency is OK. + */ + +import type { ManaLlmClient } from '../clients/mana-llm'; + +export type QueryType = 'news' | 'general' | 'semantic' | 'academic' | 'code' | 'conversational'; + +export interface ClassifyOptions { + useLlm?: boolean; + signal?: AbortSignal; + llm?: ManaLlmClient; +} + +const NEWS_PATTERNS = [ + /\b(latest|recent|news|heute|aktuell|today|yesterday|breaking|gerade|neueste|this week)\b/i, +]; + +const ACADEMIC_PATTERNS = [ + /\b(paper|arxiv|research|study|studie|journal|doi|citation|pubmed|nature|science)\b/i, +]; + +const CODE_PATTERNS = [ + /\b(github|code|function|library|framework|npm package|pip install|error:|exception:|stack trace)\b/i, + /[a-z_]+\([^)]*\)/i, +]; + +const CONVERSATIONAL_PATTERNS = [ + /^(how|why|what|when|where|who|can you|could you|should i|is there|erklär|explain|zusammenfass)/i, + /\?\s*$/, +]; + +const SEMANTIC_PATTERNS = [ + /\b(similar to|like this|related to|ähnlich|vergleichbar|find sites like)\b/i, + /^https?:\/\//, +]; + +export function classifyRegex(query: string): { type: QueryType; confidence: number } { + const q = query.trim(); + + for (const p of SEMANTIC_PATTERNS) if (p.test(q)) return { type: 'semantic', confidence: 0.9 }; + for (const p of ACADEMIC_PATTERNS) if (p.test(q)) return { type: 'academic', confidence: 0.8 }; + for (const p of CODE_PATTERNS) if (p.test(q)) return { type: 'code', confidence: 0.8 }; + for (const p of NEWS_PATTERNS) if (p.test(q)) return { type: 'news', confidence: 0.7 }; + for (const p of CONVERSATIONAL_PATTERNS) + if (p.test(q)) return { type: 'conversational', confidence: 0.6 }; + + return { type: 'general', confidence: 0.4 }; +} + +const LLM_PROMPT = `You are a query classifier. Given a user search query, respond with exactly one word from this list: news, general, semantic, academic, code, conversational. + +Guidelines: +- news: current events, latest updates, breaking stories +- academic: research papers, scientific literature, DOIs, arXiv +- code: programming questions, libraries, errors, GitHub +- semantic: "find similar to", URL-based, related-to queries +- conversational: open-ended questions, "how does X work" +- general: anything else + +Respond with just the label, no punctuation.`; + +export async function classify( + query: string, + opts: ClassifyOptions = {} +): Promise<{ type: QueryType; confidence: number; source: 'regex' | 'llm' }> { + const regex = classifyRegex(query); + + if (!opts.useLlm || !opts.llm || regex.confidence >= 0.8) { + return { ...regex, source: 'regex' }; + } + + try { + const { content } = await opts.llm.chat( + [ + { role: 'system', content: LLM_PROMPT }, + { role: 'user', content: query }, + ], + { maxTokens: 10, temperature: 0, signal: opts.signal } + ); + const raw = content + .trim() + .toLowerCase() + .replace(/[^a-z]/g, ''); + const valid: QueryType[] = [ + 'news', + 'general', + 'semantic', + 'academic', + 'code', + 'conversational', + ]; + if ((valid as string[]).includes(raw)) { + return { type: raw as QueryType, confidence: 0.9, source: 'llm' }; + } + } catch { + /* fall through */ + } + + return { ...regex, source: 'regex' }; +} diff --git a/services/mana-research/src/routes/extract.ts b/services/mana-research/src/routes/extract.ts new file mode 100644 index 000000000..40fcb9a21 --- /dev/null +++ b/services/mana-research/src/routes/extract.ts @@ -0,0 +1,157 @@ +/** + * POST /v1/extract — single-provider extract + * POST /v1/extract/compare — fan-out + */ + +import { Hono } from 'hono'; +import { z } from 'zod'; +import type { ExtractedContent } from '@mana/shared-research'; +import { EXTRACT_PROVIDER_IDS, extractOptionsSchema } from '@mana/shared-research'; +import type { ExecutorDeps } from '../executor/execute-extract'; +import { executeExtract } from '../executor/execute-extract'; +import type { HonoEnv } from '../lib/hono-env'; +import type { ProviderRegistry } from '../providers/registry'; +import { getExtractProvider } from '../providers/registry'; +import type { RunStorage } from '../storage/runs'; +import { BadRequestError } from '../lib/errors'; +import type { Config } from '../config'; +import { EXTRACT_ROUTE_DEFAULT, pickExtractProvider } from '../router/auto-route'; + +const MAX_COMPARE_PROVIDERS = 4; + +const extractBodySchema = z.object({ + url: z.string().url(), + provider: z.enum(EXTRACT_PROVIDER_IDS).optional(), + options: extractOptionsSchema.optional(), +}); + +const extractCompareBodySchema = z.object({ + url: z.string().url(), + providers: z.array(z.enum(EXTRACT_PROVIDER_IDS)).min(1).max(MAX_COMPARE_PROVIDERS), + options: extractOptionsSchema.optional(), +}); + +export function createExtractRoutes( + registry: ProviderRegistry, + storage: RunStorage, + deps: ExecutorDeps, + config: Config +) { + return new Hono() + .post('/', async (c) => { + const user = c.get('user'); + const body = extractBodySchema.parse(await c.req.json()); + + const providerId = + body.provider ?? pickExtractProvider(EXTRACT_ROUTE_DEFAULT, config) ?? 'readability'; + + const provider = getExtractProvider(registry, providerId); + + const run = await storage.createRun({ + userId: user.userId, + query: body.url, + mode: body.provider ? 'single' : 'auto', + category: 'extract', + providersRequested: [providerId], + billingMode: provider.requiresApiKey ? 'server-key' : 'free', + }); + + const out = await executeExtract( + { + provider, + url: body.url, + options: body.options ?? {}, + userId: user.userId, + }, + deps + ); + + await storage.addResult({ + runId: run.id, + providerId, + success: out.success, + latencyMs: out.meta.latencyMs, + costCredits: out.meta.costCredits, + billingMode: out.meta.billingMode, + cacheHit: out.meta.cacheHit, + normalizedResult: out.data ?? null, + errorCode: out.meta.errorCode ?? null, + }); + + if (out.meta.costCredits > 0) { + await storage.finalizeRunCost(run.id, out.meta.costCredits); + } + + return c.json({ + runId: run.id, + url: body.url, + provider: providerId, + success: out.success, + data: out.data, + meta: out.meta, + }); + }) + .post('/compare', async (c) => { + const user = c.get('user'); + const body = extractCompareBodySchema.parse(await c.req.json()); + + if (new Set(body.providers).size !== body.providers.length) { + throw new BadRequestError('Duplicate providers in request'); + } + + const providers = body.providers.map((id) => getExtractProvider(registry, id)); + const anyPaid = providers.some((p) => p.requiresApiKey); + + const run = await storage.createRun({ + userId: user.userId, + query: body.url, + mode: 'compare', + category: 'extract', + providersRequested: body.providers, + billingMode: anyPaid ? 'mixed' : 'free', + }); + + const settled = await Promise.all( + providers.map((provider) => + executeExtract( + { + provider, + url: body.url, + options: body.options ?? {}, + userId: user.userId, + }, + deps + ) + ) + ); + + let totalCost = 0; + for (let i = 0; i < providers.length; i++) { + const out = settled[i]; + totalCost += out.meta.costCredits; + await storage.addResult({ + runId: run.id, + providerId: providers[i].id, + success: out.success, + latencyMs: out.meta.latencyMs, + costCredits: out.meta.costCredits, + billingMode: out.meta.billingMode, + cacheHit: out.meta.cacheHit, + normalizedResult: out.data ?? null, + errorCode: out.meta.errorCode ?? null, + }); + } + if (totalCost > 0) await storage.finalizeRunCost(run.id, totalCost); + + return c.json({ + runId: run.id, + url: body.url, + results: providers.map((provider, i) => ({ + provider: provider.id, + success: settled[i].success, + data: settled[i].data as { content: ExtractedContent } | undefined, + meta: settled[i].meta, + })), + }); + }); +} diff --git a/services/mana-research/src/routes/health.ts b/services/mana-research/src/routes/health.ts new file mode 100644 index 000000000..44ccf74db --- /dev/null +++ b/services/mana-research/src/routes/health.ts @@ -0,0 +1,10 @@ +import { Hono } from 'hono'; + +export const healthRoutes = new Hono().get('/', (c) => + c.json({ + status: 'ok', + service: 'mana-research', + version: '0.1.0', + timestamp: new Date().toISOString(), + }) +); diff --git a/services/mana-research/src/routes/providers.ts b/services/mana-research/src/routes/providers.ts new file mode 100644 index 000000000..dcdd85366 --- /dev/null +++ b/services/mana-research/src/routes/providers.ts @@ -0,0 +1,77 @@ +/** + * GET /v1/providers — registered providers with capabilities + pricing + * GET /v1/providers/health — which providers are currently usable (key present) + */ + +import { Hono } from 'hono'; +import type { ProviderRegistry } from '../providers/registry'; +import { listProviders } from '../providers/registry'; +import { PROVIDER_PRICING } from '../lib/pricing'; +import type { Config } from '../config'; + +export function createProvidersRoutes(registry: ProviderRegistry, config: Config) { + return new Hono() + .get('/', (c) => { + const list = listProviders(registry); + return c.json({ + search: list.search.map((p) => ({ ...p, pricing: PROVIDER_PRICING[p.id] })), + extract: list.extract.map((p) => ({ ...p, pricing: PROVIDER_PRICING[p.id] })), + agent: list.agent, + }); + }) + .get('/health', (c) => { + const keys = config.providerKeys; + type Entry = { + id: string; + category: 'search' | 'extract' | 'agent'; + requiresApiKey: boolean; + serverKeyAvailable: boolean; + status: 'ready' | 'needs-key' | 'free'; + }; + + const check = ( + id: string, + requiresKey: boolean, + serverKeyPresent: boolean + ): Entry['status'] => { + if (!requiresKey) return 'free'; + return serverKeyPresent ? 'ready' : 'needs-key'; + }; + + const keyMap: Record = { + brave: !!keys.brave, + tavily: !!keys.tavily, + exa: !!keys.exa, + serper: !!keys.serper, + 'jina-reader': !!keys.jina, + firecrawl: !!keys.firecrawl, + scrapingbee: !!keys.scrapingbee, + }; + + const list = listProviders(registry); + const entries: Entry[] = [ + ...list.search.map((p) => ({ + id: p.id, + category: 'search' as const, + requiresApiKey: p.requiresApiKey, + serverKeyAvailable: !!keyMap[p.id], + status: check(p.id, p.requiresApiKey, !!keyMap[p.id]), + })), + ...list.extract.map((p) => ({ + id: p.id, + category: 'extract' as const, + requiresApiKey: p.requiresApiKey, + serverKeyAvailable: !!keyMap[p.id], + status: check(p.id, p.requiresApiKey, !!keyMap[p.id]), + })), + ]; + + return c.json({ + providers: entries, + summary: { + ready: entries.filter((e) => e.status === 'ready' || e.status === 'free').length, + total: entries.length, + }, + }); + }); +} diff --git a/services/mana-research/src/routes/runs.ts b/services/mana-research/src/routes/runs.ts new file mode 100644 index 000000000..9f4bea190 --- /dev/null +++ b/services/mana-research/src/routes/runs.ts @@ -0,0 +1,44 @@ +/** + * /v1/runs — user's saved eval runs + per-result rating. + */ + +import { Hono } from 'hono'; +import { z } from 'zod'; +import type { HonoEnv } from '../lib/hono-env'; +import { BadRequestError, NotFoundError } from '../lib/errors'; +import type { RunStorage } from '../storage/runs'; + +const rateSchema = z.object({ + rating: z.number().int().min(1).max(5), + notes: z.string().max(2000).optional(), +}); + +export function createRunsRoutes(storage: RunStorage) { + return new Hono() + .get('/', async (c) => { + const user = c.get('user'); + const limit = Math.min(parseInt(c.req.query('limit') || '50', 10), 200); + const offset = parseInt(c.req.query('offset') || '0', 10); + const runs = await storage.listRuns(user.userId, limit, offset); + return c.json({ runs }); + }) + .get('/:id', async (c) => { + const user = c.get('user'); + const id = c.req.param('id'); + const out = await storage.getRunWithResults(id, user.userId); + if (!out) throw new NotFoundError('Run not found'); + return c.json(out); + }) + .post('/:runId/results/:resultId/rate', async (c) => { + const user = c.get('user'); + const body = rateSchema.parse(await c.req.json()); + const ok = await storage.rateResult( + c.req.param('resultId'), + user.userId, + body.rating, + body.notes + ); + if (!ok) throw new BadRequestError('Cannot rate this result'); + return c.json({ success: true }); + }); +} diff --git a/services/mana-research/src/routes/search.ts b/services/mana-research/src/routes/search.ts new file mode 100644 index 000000000..e8d4f682a --- /dev/null +++ b/services/mana-research/src/routes/search.ts @@ -0,0 +1,178 @@ +/** + * POST /v1/search — single-provider search (or auto-routed if provider omitted) + * POST /v1/search/compare — fan-out to N providers, persist as eval_run + */ + +import { Hono } from 'hono'; +import { z } from 'zod'; +import type { ProviderId, SearchHit } from '@mana/shared-research'; +import { SEARCH_PROVIDER_IDS, searchOptionsSchema } from '@mana/shared-research'; +import type { ExecutorDeps } from '../executor/execute'; +import { executeSearch } from '../executor/execute'; +import type { HonoEnv } from '../lib/hono-env'; +import type { ProviderRegistry } from '../providers/registry'; +import { getSearchProvider } from '../providers/registry'; +import type { RunStorage } from '../storage/runs'; +import { BadRequestError } from '../lib/errors'; +import type { Config } from '../config'; +import { SEARCH_ROUTE_MAP, pickSearchProvider } from '../router/auto-route'; +import { classify } from '../router/classify'; +import type { ManaLlmClient } from '../clients/mana-llm'; + +const MAX_COMPARE_PROVIDERS = 5; + +const searchBodySchema = z.object({ + query: z.string().min(1).max(1000), + provider: z.enum(SEARCH_PROVIDER_IDS).optional(), + options: searchOptionsSchema.optional(), + useLlmClassifier: z.boolean().optional(), +}); + +const compareBodySchema = z.object({ + query: z.string().min(1).max(1000), + providers: z.array(z.enum(SEARCH_PROVIDER_IDS)).min(1).max(MAX_COMPARE_PROVIDERS), + options: searchOptionsSchema.optional(), +}); + +export function createSearchRoutes( + registry: ProviderRegistry, + storage: RunStorage, + deps: ExecutorDeps, + config: Config, + llm: ManaLlmClient +) { + return new Hono() + .post('/', async (c) => { + const user = c.get('user'); + const body = searchBodySchema.parse(await c.req.json()); + + // Auto-route when no explicit provider + let providerId: ProviderId; + let queryType: string | undefined; + let mode: 'single' | 'auto' = 'single'; + + if (body.provider) { + providerId = body.provider; + } else { + mode = 'auto'; + const cls = await classify(body.query, { + useLlm: body.useLlmClassifier, + llm, + }); + queryType = cls.type; + const picked = pickSearchProvider(SEARCH_ROUTE_MAP[cls.type], config); + providerId = picked ?? 'searxng'; + } + + const provider = getSearchProvider(registry, providerId); + + const run = await storage.createRun({ + userId: user.userId, + query: body.query, + queryType, + mode, + category: 'search', + providersRequested: [providerId], + billingMode: provider.requiresApiKey ? 'server-key' : 'free', + }); + + const out = await executeSearch( + { + provider, + query: body.query, + options: body.options ?? {}, + userId: user.userId, + }, + deps + ); + + await storage.addResult({ + runId: run.id, + providerId, + success: out.success, + latencyMs: out.meta.latencyMs, + costCredits: out.meta.costCredits, + billingMode: out.meta.billingMode, + cacheHit: out.meta.cacheHit, + normalizedResult: out.data ?? null, + errorCode: out.meta.errorCode ?? null, + }); + + if (out.meta.costCredits > 0) { + await storage.finalizeRunCost(run.id, out.meta.costCredits); + } + + return c.json({ + runId: run.id, + query: body.query, + provider: providerId, + queryType, + success: out.success, + data: out.data, + meta: out.meta, + }); + }) + .post('/compare', async (c) => { + const user = c.get('user'); + const body = compareBodySchema.parse(await c.req.json()); + + if (new Set(body.providers).size !== body.providers.length) { + throw new BadRequestError('Duplicate providers in request'); + } + + const providers = body.providers.map((id) => getSearchProvider(registry, id)); + const anyPaid = providers.some((p) => p.requiresApiKey); + + const run = await storage.createRun({ + userId: user.userId, + query: body.query, + mode: 'compare', + category: 'search', + providersRequested: body.providers, + billingMode: anyPaid ? 'mixed' : 'free', + }); + + const settled = await Promise.all( + providers.map((provider) => + executeSearch( + { + provider, + query: body.query, + options: body.options ?? {}, + userId: user.userId, + }, + deps + ) + ) + ); + + let totalCost = 0; + for (let i = 0; i < providers.length; i++) { + const out = settled[i]; + totalCost += out.meta.costCredits; + await storage.addResult({ + runId: run.id, + providerId: providers[i].id, + success: out.success, + latencyMs: out.meta.latencyMs, + costCredits: out.meta.costCredits, + billingMode: out.meta.billingMode, + cacheHit: out.meta.cacheHit, + normalizedResult: out.data ?? null, + errorCode: out.meta.errorCode ?? null, + }); + } + if (totalCost > 0) await storage.finalizeRunCost(run.id, totalCost); + + return c.json({ + runId: run.id, + query: body.query, + results: providers.map((provider, i) => ({ + provider: provider.id, + success: settled[i].success, + data: settled[i].data as { results: SearchHit[] } | undefined, + meta: settled[i].meta, + })), + }); + }); +} diff --git a/services/mana-research/src/storage/configs.ts b/services/mana-research/src/storage/configs.ts new file mode 100644 index 000000000..b0dccb125 --- /dev/null +++ b/services/mana-research/src/storage/configs.ts @@ -0,0 +1,31 @@ +/** + * Access to research.provider_configs — per-user BYO-key + budget storage. + * + * Phase 1: plaintext storage with a TODO for encryption in Phase 4 when the + * Settings UI ships. Column name is still `apiKeyEncrypted` so the schema + * doesn't need a rename later. + */ + +import { and, eq } from 'drizzle-orm'; +import type { Database } from '../db/connection'; +import { providerConfigs } from '../db/schema/research'; +import type { ProviderConfig } from '../db/schema/research'; + +export class ConfigStorage { + constructor(private db: Database) {} + + async getForUser(userId: string, providerId: string): Promise { + const [row] = await this.db + .select() + .from(providerConfigs) + .where(and(eq(providerConfigs.userId, userId), eq(providerConfigs.providerId, providerId))) + .limit(1); + return row ?? null; + } + + async decryptKey(config: ProviderConfig): Promise { + // TODO (Phase 4): AES-GCM-256 decryption via MANA_RESEARCH_KEK or mana-auth KEK wrapping. + // Phase 1: plaintext passthrough — BYO-key UX isn't built yet, so this path stays unused. + return config.apiKeyEncrypted ?? null; + } +} diff --git a/services/mana-research/src/storage/runs.ts b/services/mana-research/src/storage/runs.ts new file mode 100644 index 000000000..4fb90589b --- /dev/null +++ b/services/mana-research/src/storage/runs.ts @@ -0,0 +1,105 @@ +/** + * Persist eval runs + results to research.* tables. + */ + +import { desc, eq, sql } from 'drizzle-orm'; +import type { Database } from '../db/connection'; +import { evalResults, evalRuns, providerStats } from '../db/schema/research'; +import type { EvalRun, EvalResult, NewEvalRun, NewEvalResult } from '../db/schema/research'; + +export class RunStorage { + constructor(private db: Database) {} + + async createRun(input: NewEvalRun): Promise { + const [row] = await this.db.insert(evalRuns).values(input).returning(); + return row; + } + + async addResult(input: NewEvalResult): Promise { + const [row] = await this.db.insert(evalResults).values(input).returning(); + + // Fire-and-forget stats rollup (no error propagation) + this.bumpStats(input).catch((err) => console.warn('[storage] stats rollup failed:', err)); + + return row; + } + + async finalizeRunCost(runId: string, totalCostCredits: number): Promise { + await this.db.update(evalRuns).set({ totalCostCredits }).where(eq(evalRuns.id, runId)); + } + + async listRuns(userId: string, limit = 50, offset = 0) { + return this.db + .select() + .from(evalRuns) + .where(eq(evalRuns.userId, userId)) + .orderBy(desc(evalRuns.createdAt)) + .limit(limit) + .offset(offset); + } + + async getRunWithResults(runId: string, userId: string) { + const [run] = await this.db.select().from(evalRuns).where(eq(evalRuns.id, runId)).limit(1); + if (!run || run.userId !== userId) return null; + + const results = await this.db + .select() + .from(evalResults) + .where(eq(evalResults.runId, runId)) + .orderBy(evalResults.providerId); + + return { run, results }; + } + + async rateResult( + resultId: string, + userId: string, + rating: number, + notes?: string + ): Promise { + // Verify ownership via join + const [result] = await this.db + .select({ runUserId: evalRuns.userId, resultId: evalResults.id }) + .from(evalResults) + .innerJoin(evalRuns, eq(evalResults.runId, evalRuns.id)) + .where(eq(evalResults.id, resultId)) + .limit(1); + + if (!result || result.runUserId !== userId) return false; + + await this.db + .update(evalResults) + .set({ userRating: rating, userNotes: notes }) + .where(eq(evalResults.id, resultId)); + + return true; + } + + private async bumpStats(result: NewEvalResult): Promise { + const day = new Date().toISOString().slice(0, 10); + const success = result.success ? 1 : 0; + const error = result.success ? 0 : 1; + + await this.db + .insert(providerStats) + .values({ + providerId: result.providerId, + day, + totalCalls: 1, + totalLatencyMs: result.latencyMs, + totalCostCredits: result.costCredits ?? 0, + successCount: success, + errorCount: error, + }) + .onConflictDoUpdate({ + target: [providerStats.providerId, providerStats.day], + set: { + totalCalls: sql`${providerStats.totalCalls} + 1`, + totalLatencyMs: sql`${providerStats.totalLatencyMs} + ${result.latencyMs}`, + totalCostCredits: sql`${providerStats.totalCostCredits} + ${result.costCredits ?? 0}`, + successCount: sql`${providerStats.successCount} + ${success}`, + errorCount: sql`${providerStats.errorCount} + ${error}`, + }, + }); + } +} diff --git a/services/mana-research/tsconfig.json b/services/mana-research/tsconfig.json new file mode 100644 index 000000000..25b285f99 --- /dev/null +++ b/services/mana-research/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ESNext", + "module": "ESNext", + "moduleResolution": "bundler", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "outDir": "dist", + "rootDir": "src", + "declaration": true, + "types": ["bun-types"], + "paths": { + "@/*": ["./src/*"] + } + }, + "include": ["src/**/*.ts"] +}