diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3342ce302..dbbd00ea8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,7 +51,6 @@ jobs: outputs: mana-core-auth: ${{ steps.changes.outputs.mana-core-auth }} mana-search: ${{ steps.changes.outputs.mana-search }} - api-gateway: ${{ steps.changes.outputs.api-gateway }} manacore-web: ${{ steps.changes.outputs.manacore-web }} chat-backend: ${{ steps.changes.outputs.chat-backend }} chat-web: ${{ steps.changes.outputs.chat-web }} @@ -89,7 +88,6 @@ jobs: echo "Force rebuild all services requested" echo "mana-core-auth=true" >> $GITHUB_OUTPUT echo "mana-search=true" >> $GITHUB_OUTPUT - echo "api-gateway=true" >> $GITHUB_OUTPUT echo "manacore-web=true" >> $GITHUB_OUTPUT echo "chat-backend=true" >> $GITHUB_OUTPUT echo "chat-web=true" >> $GITHUB_OUTPUT @@ -131,7 +129,6 @@ jobs: echo "Workflow dispatch without force_build_all - building all" echo "mana-core-auth=true" >> $GITHUB_OUTPUT echo "mana-search=true" >> $GITHUB_OUTPUT - echo "api-gateway=true" >> $GITHUB_OUTPUT echo "manacore-web=true" >> $GITHUB_OUTPUT echo "chat-backend=true" >> $GITHUB_OUTPUT echo "chat-web=true" >> $GITHUB_OUTPUT @@ -202,14 +199,6 @@ jobs: echo "mana-search=false" >> $GITHUB_OUTPUT fi - # api-gateway: services/mana-api-gateway + packages/shared-nestjs-auth - GATEWAY_CHANGED=$(check_pattern "services/mana-api-gateway/|packages/shared-nestjs-auth/") - if [ "$COMMON_CHANGED" == "true" ] || [ "$GATEWAY_CHANGED" == "true" ]; then - echo "api-gateway=true" >> $GITHUB_OUTPUT - else - echo "api-gateway=false" >> $GITHUB_OUTPUT - fi - # manacore-web: apps/manacore/apps/web + shared packages MANACORE_WEB_CHANGED=$(check_pattern "apps/manacore/apps/web/|apps/manacore/packages/") if [ "$COMMON_CHANGED" == "true" ] || [ "$SHARED_AUTH_CHANGED" == "true" ] || [ "$SHARED_UI_CHANGED" == "true" ] || [ "$SHARED_WEB_CHANGED" == "true" ] || [ "$MANACORE_WEB_CHANGED" == "true" ]; then @@ -401,7 +390,6 @@ jobs: echo "|---------|------------|" >> $GITHUB_STEP_SUMMARY echo "| mana-core-auth | ${{ steps.changes.outputs.mana-core-auth }} |" >> $GITHUB_STEP_SUMMARY echo "| mana-search | ${{ steps.changes.outputs.mana-search }} |" >> $GITHUB_STEP_SUMMARY - echo "| api-gateway | ${{ steps.changes.outputs.api-gateway }} |" >> $GITHUB_STEP_SUMMARY echo "| manacore-web | ${{ steps.changes.outputs.manacore-web }} |" >> $GITHUB_STEP_SUMMARY echo "| chat-backend | ${{ steps.changes.outputs.chat-backend }} |" >> $GITHUB_STEP_SUMMARY echo "| chat-web | ${{ steps.changes.outputs.chat-web }} |" >> $GITHUB_STEP_SUMMARY @@ -546,35 +534,6 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max - build-api-gateway: - name: Build api-gateway - runs-on: ubuntu-latest - needs: detect-changes - if: needs.detect-changes.outputs.api-gateway == 'true' - steps: - - uses: actions/checkout@v4 - - uses: docker/setup-qemu-action@v3 - - uses: docker/setup-buildx-action@v3 - - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - uses: docker/metadata-action@v5 - id: meta - with: - images: ghcr.io/${{ github.repository_owner }}/api-gateway - tags: type=raw,value=latest - - uses: docker/build-push-action@v5 - with: - context: . - file: services/mana-api-gateway/Dockerfile - platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ steps.meta.outputs.tags }} - cache-from: type=gha - cache-to: type=gha,mode=max - build-manacore-web: name: Build manacore-web runs-on: ubuntu-latest diff --git a/CLAUDE.md b/CLAUDE.md index 84f5f4634..caa78a607 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -130,7 +130,8 @@ manacore-monorepo/ │ └── {game-name}/ # Individual games ├── services/ # Standalone microservices │ ├── mana-core-auth/ # Central authentication service -│ ├── mana-search/ # Central search & content extraction service +│ ├── mana-search/ # Central search & content extraction (NestJS, legacy) +│ ├── mana-search-go/ # Central search & content extraction (Go, active) │ ├── mana-crawler/ # Web crawler service │ ├── mana-llm/ # Central LLM abstraction service │ ├── mana-landing-builder/# Org landing page builder (Astro → Cloudflare Pages) @@ -932,7 +933,8 @@ Each project has its own `CLAUDE.md` with detailed information: - `apps/chat/CLAUDE.md` - Chat API endpoints, AI models - `apps/picture/CLAUDE.md` - AI image generation - `services/mana-core-auth/` - Central authentication service -- `services/mana-search/CLAUDE.md` - Search & content extraction service +- `services/mana-search/CLAUDE.md` - Search & content extraction service (NestJS, legacy) +- `services/mana-search-go/CLAUDE.md` - Search & content extraction service (Go, active) - `services/mana-crawler/CLAUDE.md` - Web crawler service - `services/mana-llm/CLAUDE.md` - Central LLM abstraction service - `services/mana-landing-builder/CLAUDE.md` - Org landing page builder service diff --git a/docker-compose.macmini.yml b/docker-compose.macmini.yml index 44404f71d..878c5e418 100644 --- a/docker-compose.macmini.yml +++ b/docker-compose.macmini.yml @@ -302,16 +302,14 @@ services: mana-search: build: context: . - dockerfile: services/mana-search/Dockerfile + dockerfile: services/mana-search-go/Dockerfile image: mana-search:local container_name: mana-core-search restart: always depends_on: searxng: condition: service_healthy - # Removed: redis - lazy connect environment: - NODE_ENV: production PORT: 3020 SEARXNG_URL: http://searxng:8080 SEARXNG_TIMEOUT: 15000 @@ -326,11 +324,11 @@ services: ports: - "3020:3020" healthcheck: - test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://127.0.0.1:3020/api/v1/health"] + test: ["CMD", "wget", "-q", "--spider", "http://localhost:3020/health"] interval: 120s timeout: 10s retries: 3 - start_period: 40s + start_period: 5s mana-media: build: diff --git a/services/mana-api-gateway/.env.example b/services/mana-api-gateway/.env.example deleted file mode 100644 index 68d98c052..000000000 --- a/services/mana-api-gateway/.env.example +++ /dev/null @@ -1,37 +0,0 @@ -# Server -PORT=3030 -NODE_ENV=development - -# Database (same as mana-core-auth) -DATABASE_URL=postgresql://manacore:devpassword@localhost:5432/manacore - -# Redis -REDIS_HOST=localhost -REDIS_PORT=6379 -REDIS_PASSWORD= - -# Backend Services -SEARCH_SERVICE_URL=http://localhost:3021 -STT_SERVICE_URL=http://localhost:3020 -TTS_SERVICE_URL=http://localhost:3022 - -# Auth Service (for JWT validation & credits) -MANA_CORE_AUTH_URL=http://localhost:3001 - -# API Key Generation -API_KEY_PREFIX_LIVE=sk_live_ -API_KEY_PREFIX_TEST=sk_test_ - -# Rate Limiting Defaults -DEFAULT_RATE_LIMIT=10 -DEFAULT_MONTHLY_CREDITS=100 - -# CORS -CORS_ORIGINS=http://localhost:3000,http://localhost:5173 - -# Development Auth Bypass -DEV_BYPASS_AUTH=true -DEV_USER_ID=00000000-0000-0000-0000-000000000000 - -# Admin Access (comma-separated user IDs) -ADMIN_USER_IDS= diff --git a/services/mana-api-gateway/.gitignore b/services/mana-api-gateway/.gitignore deleted file mode 100644 index 6684e57ab..000000000 --- a/services/mana-api-gateway/.gitignore +++ /dev/null @@ -1,31 +0,0 @@ -# Dependencies -node_modules/ - -# Build output -dist/ - -# Environment files -.env -.env.local -.env.*.local - -# IDE -.idea/ -.vscode/ -*.swp -*.swo - -# OS -.DS_Store -Thumbs.db - -# Logs -*.log -npm-debug.log* -pnpm-debug.log* - -# Test coverage -coverage/ - -# Drizzle -drizzle/meta/ diff --git a/services/mana-api-gateway/CLAUDE.md b/services/mana-api-gateway/CLAUDE.md deleted file mode 100644 index 39898d238..000000000 --- a/services/mana-api-gateway/CLAUDE.md +++ /dev/null @@ -1,287 +0,0 @@ -# Mana API Gateway - -Custom NestJS API Gateway for monetizing ManaCore services (mana-search, mana-stt, mana-tts). - -## Overview - -- **Port**: 3030 -- **Technology**: NestJS 10 + Drizzle ORM + Redis -- **Purpose**: API Key Management, Usage Tracking, Rate Limiting, Credit-based Billing - -## Architecture - -``` - ┌─────────────────────────┐ - │ API Gateway │ - │ (Port 3030) │ - │ │ - Clients ───────────>│ • API Key Validation │ - (X-API-Key Header) │ • Rate Limiting │ - │ • Usage Tracking │ - │ • Credit Deduction │ - └───────────┬─────────────┘ - │ - ┌─────────────────────┼─────────────────────┐ - │ │ │ - ▼ ▼ ▼ - ┌───────────────┐ ┌───────────────┐ ┌───────────────┐ - │ mana-search │ │ mana-stt │ │ mana-tts │ - │ (Port 3021) │ │ (Port 3020) │ │ (Port 3022) │ - └───────────────┘ └───────────────┘ └───────────────┘ -``` - -## Quick Start - -### Development - -```bash -# Install dependencies -pnpm install - -# Push database schema -pnpm db:push - -# Start in development mode -pnpm dev -``` - -### Production - -```bash -# Build -pnpm build - -# Start -pnpm start -``` - -## API Endpoints - -### Public API (with API Key) - -| Method | Endpoint | Description | Credits | -|--------|----------|-------------|---------| -| POST | `/v1/search` | Web search | 1 | -| POST | `/v1/extract` | Content extraction | 1 | -| POST | `/v1/extract/bulk` | Bulk extraction | 1 per URL | -| GET | `/v1/search/engines` | Available search engines | 0 | -| POST | `/v1/stt/transcribe` | Audio → Text | 10/min | -| GET | `/v1/stt/models` | Available STT models | 0 | -| GET | `/v1/stt/languages` | Supported languages | 0 | -| POST | `/v1/tts/synthesize` | Text → Audio | 1/1000 chars | -| GET | `/v1/tts/voices` | Available voices | 0 | -| GET | `/v1/tts/languages` | Supported languages | 0 | - -### Management API (with JWT) - -| Method | Endpoint | Description | -|--------|----------|-------------| -| POST | `/api-keys` | Create new API key | -| GET | `/api-keys` | List all API keys | -| GET | `/api-keys/:id` | Get API key details | -| PATCH | `/api-keys/:id` | Update API key | -| DELETE | `/api-keys/:id` | Delete API key | -| POST | `/api-keys/:id/regenerate` | Regenerate API key | -| GET | `/api-keys/:id/usage` | Get usage statistics | -| GET | `/api-keys/:id/usage/summary` | Get usage summary | - -### Admin API (with JWT + Admin Role) - -| Method | Endpoint | Description | -|--------|----------|-------------| -| GET | `/admin/api-keys` | List all API keys (paginated) | -| GET | `/admin/api-keys/:id` | Get any API key details | -| PATCH | `/admin/api-keys/:id` | Update any key (tier, credits, limits) | -| DELETE | `/admin/api-keys/:id` | Delete any API key | -| GET | `/admin/usage/summary` | System-wide usage stats | -| GET | `/admin/usage/top-users` | Top users by usage | - -### System - -| Method | Endpoint | Description | -|--------|----------|-------------| -| GET | `/health` | Health check | -| GET | `/metrics` | Prometheus metrics | -| GET | `/docs` | Swagger/OpenAPI documentation | - -## Pricing Tiers - -| Tier | Rate Limit | Monthly Credits | Endpoints | Price | -|------|------------|-----------------|-----------|-------| -| Free | 10 req/min | 100 | Search only | Free | -| Pro | 100 req/min | 5,000 | All | €19/month | -| Enterprise | 1,000 req/min | 50,000 | All | €99/month | - -## Credit Costs - -| Operation | Cost | -|-----------|------| -| Search | 1 credit | -| Extract | 1 credit | -| STT | 10 credits/minute | -| TTS | 1 credit/1000 chars | - -## Usage Examples - -### Create an API Key - -```bash -# First, get a JWT token from mana-core-auth -TOKEN=$(curl -s -X POST http://localhost:3001/api/v1/auth/login \ - -H "Content-Type: application/json" \ - -d '{"email": "user@example.com", "password": "password"}' | jq -r '.accessToken') - -# Create an API key -curl -X POST http://localhost:3030/api-keys \ - -H "Authorization: Bearer $TOKEN" \ - -H "Content-Type: application/json" \ - -d '{"name": "My API Key", "tier": "free"}' -``` - -### Use the API - -```bash -# Search -curl -X POST http://localhost:3030/v1/search \ - -H "X-API-Key: sk_live_xxx" \ - -H "Content-Type: application/json" \ - -d '{"query": "quantum computing"}' - -# Extract content -curl -X POST http://localhost:3030/v1/extract \ - -H "X-API-Key: sk_live_xxx" \ - -H "Content-Type: application/json" \ - -d '{"url": "https://example.com/article"}' - -# Text-to-Speech -curl -X POST http://localhost:3030/v1/tts/synthesize \ - -H "X-API-Key: sk_live_xxx" \ - -H "Content-Type: application/json" \ - -d '{"text": "Hello, world!", "voice": "en-US-1"}' \ - --output audio.mp3 - -# Speech-to-Text -curl -X POST http://localhost:3030/v1/stt/transcribe \ - -H "X-API-Key: sk_live_xxx" \ - -F "file=@audio.wav" -``` - -### Check Usage - -```bash -curl http://localhost:3030/api-keys/{id}/usage \ - -H "Authorization: Bearer $TOKEN" -``` - -## Environment Variables - -| Variable | Default | Description | -|----------|---------|-------------| -| `PORT` | 3030 | API port | -| `DATABASE_URL` | - | PostgreSQL connection URL | -| `REDIS_HOST` | localhost | Redis host | -| `REDIS_PORT` | 6379 | Redis port | -| `SEARCH_SERVICE_URL` | http://localhost:3021 | mana-search URL | -| `STT_SERVICE_URL` | http://localhost:3020 | mana-stt URL | -| `TTS_SERVICE_URL` | http://localhost:3022 | mana-tts URL | -| `MANA_CORE_AUTH_URL` | http://localhost:3001 | Auth service URL | -| `ADMIN_USER_IDS` | - | Comma-separated admin user IDs | - -## Development Commands - -```bash -# Install dependencies -pnpm install - -# Start development server -pnpm dev - -# Build for production -pnpm build - -# Start production server -pnpm start - -# Type checking -pnpm type-check - -# Linting -pnpm lint - -# Database commands -pnpm db:push # Push schema to database -pnpm db:generate # Generate migrations -pnpm db:migrate # Run migrations -pnpm db:studio # Open Drizzle Studio -``` - -## Database Schema - -The gateway uses its own schema (`api_gateway`) in the shared ManaCore database: - -- `api_gateway.api_keys` - API key storage and configuration -- `api_gateway.api_usage` - Detailed usage logs -- `api_gateway.api_usage_daily` - Aggregated daily usage for billing - -## Rate Limiting - -Rate limiting uses Redis with a sliding window algorithm: -- Each API key has a configurable rate limit (requests per minute) -- Rate limit headers are included in responses: - - `X-RateLimit-Limit` - Maximum requests per minute - - `X-RateLimit-Remaining` - Remaining requests - - `X-RateLimit-Reset` - Unix timestamp when limit resets - -## Authentication - -Two types of authentication: -1. **X-API-Key header** - For public API endpoints (`/v1/*`) -2. **Bearer JWT token** - For management endpoints (`/api-keys/*`) - -The JWT token is validated against mana-core-auth service. - -## Project Structure - -``` -services/mana-api-gateway/ -├── src/ -│ ├── main.ts # Application entry point -│ ├── app.module.ts # Root module -│ ├── config/ -│ │ ├── configuration.ts # App configuration -│ │ └── pricing.ts # Pricing tiers and credit costs -│ ├── db/ -│ │ ├── schema/ # Drizzle schemas -│ │ ├── database.module.ts # Database provider -│ │ ├── connection.ts # DB connection -│ │ └── migrate.ts # Migration script -│ ├── api-keys/ # API key management -│ ├── usage/ # Usage tracking -│ ├── proxy/ # Proxy services to backends -│ ├── guards/ # Auth, rate limit, credits guards -│ ├── common/ # Decorators, filters, interceptors -│ ├── credits/ # Credits service (mana-core-auth client) -│ ├── metrics/ # Prometheus metrics -│ └── health/ # Health check endpoint -├── drizzle.config.ts # Drizzle Kit configuration -├── package.json -├── tsconfig.json -└── Dockerfile -``` - -## Troubleshooting - -### API Key not working - -1. Check the key is valid: `curl -H "X-API-Key: $KEY" http://localhost:3030/health` -2. Check the key is active in the database -3. Check the key hasn't expired -4. Check the endpoint is allowed for the key's tier - -### Rate limit exceeded - -Wait for the `X-RateLimit-Reset` timestamp, or upgrade to a higher tier. - -### Credits exhausted - -Check usage with the management API, or wait for monthly reset. diff --git a/services/mana-api-gateway/Dockerfile b/services/mana-api-gateway/Dockerfile deleted file mode 100644 index 7c36f7a12..000000000 --- a/services/mana-api-gateway/Dockerfile +++ /dev/null @@ -1,49 +0,0 @@ -# syntax=docker/dockerfile:1 -# ================================ -# Build Stage (Monorepo-aware) -# ================================ -FROM node:20-alpine AS base -RUN corepack enable && corepack prepare pnpm@9.15.0 --activate -WORKDIR /app - -# Install dependencies -FROM base AS deps -COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ -COPY services/mana-api-gateway/package.json ./services/mana-api-gateway/ -COPY packages/shared-nestjs-auth/package.json ./packages/shared-nestjs-auth/ -COPY packages/shared-drizzle-config/package.json ./packages/shared-drizzle-config/ -RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --frozen-lockfile --filter @manacore/api-gateway... - -# Build the application -FROM base AS builder -COPY --from=deps /app/node_modules ./node_modules -COPY --from=deps /app/services/mana-api-gateway/node_modules ./services/mana-api-gateway/node_modules -COPY --from=deps /app/packages/shared-nestjs-auth/node_modules ./packages/shared-nestjs-auth/node_modules 2>/dev/null || true -COPY packages/shared-nestjs-auth ./packages/shared-nestjs-auth -COPY packages/shared-drizzle-config ./packages/shared-drizzle-config -COPY services/mana-api-gateway ./services/mana-api-gateway -WORKDIR /app/services/mana-api-gateway -RUN pnpm build - -# ================================ -# Production Stage -# ================================ -FROM node:20-alpine AS runner -RUN corepack enable && corepack prepare pnpm@9.15.0 --activate -ENV NODE_ENV=production - -# Create non-root user -RUN addgroup --system --gid 1001 nodejs -RUN adduser --system --uid 1001 nestjs -USER nestjs - -WORKDIR /app - -# Copy built application -COPY --from=builder --chown=nestjs:nodejs /app/services/mana-api-gateway/dist ./dist -COPY --from=builder --chown=nestjs:nodejs /app/services/mana-api-gateway/node_modules ./node_modules -COPY --from=builder --chown=nestjs:nodejs /app/services/mana-api-gateway/package.json ./ - -EXPOSE 3030 - -CMD ["node", "dist/main.js"] diff --git a/services/mana-api-gateway/drizzle.config.ts b/services/mana-api-gateway/drizzle.config.ts deleted file mode 100644 index f6741218a..000000000 --- a/services/mana-api-gateway/drizzle.config.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { createDrizzleConfig } from '@manacore/shared-drizzle-config'; - -export default createDrizzleConfig({ - dbName: 'manacore', - schemaFilter: ['api_gateway'], -}); diff --git a/services/mana-api-gateway/nest-cli.json b/services/mana-api-gateway/nest-cli.json deleted file mode 100644 index 95538fb90..000000000 --- a/services/mana-api-gateway/nest-cli.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "$schema": "https://json.schemastore.org/nest-cli", - "collection": "@nestjs/schematics", - "sourceRoot": "src", - "compilerOptions": { - "deleteOutDir": true - } -} diff --git a/services/mana-api-gateway/package.json b/services/mana-api-gateway/package.json deleted file mode 100644 index 94ee43ee7..000000000 --- a/services/mana-api-gateway/package.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "name": "@manacore/api-gateway", - "version": "1.0.0", - "description": "ManaCore API Gateway for monetizing core services", - "private": true, - "license": "UNLICENSED", - "scripts": { - "build": "nest build", - "dev": "nest start --watch", - "start": "node dist/main", - "start:dev": "nest start --watch", - "start:debug": "nest start --debug --watch", - "start:prod": "node dist/main", - "lint": "eslint \"{src,test}/**/*.ts\" --fix", - "type-check": "tsc --noEmit", - "test": "jest", - "test:watch": "jest --watch", - "test:cov": "jest --coverage", - "db:push": "drizzle-kit push", - "db:generate": "drizzle-kit generate", - "db:migrate": "tsx src/db/migrate.ts", - "db:studio": "drizzle-kit studio" - }, - "dependencies": { - "@manacore/shared-nestjs-auth": "workspace:*", - "@nestjs/common": "^10.4.17", - "@nestjs/config": "^3.3.0", - "@nestjs/core": "^10.4.17", - "@nestjs/platform-express": "^10.4.17", - "@nestjs/schedule": "^4.1.2", - "@nestjs/swagger": "^11.2.5", - "class-transformer": "^0.5.1", - "class-validator": "^0.14.1", - "drizzle-orm": "^0.38.4", - "ioredis": "^5.4.2", - "multer": "^1.4.5-lts.1", - "postgres": "^3.4.5", - "prom-client": "^15.1.3", - "reflect-metadata": "^0.2.2", - "rxjs": "^7.8.1" - }, - "devDependencies": { - "@manacore/shared-drizzle-config": "workspace:*", - "@nestjs/cli": "^10.4.9", - "@nestjs/schematics": "^10.2.3", - "@nestjs/testing": "^10.4.17", - "@types/express": "^5.0.0", - "@types/multer": "^1.4.12", - "@types/node": "^22.10.7", - "drizzle-kit": "^0.30.4", - "jest": "^29.7.0", - "ts-jest": "^29.2.5", - "ts-node": "^10.9.2", - "tsx": "^4.19.2", - "typescript": "^5.7.3" - }, - "engines": { - "node": ">=20.0.0", - "pnpm": ">=9.0.0" - } -} diff --git a/services/mana-api-gateway/src/admin/admin.controller.ts b/services/mana-api-gateway/src/admin/admin.controller.ts deleted file mode 100644 index 3dd871687..000000000 --- a/services/mana-api-gateway/src/admin/admin.controller.ts +++ /dev/null @@ -1,118 +0,0 @@ -import { Controller, Get, Patch, Delete, Param, Query, Body, UseGuards } from '@nestjs/common'; -import { - ApiTags, - ApiBearerAuth, - ApiOperation, - ApiResponse, - ApiParam, - ApiQuery, -} from '@nestjs/swagger'; -import { JwtAuthGuard, CurrentUser, CurrentUserData } from '@manacore/shared-nestjs-auth'; -import { AdminGuard } from '../guards/admin.guard'; -import { AdminService } from './admin.service'; -import { AdminUpdateKeyDto } from './dto/admin-update-key.dto'; - -@ApiTags('Admin') -@ApiBearerAuth('jwt') -@Controller('admin') -@UseGuards(JwtAuthGuard, AdminGuard) -export class AdminController { - constructor(private readonly adminService: AdminService) {} - - @Get('api-keys') - @ApiOperation({ - summary: 'List all API keys', - description: 'Returns all API keys in the system. Requires admin role.', - }) - @ApiQuery({ name: 'page', required: false, type: Number }) - @ApiQuery({ name: 'limit', required: false, type: Number }) - @ApiQuery({ name: 'userId', required: false, type: String }) - @ApiQuery({ name: 'tier', required: false, enum: ['free', 'pro', 'enterprise'] }) - @ApiQuery({ name: 'active', required: false, type: Boolean }) - @ApiResponse({ status: 200, description: 'List of all API keys' }) - @ApiResponse({ status: 403, description: 'Forbidden - admin role required' }) - async listAllKeys( - @Query('page') page?: string, - @Query('limit') limit?: string, - @Query('userId') userId?: string, - @Query('tier') tier?: string, - @Query('active') active?: string - ) { - const pageNum = parseInt(page || '1', 10); - const limitNum = parseInt(limit || '50', 10); - const isActive = active === undefined ? undefined : active === 'true'; - - const result = await this.adminService.listAllKeys({ - page: pageNum, - limit: limitNum, - userId, - tier, - active: isActive, - }); - - return result; - } - - @Get('api-keys/:id') - @ApiOperation({ - summary: 'Get API key details (admin)', - description: 'Returns full details of any API key including usage stats.', - }) - @ApiParam({ name: 'id', description: 'API key ID (UUID)' }) - @ApiResponse({ status: 200, description: 'API key details with usage' }) - @ApiResponse({ status: 404, description: 'API key not found' }) - async getKey(@Param('id') id: string) { - return this.adminService.getKeyDetails(id); - } - - @Patch('api-keys/:id') - @ApiOperation({ - summary: 'Update API key (admin)', - description: 'Update any API key including tier, credits, rate limits.', - }) - @ApiParam({ name: 'id', description: 'API key ID (UUID)' }) - @ApiResponse({ status: 200, description: 'API key updated' }) - @ApiResponse({ status: 404, description: 'API key not found' }) - async updateKey(@Param('id') id: string, @Body() dto: AdminUpdateKeyDto) { - return this.adminService.updateKey(id, dto); - } - - @Delete('api-keys/:id') - @ApiOperation({ - summary: 'Delete API key (admin)', - description: 'Permanently delete any API key.', - }) - @ApiParam({ name: 'id', description: 'API key ID (UUID)' }) - @ApiResponse({ status: 200, description: 'API key deleted' }) - @ApiResponse({ status: 404, description: 'API key not found' }) - async deleteKey(@Param('id') id: string) { - await this.adminService.deleteKey(id); - return { message: 'API key deleted successfully' }; - } - - @Get('usage/summary') - @ApiOperation({ - summary: 'Get system-wide usage summary', - description: 'Returns aggregated usage stats for all API keys.', - }) - @ApiQuery({ name: 'days', required: false, type: Number }) - @ApiResponse({ status: 200, description: 'System usage summary' }) - async getSystemUsage(@Query('days') days?: string) { - const daysNum = parseInt(days || '30', 10); - return this.adminService.getSystemUsage(daysNum); - } - - @Get('usage/top-users') - @ApiOperation({ - summary: 'Get top users by usage', - description: 'Returns users with highest API usage.', - }) - @ApiQuery({ name: 'limit', required: false, type: Number }) - @ApiQuery({ name: 'days', required: false, type: Number }) - @ApiResponse({ status: 200, description: 'Top users by usage' }) - async getTopUsers(@Query('limit') limit?: string, @Query('days') days?: string) { - const limitNum = parseInt(limit || '10', 10); - const daysNum = parseInt(days || '30', 10); - return this.adminService.getTopUsers(limitNum, daysNum); - } -} diff --git a/services/mana-api-gateway/src/admin/admin.module.ts b/services/mana-api-gateway/src/admin/admin.module.ts deleted file mode 100644 index 3895eb445..000000000 --- a/services/mana-api-gateway/src/admin/admin.module.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { Module } from '@nestjs/common'; -import { AdminController } from './admin.controller'; -import { AdminService } from './admin.service'; - -@Module({ - controllers: [AdminController], - providers: [AdminService], - exports: [AdminService], -}) -export class AdminModule {} diff --git a/services/mana-api-gateway/src/admin/admin.service.ts b/services/mana-api-gateway/src/admin/admin.service.ts deleted file mode 100644 index fd9c6a92c..000000000 --- a/services/mana-api-gateway/src/admin/admin.service.ts +++ /dev/null @@ -1,264 +0,0 @@ -import { Injectable, Inject, NotFoundException } from '@nestjs/common'; -import { eq, sql, desc, and, gte } from 'drizzle-orm'; -import { DATABASE_CONNECTION } from '../db/database.module'; -import { apiKeys, apiUsage, apiUsageDaily } from '../db/schema'; -import { PRICING_TIERS, PricingTier } from '../config/pricing'; -import { AdminUpdateKeyDto } from './dto/admin-update-key.dto'; - -interface ListKeysOptions { - page: number; - limit: number; - userId?: string; - tier?: string; - active?: boolean; -} - -@Injectable() -export class AdminService { - constructor( - @Inject(DATABASE_CONNECTION) - private readonly db: ReturnType - ) {} - - async listAllKeys(options: ListKeysOptions) { - const { page, limit, userId, tier, active } = options; - const offset = (page - 1) * limit; - - // Build conditions - const conditions = []; - if (userId) { - conditions.push(eq(apiKeys.userId, userId)); - } - if (tier) { - conditions.push(eq(apiKeys.tier, tier)); - } - if (active !== undefined) { - conditions.push(eq(apiKeys.active, active)); - } - - const whereClause = conditions.length > 0 ? and(...conditions) : undefined; - - const [keys, countResult] = await Promise.all([ - this.db - .select({ - id: apiKeys.id, - name: apiKeys.name, - keyPrefix: apiKeys.keyPrefix, - userId: apiKeys.userId, - organizationId: apiKeys.organizationId, - tier: apiKeys.tier, - rateLimit: apiKeys.rateLimit, - monthlyCredits: apiKeys.monthlyCredits, - creditsUsed: apiKeys.creditsUsed, - active: apiKeys.active, - lastUsedAt: apiKeys.lastUsedAt, - createdAt: apiKeys.createdAt, - }) - .from(apiKeys) - .where(whereClause) - .orderBy(desc(apiKeys.createdAt)) - .limit(limit) - .offset(offset), - this.db - .select({ count: sql`count(*)` }) - .from(apiKeys) - .where(whereClause), - ]); - - const total = Number(countResult[0]?.count || 0); - - return { - apiKeys: keys, - pagination: { - page, - limit, - total, - totalPages: Math.ceil(total / limit), - }, - }; - } - - async getKeyDetails(id: string) { - const key = await this.db.select().from(apiKeys).where(eq(apiKeys.id, id)).limit(1); - - if (!key.length) { - throw new NotFoundException('API key not found'); - } - - // Get recent usage - const thirtyDaysAgo = new Date(); - thirtyDaysAgo.setDate(thirtyDaysAgo.getDate() - 30); - - const recentUsage = await this.db - .select({ - endpoint: apiUsageDaily.endpoint, - requestCount: sql`sum(${apiUsageDaily.requestCount})`, - creditsUsed: sql`sum(${apiUsageDaily.creditsUsed})`, - }) - .from(apiUsageDaily) - .where( - and( - eq(apiUsageDaily.apiKeyId, id), - gte(apiUsageDaily.date, thirtyDaysAgo.toISOString().split('T')[0]) - ) - ) - .groupBy(apiUsageDaily.endpoint); - - return { - apiKey: { - ...key[0], - keyHash: undefined, // Don't expose hash - }, - usage: { - last30Days: recentUsage, - }, - }; - } - - async updateKey(id: string, dto: AdminUpdateKeyDto) { - // Verify key exists - const existing = await this.db.select().from(apiKeys).where(eq(apiKeys.id, id)).limit(1); - - if (!existing.length) { - throw new NotFoundException('API key not found'); - } - - // Build update object - const updates: Record = { - updatedAt: new Date(), - }; - - if (dto.name !== undefined) updates.name = dto.name; - if (dto.description !== undefined) updates.description = dto.description; - if (dto.active !== undefined) updates.active = dto.active; - if (dto.expiresAt !== undefined) updates.expiresAt = new Date(dto.expiresAt); - if (dto.rateLimit !== undefined) updates.rateLimit = dto.rateLimit; - if (dto.monthlyCredits !== undefined) updates.monthlyCredits = dto.monthlyCredits; - - if (dto.allowedEndpoints !== undefined) { - updates.allowedEndpoints = JSON.stringify(dto.allowedEndpoints); - } - - if (dto.allowedIps !== undefined) { - updates.allowedIps = JSON.stringify(dto.allowedIps); - } - - if (dto.resetCredits) { - updates.creditsUsed = 0; - } - - // If tier is changed, apply tier defaults - if (dto.tier !== undefined) { - const tierConfig = PRICING_TIERS[dto.tier as PricingTier]; - updates.tier = dto.tier; - // Only apply tier defaults if not explicitly set - if (dto.rateLimit === undefined) updates.rateLimit = tierConfig.rateLimit; - if (dto.monthlyCredits === undefined) updates.monthlyCredits = tierConfig.monthlyCredits; - } - - const [updated] = await this.db - .update(apiKeys) - .set(updates) - .where(eq(apiKeys.id, id)) - .returning(); - - return { - apiKey: { - ...updated, - keyHash: undefined, - }, - }; - } - - async deleteKey(id: string) { - const existing = await this.db.select().from(apiKeys).where(eq(apiKeys.id, id)).limit(1); - - if (!existing.length) { - throw new NotFoundException('API key not found'); - } - - // Delete usage data first (foreign key constraint) - await this.db.delete(apiUsageDaily).where(eq(apiUsageDaily.apiKeyId, id)); - await this.db.delete(apiUsage).where(eq(apiUsage.apiKeyId, id)); - - // Delete the key - await this.db.delete(apiKeys).where(eq(apiKeys.id, id)); - } - - async getSystemUsage(days: number) { - const startDate = new Date(); - startDate.setDate(startDate.getDate() - days); - - const dailyStats = await this.db - .select({ - date: apiUsageDaily.date, - requestCount: sql`sum(${apiUsageDaily.requestCount})`, - creditsUsed: sql`sum(${apiUsageDaily.creditsUsed})`, - errorCount: sql`sum(${apiUsageDaily.errorCount})`, - }) - .from(apiUsageDaily) - .where(gte(apiUsageDaily.date, startDate.toISOString().split('T')[0])) - .groupBy(apiUsageDaily.date) - .orderBy(apiUsageDaily.date); - - const endpointStats = await this.db - .select({ - endpoint: apiUsageDaily.endpoint, - requestCount: sql`sum(${apiUsageDaily.requestCount})`, - creditsUsed: sql`sum(${apiUsageDaily.creditsUsed})`, - }) - .from(apiUsageDaily) - .where(gte(apiUsageDaily.date, startDate.toISOString().split('T')[0])) - .groupBy(apiUsageDaily.endpoint); - - const tierStats = await this.db - .select({ - tier: apiKeys.tier, - keyCount: sql`count(*)`, - activeCount: sql`sum(case when ${apiKeys.active} then 1 else 0 end)`, - }) - .from(apiKeys) - .groupBy(apiKeys.tier); - - return { - period: { - start: startDate.toISOString().split('T')[0], - end: new Date().toISOString().split('T')[0], - days, - }, - daily: dailyStats, - byEndpoint: endpointStats, - byTier: tierStats, - }; - } - - async getTopUsers(limit: number, days: number) { - const startDate = new Date(); - startDate.setDate(startDate.getDate() - days); - - const topUsers = await this.db - .select({ - apiKeyId: apiUsageDaily.apiKeyId, - keyName: apiKeys.name, - userId: apiKeys.userId, - tier: apiKeys.tier, - requestCount: sql`sum(${apiUsageDaily.requestCount})`, - creditsUsed: sql`sum(${apiUsageDaily.creditsUsed})`, - }) - .from(apiUsageDaily) - .innerJoin(apiKeys, eq(apiUsageDaily.apiKeyId, apiKeys.id)) - .where(gte(apiUsageDaily.date, startDate.toISOString().split('T')[0])) - .groupBy(apiUsageDaily.apiKeyId, apiKeys.name, apiKeys.userId, apiKeys.tier) - .orderBy(desc(sql`sum(${apiUsageDaily.requestCount})`)) - .limit(limit); - - return { - period: { - start: startDate.toISOString().split('T')[0], - end: new Date().toISOString().split('T')[0], - days, - }, - topUsers, - }; - } -} diff --git a/services/mana-api-gateway/src/admin/dto/admin-update-key.dto.ts b/services/mana-api-gateway/src/admin/dto/admin-update-key.dto.ts deleted file mode 100644 index 77110049f..000000000 --- a/services/mana-api-gateway/src/admin/dto/admin-update-key.dto.ts +++ /dev/null @@ -1,100 +0,0 @@ -import { - IsString, - IsOptional, - IsBoolean, - IsArray, - IsDateString, - IsInt, - IsEnum, - Min, -} from 'class-validator'; -import { ApiPropertyOptional } from '@nestjs/swagger'; - -export class AdminUpdateKeyDto { - @ApiPropertyOptional({ - description: 'Update the display name', - example: 'Updated API Key Name', - }) - @IsString() - @IsOptional() - name?: string; - - @ApiPropertyOptional({ - description: 'Update the description', - example: 'Updated description', - }) - @IsString() - @IsOptional() - description?: string; - - @ApiPropertyOptional({ - description: 'Change the pricing tier', - enum: ['free', 'pro', 'enterprise'], - }) - @IsString() - @IsOptional() - @IsEnum(['free', 'pro', 'enterprise']) - tier?: 'free' | 'pro' | 'enterprise'; - - @ApiPropertyOptional({ - description: 'Custom rate limit (requests per minute)', - example: 100, - }) - @IsInt() - @IsOptional() - @Min(1) - rateLimit?: number; - - @ApiPropertyOptional({ - description: 'Custom monthly credits limit', - example: 5000, - }) - @IsInt() - @IsOptional() - @Min(0) - monthlyCredits?: number; - - @ApiPropertyOptional({ - description: 'Reset credits used to 0', - example: true, - }) - @IsBoolean() - @IsOptional() - resetCredits?: boolean; - - @ApiPropertyOptional({ - description: 'Update allowed endpoints', - example: ['search', 'stt', 'tts'], - type: [String], - }) - @IsArray() - @IsString({ each: true }) - @IsOptional() - allowedEndpoints?: string[]; - - @ApiPropertyOptional({ - description: 'Update IP whitelist', - example: ['192.168.1.0/24'], - type: [String], - }) - @IsArray() - @IsString({ each: true }) - @IsOptional() - allowedIps?: string[]; - - @ApiPropertyOptional({ - description: 'Enable or disable the API key', - example: true, - }) - @IsBoolean() - @IsOptional() - active?: boolean; - - @ApiPropertyOptional({ - description: 'Update expiration date (ISO 8601)', - example: '2025-12-31T23:59:59Z', - }) - @IsDateString() - @IsOptional() - expiresAt?: string; -} diff --git a/services/mana-api-gateway/src/api-keys/api-keys.controller.ts b/services/mana-api-gateway/src/api-keys/api-keys.controller.ts deleted file mode 100644 index 57cec3a88..000000000 --- a/services/mana-api-gateway/src/api-keys/api-keys.controller.ts +++ /dev/null @@ -1,160 +0,0 @@ -import { - Controller, - Get, - Post, - Patch, - Delete, - Body, - Param, - Query, - UseGuards, -} from '@nestjs/common'; -import { - ApiTags, - ApiBearerAuth, - ApiOperation, - ApiResponse, - ApiParam, - ApiQuery, -} from '@nestjs/swagger'; -import { JwtAuthGuard, CurrentUser, CurrentUserData } from '@manacore/shared-nestjs-auth'; -import { ApiKeysService } from './api-keys.service'; -import { CreateApiKeyDto, UpdateApiKeyDto } from './dto'; -import { UsageService } from '../usage/usage.service'; - -@ApiTags('API Keys') -@ApiBearerAuth('jwt') -@Controller('api-keys') -@UseGuards(JwtAuthGuard) -export class ApiKeysController { - constructor( - private readonly apiKeyService: ApiKeysService, - private readonly usageService: UsageService - ) {} - - @Post() - @ApiOperation({ - summary: 'Create API key', - description: 'Creates a new API key. The full key is only returned once - save it securely.', - }) - @ApiResponse({ status: 201, description: 'API key created successfully' }) - @ApiResponse({ status: 401, description: 'Unauthorized - invalid or missing JWT' }) - async create(@CurrentUser() user: CurrentUserData, @Body() dto: CreateApiKeyDto) { - const result = await this.apiKeyService.create(user.userId, dto); - return { - message: 'API key created successfully. Save your key - it will not be shown again.', - key: result.key, - apiKey: result.apiKey, - }; - } - - @Get() - @ApiOperation({ - summary: 'List API keys', - description: 'Returns all API keys for the authenticated user (without full key values)', - }) - @ApiResponse({ status: 200, description: 'List of API keys' }) - async list(@CurrentUser() user: CurrentUserData) { - const keys = await this.apiKeyService.listByUser(user.userId); - return { apiKeys: keys }; - } - - @Get(':id') - @ApiOperation({ - summary: 'Get API key details', - description: 'Returns details for a specific API key', - }) - @ApiParam({ name: 'id', description: 'API key ID (UUID)' }) - @ApiResponse({ status: 200, description: 'API key details' }) - @ApiResponse({ status: 404, description: 'API key not found' }) - async get(@CurrentUser() user: CurrentUserData, @Param('id') id: string) { - const key = await this.apiKeyService.getByIdAndUser(id, user.userId); - return { apiKey: key }; - } - - @Patch(':id') - @ApiOperation({ - summary: 'Update API key', - description: 'Updates name, description, allowed endpoints, IP whitelist, or active status', - }) - @ApiParam({ name: 'id', description: 'API key ID (UUID)' }) - @ApiResponse({ status: 200, description: 'API key updated successfully' }) - @ApiResponse({ status: 404, description: 'API key not found' }) - async update( - @CurrentUser() user: CurrentUserData, - @Param('id') id: string, - @Body() dto: UpdateApiKeyDto - ) { - const key = await this.apiKeyService.update(id, user.userId, dto); - return { apiKey: key }; - } - - @Delete(':id') - @ApiOperation({ - summary: 'Delete API key', - description: 'Permanently deletes an API key. This action cannot be undone.', - }) - @ApiParam({ name: 'id', description: 'API key ID (UUID)' }) - @ApiResponse({ status: 200, description: 'API key deleted successfully' }) - @ApiResponse({ status: 404, description: 'API key not found' }) - async delete(@CurrentUser() user: CurrentUserData, @Param('id') id: string) { - await this.apiKeyService.delete(id, user.userId); - return { message: 'API key deleted successfully' }; - } - - @Post(':id/regenerate') - @ApiOperation({ - summary: 'Regenerate API key', - description: - 'Generates a new key value for an existing API key. The old key immediately stops working.', - }) - @ApiParam({ name: 'id', description: 'API key ID (UUID)' }) - @ApiResponse({ status: 200, description: 'New key generated successfully' }) - @ApiResponse({ status: 404, description: 'API key not found' }) - async regenerate(@CurrentUser() user: CurrentUserData, @Param('id') id: string) { - const result = await this.apiKeyService.regenerate(id, user.userId); - return { - message: 'API key regenerated successfully. Save your new key - it will not be shown again.', - key: result.key, - apiKey: result.apiKey, - }; - } - - @Get(':id/usage') - @ApiOperation({ - summary: 'Get daily usage', - description: 'Returns daily usage statistics for an API key', - }) - @ApiParam({ name: 'id', description: 'API key ID (UUID)' }) - @ApiQuery({ name: 'days', required: false, description: 'Number of days (default: 30)' }) - @ApiResponse({ status: 200, description: 'Daily usage statistics' }) - async getUsage( - @CurrentUser() user: CurrentUserData, - @Param('id') id: string, - @Query('days') days?: string - ) { - // Verify ownership - await this.apiKeyService.getByIdAndUser(id, user.userId); - - const daysNum = parseInt(days || '30', 10); - const usage = await this.usageService.getDailyUsage(id, daysNum); - - return { usage }; - } - - @Get(':id/usage/summary') - @ApiOperation({ - summary: 'Get usage summary', - description: 'Returns aggregated usage summary for an API key', - }) - @ApiParam({ name: 'id', description: 'API key ID (UUID)' }) - @ApiResponse({ status: 200, description: 'Usage summary' }) - async getUsageSummary(@CurrentUser() user: CurrentUserData, @Param('id') id: string) { - // Verify ownership - await this.apiKeyService.getByIdAndUser(id, user.userId); - - const summary = await this.usageService.getUsageSummary(id); - - return { summary }; - } -} diff --git a/services/mana-api-gateway/src/api-keys/api-keys.module.ts b/services/mana-api-gateway/src/api-keys/api-keys.module.ts deleted file mode 100644 index 3c577b564..000000000 --- a/services/mana-api-gateway/src/api-keys/api-keys.module.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { Module, forwardRef } from '@nestjs/common'; -import { ApiKeysController } from './api-keys.controller'; -import { ApiKeysService } from './api-keys.service'; -import { UsageModule } from '../usage/usage.module'; - -@Module({ - imports: [forwardRef(() => UsageModule)], - controllers: [ApiKeysController], - providers: [ApiKeysService], - exports: [ApiKeysService], -}) -export class ApiKeysModule {} diff --git a/services/mana-api-gateway/src/api-keys/api-keys.service.ts b/services/mana-api-gateway/src/api-keys/api-keys.service.ts deleted file mode 100644 index 66b369145..000000000 --- a/services/mana-api-gateway/src/api-keys/api-keys.service.ts +++ /dev/null @@ -1,277 +0,0 @@ -import { Injectable, Inject, NotFoundException, ForbiddenException } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { eq, and } from 'drizzle-orm'; -import * as crypto from 'crypto'; -import { DATABASE_CONNECTION } from '../db/database.module'; -import { apiKeys, ApiKey, NewApiKey } from '../db/schema'; -import { CreateApiKeyDto, UpdateApiKeyDto } from './dto'; -import { PRICING_TIERS, PricingTier } from '../config/pricing'; - -export interface ApiKeyData { - id: string; - userId: string | null; - organizationId: string | null; - name: string; - tier: string; - rateLimit: number; - monthlyCredits: number; - creditsUsed: number; - allowedEndpoints: string | null; - allowedIps: string | null; - active: boolean; - expiresAt: Date | null; - lastUsedAt: Date | null; -} - -@Injectable() -export class ApiKeysService { - private readonly keyPrefixLive: string; - private readonly keyPrefixTest: string; - - constructor( - @Inject(DATABASE_CONNECTION) - private readonly db: ReturnType, - private readonly configService: ConfigService - ) { - this.keyPrefixLive = this.configService.get('apiKey.prefixLive') || 'sk_live_'; - this.keyPrefixTest = this.configService.get('apiKey.prefixTest') || 'sk_test_'; - } - - /** - * Generate a new API key - */ - private generateKey(isTest: boolean = false): { key: string; hash: string; prefix: string } { - const prefix = isTest ? this.keyPrefixTest : this.keyPrefixLive; - const randomPart = crypto.randomBytes(24).toString('base64url'); - const key = `${prefix}${randomPart}`; - const hash = crypto.createHash('sha256').update(key).digest('hex'); - return { key, hash, prefix }; - } - - /** - * Create a new API key for a user - */ - async create(userId: string, dto: CreateApiKeyDto): Promise<{ key: string; apiKey: ApiKey }> { - const { key, hash, prefix } = this.generateKey(dto.isTest); - const tier = (dto.tier || 'free') as PricingTier; - const tierConfig = PRICING_TIERS[tier]; - - const newKey: NewApiKey = { - key: key, - keyHash: hash, - keyPrefix: prefix, - userId, - name: dto.name, - description: dto.description, - tier, - rateLimit: tierConfig.rateLimit, - monthlyCredits: tierConfig.monthlyCredits, - creditsUsed: 0, - creditsResetAt: this.getNextMonthReset(), - allowedEndpoints: dto.allowedEndpoints - ? JSON.stringify(dto.allowedEndpoints) - : JSON.stringify(tierConfig.endpoints), - allowedIps: dto.allowedIps ? JSON.stringify(dto.allowedIps) : null, - active: true, - expiresAt: dto.expiresAt ? new Date(dto.expiresAt) : null, - }; - - const [created] = await this.db.insert(apiKeys).values(newKey).returning(); - - // Return the full key only on creation (it's not stored) - return { - key, - apiKey: { ...created, key: this.maskKey(key) }, - }; - } - - /** - * List all API keys for a user (keys are masked) - */ - async listByUser(userId: string): Promise { - const keys = await this.db.select().from(apiKeys).where(eq(apiKeys.userId, userId)); - - return keys.map((k) => ({ - ...k, - key: this.maskKey(k.key), - })); - } - - /** - * Get a single API key by ID (verified for user ownership) - */ - async getByIdAndUser(id: string, userId: string): Promise { - const [key] = await this.db - .select() - .from(apiKeys) - .where(and(eq(apiKeys.id, id), eq(apiKeys.userId, userId))); - - if (!key) { - throw new NotFoundException('API key not found'); - } - - return { ...key, key: this.maskKey(key.key) }; - } - - /** - * Validate an API key and return its data - */ - async validateKey(rawKey: string): Promise { - const hash = crypto.createHash('sha256').update(rawKey).digest('hex'); - - const [key] = await this.db.select().from(apiKeys).where(eq(apiKeys.keyHash, hash)); - - if (!key) { - return null; - } - - // Update last used timestamp - await this.db.update(apiKeys).set({ lastUsedAt: new Date() }).where(eq(apiKeys.id, key.id)); - - return { - id: key.id, - userId: key.userId, - organizationId: key.organizationId, - name: key.name, - tier: key.tier, - rateLimit: key.rateLimit, - monthlyCredits: key.monthlyCredits, - creditsUsed: key.creditsUsed, - allowedEndpoints: key.allowedEndpoints, - allowedIps: key.allowedIps, - active: key.active, - expiresAt: key.expiresAt, - lastUsedAt: key.lastUsedAt, - }; - } - - /** - * Update an API key - */ - async update(id: string, userId: string, dto: UpdateApiKeyDto): Promise { - // Verify ownership - await this.getByIdAndUser(id, userId); - - const updates: Partial = { - updatedAt: new Date(), - }; - - if (dto.name !== undefined) updates.name = dto.name; - if (dto.description !== undefined) updates.description = dto.description; - if (dto.allowedEndpoints !== undefined) { - updates.allowedEndpoints = JSON.stringify(dto.allowedEndpoints); - } - if (dto.allowedIps !== undefined) { - updates.allowedIps = JSON.stringify(dto.allowedIps); - } - if (dto.active !== undefined) updates.active = dto.active; - if (dto.expiresAt !== undefined) { - updates.expiresAt = dto.expiresAt ? new Date(dto.expiresAt) : null; - } - - const [updated] = await this.db - .update(apiKeys) - .set(updates) - .where(eq(apiKeys.id, id)) - .returning(); - - return { ...updated, key: this.maskKey(updated.key) }; - } - - /** - * Delete an API key - */ - async delete(id: string, userId: string): Promise { - // Verify ownership - await this.getByIdAndUser(id, userId); - - await this.db.delete(apiKeys).where(eq(apiKeys.id, id)); - } - - /** - * Regenerate an API key - */ - async regenerate(id: string, userId: string): Promise<{ key: string; apiKey: ApiKey }> { - // Verify ownership - const existing = await this.getByIdAndUser(id, userId); - const isTest = existing.keyPrefix === this.keyPrefixTest; - - const { key, hash, prefix } = this.generateKey(isTest); - - const [updated] = await this.db - .update(apiKeys) - .set({ - key, - keyHash: hash, - keyPrefix: prefix, - updatedAt: new Date(), - }) - .where(eq(apiKeys.id, id)) - .returning(); - - return { - key, - apiKey: { ...updated, key: this.maskKey(key) }, - }; - } - - /** - * Increment credits used for an API key - */ - async incrementCreditsUsed(id: string, amount: number): Promise { - const [key] = await this.db.select().from(apiKeys).where(eq(apiKeys.id, id)); - - if (key) { - // Check if we need to reset credits - if (key.creditsResetAt && new Date() > key.creditsResetAt) { - await this.db - .update(apiKeys) - .set({ - creditsUsed: amount, - creditsResetAt: this.getNextMonthReset(), - }) - .where(eq(apiKeys.id, id)); - } else { - await this.db - .update(apiKeys) - .set({ - creditsUsed: key.creditsUsed + amount, - }) - .where(eq(apiKeys.id, id)); - } - } - } - - /** - * Check if API key has enough credits - */ - async hasEnoughCredits(id: string, requiredCredits: number): Promise { - const [key] = await this.db.select().from(apiKeys).where(eq(apiKeys.id, id)); - - if (!key) return false; - - // Check if we need to reset credits - if (key.creditsResetAt && new Date() > key.creditsResetAt) { - return true; // Credits will be reset - } - - return key.creditsUsed + requiredCredits <= key.monthlyCredits; - } - - /** - * Mask an API key for display (show only prefix and last 4 chars) - */ - private maskKey(key: string): string { - if (key.length <= 12) return key; - const prefix = key.startsWith(this.keyPrefixTest) ? this.keyPrefixTest : this.keyPrefixLive; - return `${prefix}...${key.slice(-4)}`; - } - - /** - * Get the next month reset date - */ - private getNextMonthReset(): Date { - const now = new Date(); - return new Date(now.getFullYear(), now.getMonth() + 1, 1); - } -} diff --git a/services/mana-api-gateway/src/api-keys/dto/create-api-key.dto.ts b/services/mana-api-gateway/src/api-keys/dto/create-api-key.dto.ts deleted file mode 100644 index d58eae891..000000000 --- a/services/mana-api-gateway/src/api-keys/dto/create-api-key.dto.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { IsString, IsOptional, IsEnum, IsArray, IsDateString, IsBoolean } from 'class-validator'; -import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; -import { PricingTier } from '../../config/pricing'; - -export class CreateApiKeyDto { - @ApiProperty({ - description: 'Display name for the API key', - example: 'Production API Key', - }) - @IsString() - name: string; - - @ApiPropertyOptional({ - description: 'Optional description for the API key', - example: 'Used for production web application', - }) - @IsString() - @IsOptional() - description?: string; - - @ApiPropertyOptional({ - description: 'Pricing tier (determines rate limits and credits)', - enum: ['free', 'pro', 'enterprise'], - default: 'free', - }) - @IsString() - @IsOptional() - @IsEnum(['free', 'pro', 'enterprise']) - tier?: PricingTier; - - @ApiPropertyOptional({ - description: 'List of allowed endpoints (null = all endpoints allowed)', - example: ['search', 'tts'], - type: [String], - }) - @IsArray() - @IsString({ each: true }) - @IsOptional() - allowedEndpoints?: string[]; - - @ApiPropertyOptional({ - description: 'IP whitelist for this key (null = all IPs allowed)', - example: ['192.168.1.0/24', '10.0.0.1'], - type: [String], - }) - @IsArray() - @IsString({ each: true }) - @IsOptional() - allowedIps?: string[]; - - @ApiPropertyOptional({ - description: 'Expiration date for the API key (ISO 8601)', - example: '2025-12-31T23:59:59Z', - }) - @IsDateString() - @IsOptional() - expiresAt?: string; - - @ApiPropertyOptional({ - description: 'Create a test key (sk_test_ prefix) instead of live key', - default: false, - }) - @IsBoolean() - @IsOptional() - isTest?: boolean; -} diff --git a/services/mana-api-gateway/src/api-keys/dto/index.ts b/services/mana-api-gateway/src/api-keys/dto/index.ts deleted file mode 100644 index 0ef554cf7..000000000 --- a/services/mana-api-gateway/src/api-keys/dto/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './create-api-key.dto'; -export * from './update-api-key.dto'; diff --git a/services/mana-api-gateway/src/api-keys/dto/update-api-key.dto.ts b/services/mana-api-gateway/src/api-keys/dto/update-api-key.dto.ts deleted file mode 100644 index 69836a6d1..000000000 --- a/services/mana-api-gateway/src/api-keys/dto/update-api-key.dto.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { IsString, IsOptional, IsBoolean, IsArray, IsDateString } from 'class-validator'; -import { ApiPropertyOptional } from '@nestjs/swagger'; - -export class UpdateApiKeyDto { - @ApiPropertyOptional({ - description: 'Update the display name', - example: 'Updated API Key Name', - }) - @IsString() - @IsOptional() - name?: string; - - @ApiPropertyOptional({ - description: 'Update the description', - example: 'Updated description for this key', - }) - @IsString() - @IsOptional() - description?: string; - - @ApiPropertyOptional({ - description: 'Update allowed endpoints', - example: ['search', 'stt', 'tts'], - type: [String], - }) - @IsArray() - @IsString({ each: true }) - @IsOptional() - allowedEndpoints?: string[]; - - @ApiPropertyOptional({ - description: 'Update IP whitelist', - example: ['192.168.1.0/24'], - type: [String], - }) - @IsArray() - @IsString({ each: true }) - @IsOptional() - allowedIps?: string[]; - - @ApiPropertyOptional({ - description: 'Enable or disable the API key', - example: true, - }) - @IsBoolean() - @IsOptional() - active?: boolean; - - @ApiPropertyOptional({ - description: 'Update expiration date (ISO 8601)', - example: '2025-12-31T23:59:59Z', - }) - @IsDateString() - @IsOptional() - expiresAt?: string; -} diff --git a/services/mana-api-gateway/src/app.module.ts b/services/mana-api-gateway/src/app.module.ts deleted file mode 100644 index 8abdf7765..000000000 --- a/services/mana-api-gateway/src/app.module.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { Module } from '@nestjs/common'; -import { ConfigModule } from '@nestjs/config'; -import configuration from './config/configuration'; -import { DatabaseModule } from './db/database.module'; -import { HealthModule } from './health/health.module'; -import { ApiKeysModule } from './api-keys/api-keys.module'; -import { UsageModule } from './usage/usage.module'; -import { ProxyModule } from './proxy/proxy.module'; -import { CreditsModule } from './credits/credits.module'; -import { MetricsModule } from './metrics/metrics.module'; -import { SchedulerModule } from './scheduler/scheduler.module'; -import { AdminModule } from './admin/admin.module'; - -@Module({ - imports: [ - ConfigModule.forRoot({ - isGlobal: true, - load: [configuration], - }), - DatabaseModule, - HealthModule, - ApiKeysModule, - UsageModule, - ProxyModule, - CreditsModule, - MetricsModule, - SchedulerModule, - AdminModule, - ], -}) -export class AppModule {} diff --git a/services/mana-api-gateway/src/common/decorators/api-key.decorator.ts b/services/mana-api-gateway/src/common/decorators/api-key.decorator.ts deleted file mode 100644 index 973fbcba6..000000000 --- a/services/mana-api-gateway/src/common/decorators/api-key.decorator.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { createParamDecorator, ExecutionContext } from '@nestjs/common'; -import { ApiKeyData } from '../../api-keys/api-keys.service'; - -/** - * Parameter decorator to extract the validated API key data from the request. - * Must be used with ApiKeyGuard. - * - * @example - * ```typescript - * @Post('search') - * @UseGuards(ApiKeyGuard) - * search(@ApiKeyData() apiKey: ApiKeyData) { - * return { keyId: apiKey.id }; - * } - * ``` - */ -export const ApiKeyParam = createParamDecorator( - (data: unknown, ctx: ExecutionContext): ApiKeyData => { - const request = ctx.switchToHttp().getRequest(); - return request.apiKey; - } -); diff --git a/services/mana-api-gateway/src/common/filters/http-exception.filter.ts b/services/mana-api-gateway/src/common/filters/http-exception.filter.ts deleted file mode 100644 index 11e292fcc..000000000 --- a/services/mana-api-gateway/src/common/filters/http-exception.filter.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { ExceptionFilter, Catch, ArgumentsHost, HttpException, HttpStatus } from '@nestjs/common'; -import { Response } from 'express'; - -@Catch() -export class HttpExceptionFilter implements ExceptionFilter { - catch(exception: unknown, host: ArgumentsHost) { - const ctx = host.switchToHttp(); - const response = ctx.getResponse(); - - let status = HttpStatus.INTERNAL_SERVER_ERROR; - let message = 'Internal server error'; - let details: any = undefined; - - if (exception instanceof HttpException) { - status = exception.getStatus(); - const exceptionResponse = exception.getResponse(); - - if (typeof exceptionResponse === 'string') { - message = exceptionResponse; - } else if (typeof exceptionResponse === 'object') { - const resp = exceptionResponse as any; - message = resp.message || message; - details = resp; - } - } else if (exception instanceof Error) { - message = exception.message; - console.error('Unhandled exception:', exception); - } - - response.status(status).json({ - statusCode: status, - message, - timestamp: new Date().toISOString(), - ...(details && status !== HttpStatus.INTERNAL_SERVER_ERROR && { details }), - }); - } -} diff --git a/services/mana-api-gateway/src/common/interceptors/usage-tracking.interceptor.ts b/services/mana-api-gateway/src/common/interceptors/usage-tracking.interceptor.ts deleted file mode 100644 index 5826f48b9..000000000 --- a/services/mana-api-gateway/src/common/interceptors/usage-tracking.interceptor.ts +++ /dev/null @@ -1,113 +0,0 @@ -import { Injectable, NestInterceptor, ExecutionContext, CallHandler } from '@nestjs/common'; -import { Observable } from 'rxjs'; -import { tap, catchError } from 'rxjs/operators'; -import { ApiKeyData } from '../../api-keys/api-keys.service'; -import { ApiKeysService } from '../../api-keys/api-keys.service'; -import { UsageService } from '../../usage/usage.service'; -import { CreditsService } from '../../credits/credits.service'; -import { CREDIT_COSTS } from '../../config/pricing'; - -@Injectable() -export class UsageTrackingInterceptor implements NestInterceptor { - constructor( - private readonly usageService: UsageService, - private readonly creditsService: CreditsService, - private readonly apiKeysService: ApiKeysService - ) {} - - intercept(context: ExecutionContext, next: CallHandler): Observable { - const request = context.switchToHttp().getRequest(); - const response = context.switchToHttp().getResponse(); - const apiKey = request.apiKey as ApiKeyData; - const startTime = Date.now(); - - if (!apiKey) { - return next.handle(); - } - - return next.handle().pipe( - tap(async (responseBody) => { - const latencyMs = Date.now() - startTime; - const endpoint = this.extractEndpoint(request.path); - - // Calculate credits - const creditsUsed = this.calculateCredits(endpoint, request, responseBody); - - // Track usage - await this.usageService.track({ - apiKeyId: apiKey.id, - endpoint, - method: request.method, - path: request.path, - latencyMs, - statusCode: response.statusCode || 200, - creditsUsed, - metadata: { - userAgent: request.headers['user-agent'], - }, - }); - - // Increment credits used on the API key - if (creditsUsed > 0) { - await this.apiKeysService.incrementCreditsUsed(apiKey.id, creditsUsed); - } - - // Deduct credits from user account if applicable - if (apiKey.userId && creditsUsed > 0) { - try { - await this.creditsService.deduct(apiKey.userId, creditsUsed, { - appId: 'api-gateway', - description: `API: ${endpoint}`, - apiKeyId: apiKey.id, - }); - } catch (error) { - // Log but don't fail the request - console.error('Failed to deduct credits from user account:', error); - } - } - }), - catchError(async (error) => { - const latencyMs = Date.now() - startTime; - const endpoint = this.extractEndpoint(request.path); - - // Track failed requests (no credits deducted) - await this.usageService.track({ - apiKeyId: apiKey.id, - endpoint, - method: request.method, - path: request.path, - latencyMs, - statusCode: error.status || 500, - creditsUsed: 0, - metadata: { - userAgent: request.headers['user-agent'], - error: error.message, - }, - }); - - throw error; - }) - ); - } - - private extractEndpoint(path: string): string { - const match = path.match(/\/v1\/(\w+)/); - return match ? match[1] : 'unknown'; - } - - private calculateCredits(endpoint: string, request: any, response: any): number { - switch (endpoint) { - case 'search': - return CREDIT_COSTS.search; - case 'tts': - const text = request.body?.text || ''; - return Math.max(1, Math.ceil(text.length / 1000) * CREDIT_COSTS.tts.per1000Chars); - case 'stt': - // Calculate from actual audio duration if available in response - const minutes = response?.duration ? response.duration / 60 : 1; - return Math.max(1, Math.ceil(minutes) * CREDIT_COSTS.stt.perMinute); - default: - return 0; - } - } -} diff --git a/services/mana-api-gateway/src/config/configuration.ts b/services/mana-api-gateway/src/config/configuration.ts deleted file mode 100644 index 4cb12dbb7..000000000 --- a/services/mana-api-gateway/src/config/configuration.ts +++ /dev/null @@ -1,47 +0,0 @@ -export default () => ({ - port: parseInt(process.env.PORT || '3030', 10), - nodeEnv: process.env.NODE_ENV || 'development', - - database: { - url: process.env.DATABASE_URL || 'postgresql://manacore:devpassword@localhost:5432/manacore', - }, - - cors: { - origins: process.env.CORS_ORIGINS?.split(',') || [ - 'http://localhost:3000', - 'http://localhost:5173', - ], - }, - - redis: { - host: process.env.REDIS_HOST || 'localhost', - port: parseInt(process.env.REDIS_PORT || '6379', 10), - password: process.env.REDIS_PASSWORD, - keyPrefix: 'api-gateway:', - }, - - services: { - search: process.env.SEARCH_SERVICE_URL || 'http://localhost:3021', - stt: process.env.STT_SERVICE_URL || 'http://localhost:3020', - tts: process.env.TTS_SERVICE_URL || 'http://localhost:3022', - }, - - auth: { - url: process.env.MANA_CORE_AUTH_URL || 'http://localhost:3001', - }, - - apiKey: { - prefixLive: process.env.API_KEY_PREFIX_LIVE || 'sk_live_', - prefixTest: process.env.API_KEY_PREFIX_TEST || 'sk_test_', - }, - - defaults: { - rateLimit: parseInt(process.env.DEFAULT_RATE_LIMIT || '10', 10), - monthlyCredits: parseInt(process.env.DEFAULT_MONTHLY_CREDITS || '100', 10), - }, - - admin: { - // Comma-separated list of user IDs that have admin access - userIds: process.env.ADMIN_USER_IDS || '', - }, -}); diff --git a/services/mana-api-gateway/src/config/pricing.ts b/services/mana-api-gateway/src/config/pricing.ts deleted file mode 100644 index 419dc30ef..000000000 --- a/services/mana-api-gateway/src/config/pricing.ts +++ /dev/null @@ -1,41 +0,0 @@ -export const PRICING_TIERS = { - free: { - name: 'Free', - rateLimit: 10, // 10 requests/minute - monthlyCredits: 100, - endpoints: ['search'] as const, // Only search - features: [] as const, - price: 0, - }, - pro: { - name: 'Pro', - rateLimit: 100, // 100 requests/minute - monthlyCredits: 5000, - endpoints: ['search', 'stt', 'tts'] as const, - features: ['priority_support'] as const, - price: 1900, // 19 EUR in cents - }, - enterprise: { - name: 'Enterprise', - rateLimit: 1000, // 1000 requests/minute - monthlyCredits: 50000, - endpoints: ['search', 'stt', 'tts'] as const, - features: ['priority_support', 'sla', 'dedicated_support'] as const, - price: 9900, // 99 EUR in cents - }, -} as const; - -export type PricingTier = keyof typeof PRICING_TIERS; - -// Credit costs per operation -export const CREDIT_COSTS = { - search: 1, // 1 credit per search - stt: { - perMinute: 10, // 10 credits per minute of audio - }, - tts: { - per1000Chars: 1, // 1 credit per 1000 characters - }, -} as const; - -export type Endpoint = 'search' | 'stt' | 'tts'; diff --git a/services/mana-api-gateway/src/credits/credits.module.ts b/services/mana-api-gateway/src/credits/credits.module.ts deleted file mode 100644 index bb26d3817..000000000 --- a/services/mana-api-gateway/src/credits/credits.module.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { Module } from '@nestjs/common'; -import { CreditsService } from './credits.service'; - -@Module({ - providers: [CreditsService], - exports: [CreditsService], -}) -export class CreditsModule {} diff --git a/services/mana-api-gateway/src/credits/credits.service.ts b/services/mana-api-gateway/src/credits/credits.service.ts deleted file mode 100644 index 9cfe190da..000000000 --- a/services/mana-api-gateway/src/credits/credits.service.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { Injectable, HttpException, HttpStatus } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; - -export interface DeductCreditsOptions { - appId: string; - description: string; - apiKeyId?: string; - metadata?: Record; -} - -export interface CreditBalance { - balance: number; - freeCreditsRemaining: number; - dailyFreeCredits: number; -} - -@Injectable() -export class CreditsService { - private readonly authUrl: string; - - constructor(private readonly configService: ConfigService) { - this.authUrl = this.configService.get('auth.url') || 'http://localhost:3001'; - } - - /** - * Deduct credits from a user's account - */ - async deduct(userId: string, amount: number, options: DeductCreditsOptions): Promise { - const response = await fetch(`${this.authUrl}/api/v1/credits/consume`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - userId, - amount, - appId: options.appId, - description: options.description, - metadata: { - ...options.metadata, - apiKeyId: options.apiKeyId, - source: 'api-gateway', - }, - }), - }); - - if (!response.ok) { - const error = await response.text(); - console.error(`Failed to deduct credits for user ${userId}:`, error); - // Don't throw - credit deduction failure shouldn't fail the request - // The API key credits are already tracked - } - } - - /** - * Get a user's credit balance - */ - async getBalance(userId: string): Promise { - try { - const response = await fetch(`${this.authUrl}/api/v1/credits/balance/${userId}`); - - if (!response.ok) { - return null; - } - - return response.json(); - } catch (error) { - console.error(`Failed to get credit balance for user ${userId}:`, error); - return null; - } - } - - /** - * Add credits to a user's account (for testing/admin) - */ - async addCredits(userId: string, amount: number, reason: string): Promise { - const response = await fetch(`${this.authUrl}/api/v1/credits/add`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - userId, - amount, - appId: 'api-gateway', - description: reason, - type: 'bonus', - }), - }); - - if (!response.ok) { - const error = await response.text(); - throw new HttpException(`Failed to add credits: ${error}`, HttpStatus.BAD_GATEWAY); - } - } -} diff --git a/services/mana-api-gateway/src/db/connection.ts b/services/mana-api-gateway/src/db/connection.ts deleted file mode 100644 index e84b0fa08..000000000 --- a/services/mana-api-gateway/src/db/connection.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { drizzle } from 'drizzle-orm/postgres-js'; -import postgres from 'postgres'; -import * as schema from './schema'; - -let connection: ReturnType | null = null; -let db: ReturnType | null = null; - -export function getConnection(databaseUrl: string) { - if (!connection) { - connection = postgres(databaseUrl, { - max: 10, - idle_timeout: 20, - connect_timeout: 10, - }); - } - return connection; -} - -export function getDb(databaseUrl: string) { - if (!db) { - const conn = getConnection(databaseUrl); - db = drizzle(conn, { schema }); - } - return db; -} - -export async function closeConnection() { - if (connection) { - await connection.end(); - connection = null; - db = null; - } -} diff --git a/services/mana-api-gateway/src/db/database.module.ts b/services/mana-api-gateway/src/db/database.module.ts deleted file mode 100644 index ffc4b6b06..000000000 --- a/services/mana-api-gateway/src/db/database.module.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { Global, Module } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { getDb } from './connection'; - -export const DATABASE_CONNECTION = 'DATABASE_CONNECTION'; - -@Global() -@Module({ - providers: [ - { - provide: DATABASE_CONNECTION, - useFactory: (configService: ConfigService) => { - const databaseUrl = configService.get('database.url'); - if (!databaseUrl) { - throw new Error('DATABASE_URL is not configured'); - } - return getDb(databaseUrl); - }, - inject: [ConfigService], - }, - ], - exports: [DATABASE_CONNECTION], -}) -export class DatabaseModule {} diff --git a/services/mana-api-gateway/src/db/migrate.ts b/services/mana-api-gateway/src/db/migrate.ts deleted file mode 100644 index 5d69c9b19..000000000 --- a/services/mana-api-gateway/src/db/migrate.ts +++ /dev/null @@ -1,25 +0,0 @@ -import 'dotenv/config'; -import { drizzle } from 'drizzle-orm/postgres-js'; -import { migrate } from 'drizzle-orm/postgres-js/migrator'; -import postgres from 'postgres'; - -async function main() { - const databaseUrl = - process.env.DATABASE_URL || 'postgresql://manacore:devpassword@localhost:5432/manacore'; - - console.log('Connecting to database...'); - const connection = postgres(databaseUrl, { max: 1 }); - const db = drizzle(connection); - - console.log('Running migrations...'); - await migrate(db, { migrationsFolder: './src/db/migrations' }); - - console.log('Migrations complete!'); - await connection.end(); - process.exit(0); -} - -main().catch((err) => { - console.error('Migration failed:', err); - process.exit(1); -}); diff --git a/services/mana-api-gateway/src/db/schema/api-keys.schema.ts b/services/mana-api-gateway/src/db/schema/api-keys.schema.ts deleted file mode 100644 index 3ab247adc..000000000 --- a/services/mana-api-gateway/src/db/schema/api-keys.schema.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { pgSchema, uuid, text, integer, boolean, timestamp, index } from 'drizzle-orm/pg-core'; - -export const apiGatewaySchema = pgSchema('api_gateway'); - -export const apiKeys = apiGatewaySchema.table( - 'api_keys', - { - id: uuid('id').defaultRandom().primaryKey(), - - // Key identifiers - key: text('key').notNull().unique(), // sk_live_xxx or sk_test_xxx - keyHash: text('key_hash').notNull(), // SHA256 hash for lookup - keyPrefix: text('key_prefix').notNull(), // sk_live_ or sk_test_ - - // Owner (can be user or organization) - userId: text('user_id'), // B2C owner - organizationId: text('organization_id'), // B2B owner - - // Metadata - name: text('name').notNull(), // "Production API Key" - description: text('description'), - - // Tier & Limits - tier: text('tier').notNull().default('free'), // free, pro, enterprise - rateLimit: integer('rate_limit').notNull().default(10), // requests/minute - monthlyCredits: integer('monthly_credits').notNull().default(100), - creditsUsed: integer('credits_used').notNull().default(0), - creditsResetAt: timestamp('credits_reset_at', { withTimezone: true }), - - // Permissions - allowedEndpoints: text('allowed_endpoints'), // JSON array: ["search", "tts"] - allowedIps: text('allowed_ips'), // JSON array or null for any - - // Status - active: boolean('active').notNull().default(true), - expiresAt: timestamp('expires_at', { withTimezone: true }), - lastUsedAt: timestamp('last_used_at', { withTimezone: true }), - - // Timestamps - createdAt: timestamp('created_at', { withTimezone: true }).defaultNow().notNull(), - updatedAt: timestamp('updated_at', { withTimezone: true }).defaultNow().notNull(), - }, - (table) => ({ - keyHashIdx: index('api_keys_key_hash_idx').on(table.keyHash), - userIdIdx: index('api_keys_user_id_idx').on(table.userId), - organizationIdIdx: index('api_keys_organization_id_idx').on(table.organizationId), - }) -); - -export type ApiKey = typeof apiKeys.$inferSelect; -export type NewApiKey = typeof apiKeys.$inferInsert; diff --git a/services/mana-api-gateway/src/db/schema/index.ts b/services/mana-api-gateway/src/db/schema/index.ts deleted file mode 100644 index 6988544a9..000000000 --- a/services/mana-api-gateway/src/db/schema/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './api-keys.schema'; -export * from './usage.schema'; diff --git a/services/mana-api-gateway/src/db/schema/usage.schema.ts b/services/mana-api-gateway/src/db/schema/usage.schema.ts deleted file mode 100644 index 2d2bdf353..000000000 --- a/services/mana-api-gateway/src/db/schema/usage.schema.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { uuid, text, integer, timestamp, jsonb, index, unique, date } from 'drizzle-orm/pg-core'; -import { apiGatewaySchema, apiKeys } from './api-keys.schema'; - -// Detailed usage log -export const apiUsage = apiGatewaySchema.table( - 'api_usage', - { - id: uuid('id').defaultRandom().primaryKey(), - - // Key reference - apiKeyId: uuid('api_key_id') - .references(() => apiKeys.id, { onDelete: 'cascade' }) - .notNull(), - - // Request details - endpoint: text('endpoint').notNull(), // search, stt, tts - method: text('method').notNull(), // POST, GET - path: text('path').notNull(), // /v1/search - - // Metrics - requestSize: integer('request_size'), // bytes - responseSize: integer('response_size'), // bytes - latencyMs: integer('latency_ms'), // milliseconds - statusCode: integer('status_code'), // 200, 400, 500... - - // Credit calculation - creditsUsed: integer('credits_used').notNull().default(0), - creditReason: text('credit_reason'), // "1000 characters TTS" - - // Metadata - metadata: jsonb('metadata'), // additional context (user agent, etc.) - - // Timestamp - createdAt: timestamp('created_at', { withTimezone: true }).defaultNow().notNull(), - }, - (table) => ({ - apiKeyIdIdx: index('api_usage_api_key_id_idx').on(table.apiKeyId), - createdAtIdx: index('api_usage_created_at_idx').on(table.createdAt), - endpointIdx: index('api_usage_endpoint_idx').on(table.endpoint), - }) -); - -// Aggregated daily usage (for dashboard/billing) -export const apiUsageDaily = apiGatewaySchema.table( - 'api_usage_daily', - { - id: uuid('id').defaultRandom().primaryKey(), - apiKeyId: uuid('api_key_id') - .references(() => apiKeys.id, { onDelete: 'cascade' }) - .notNull(), - date: date('date').notNull(), - endpoint: text('endpoint').notNull(), - - // Aggregates - requestCount: integer('request_count').notNull().default(0), - creditsUsed: integer('credits_used').notNull().default(0), - totalLatencyMs: integer('total_latency_ms').notNull().default(0), - errorCount: integer('error_count').notNull().default(0), - }, - (table) => ({ - // Unique constraint for upsert - uniqueDaily: unique('api_usage_daily_unique').on(table.apiKeyId, table.date, table.endpoint), - dateIdx: index('api_usage_daily_date_idx').on(table.date), - }) -); - -export type ApiUsage = typeof apiUsage.$inferSelect; -export type NewApiUsage = typeof apiUsage.$inferInsert; -export type ApiUsageDaily = typeof apiUsageDaily.$inferSelect; -export type NewApiUsageDaily = typeof apiUsageDaily.$inferInsert; diff --git a/services/mana-api-gateway/src/guards/admin.guard.ts b/services/mana-api-gateway/src/guards/admin.guard.ts deleted file mode 100644 index f5768e6a0..000000000 --- a/services/mana-api-gateway/src/guards/admin.guard.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { Injectable, CanActivate, ExecutionContext, ForbiddenException } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; - -@Injectable() -export class AdminGuard implements CanActivate { - private readonly adminUserIds: string[]; - - constructor(private readonly configService: ConfigService) { - // Admin user IDs from environment variable (comma-separated) - const adminIds = this.configService.get('admin.userIds') || ''; - this.adminUserIds = adminIds - .split(',') - .map((id) => id.trim()) - .filter(Boolean); - } - - canActivate(context: ExecutionContext): boolean { - const request = context.switchToHttp().getRequest(); - const user = request.user; - - if (!user || !user.userId) { - throw new ForbiddenException('User not authenticated'); - } - - // Check if user has admin role - if (user.role === 'admin') { - return true; - } - - // Check if user ID is in the admin list - if (this.adminUserIds.includes(user.userId)) { - return true; - } - - throw new ForbiddenException('Admin access required'); - } -} diff --git a/services/mana-api-gateway/src/guards/api-key.guard.ts b/services/mana-api-gateway/src/guards/api-key.guard.ts deleted file mode 100644 index 6be87f6ef..000000000 --- a/services/mana-api-gateway/src/guards/api-key.guard.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { - Injectable, - CanActivate, - ExecutionContext, - UnauthorizedException, - ForbiddenException, -} from '@nestjs/common'; -import { ApiKeysService, ApiKeyData } from '../api-keys/api-keys.service'; - -@Injectable() -export class ApiKeyGuard implements CanActivate { - constructor(private readonly apiKeyService: ApiKeysService) {} - - async canActivate(context: ExecutionContext): Promise { - const request = context.switchToHttp().getRequest(); - const apiKey = this.extractApiKey(request); - - if (!apiKey) { - throw new UnauthorizedException('API key required. Use X-API-Key header.'); - } - - // Validate key - const keyData = await this.apiKeyService.validateKey(apiKey); - - if (!keyData) { - throw new UnauthorizedException('Invalid API key'); - } - - if (!keyData.active) { - throw new UnauthorizedException('API key is disabled'); - } - - if (keyData.expiresAt && new Date(keyData.expiresAt) < new Date()) { - throw new UnauthorizedException('API key has expired'); - } - - // Check endpoint permission - const endpoint = this.extractEndpoint(request.path); - if (!this.hasEndpointPermission(keyData, endpoint)) { - throw new ForbiddenException( - `Endpoint '${endpoint}' not allowed for this API key. Upgrade your plan to access this endpoint.` - ); - } - - // Check IP restriction - if (!this.hasIpPermission(keyData, request)) { - throw new ForbiddenException('Request from this IP address is not allowed'); - } - - // Attach key data to request for later use - request.apiKey = keyData; - - return true; - } - - private extractApiKey(request: any): string | undefined { - return request.headers['x-api-key']; - } - - private extractEndpoint(path: string): string { - // /v1/search -> search, /v1/stt/transcribe -> stt - const match = path.match(/\/v1\/(\w+)/); - return match ? match[1] : 'unknown'; - } - - private hasEndpointPermission(keyData: ApiKeyData, endpoint: string): boolean { - if (!keyData.allowedEndpoints) return true; // No restrictions - try { - const allowed = JSON.parse(keyData.allowedEndpoints) as string[]; - return allowed.includes(endpoint); - } catch { - return true; - } - } - - private hasIpPermission(keyData: ApiKeyData, request: any): boolean { - if (!keyData.allowedIps) return true; // No restrictions - - try { - const allowedIps = JSON.parse(keyData.allowedIps) as string[]; - const clientIp = - request.headers['x-forwarded-for']?.split(',')[0]?.trim() || - request.connection?.remoteAddress || - request.ip; - - return allowedIps.includes(clientIp); - } catch { - return true; - } - } -} diff --git a/services/mana-api-gateway/src/guards/credits.guard.ts b/services/mana-api-gateway/src/guards/credits.guard.ts deleted file mode 100644 index acf485b8f..000000000 --- a/services/mana-api-gateway/src/guards/credits.guard.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { - Injectable, - CanActivate, - ExecutionContext, - HttpException, - HttpStatus, -} from '@nestjs/common'; -import { ApiKeysService, ApiKeyData } from '../api-keys/api-keys.service'; -import { CREDIT_COSTS } from '../config/pricing'; - -@Injectable() -export class CreditsGuard implements CanActivate { - constructor(private readonly apiKeyService: ApiKeysService) {} - - async canActivate(context: ExecutionContext): Promise { - const request = context.switchToHttp().getRequest(); - const apiKey = request.apiKey as ApiKeyData; - - if (!apiKey) { - return true; // Let ApiKeyGuard handle missing key - } - - const endpoint = this.extractEndpoint(request.path); - const estimatedCredits = this.estimateCredits(endpoint, request); - - const hasCredits = await this.apiKeyService.hasEnoughCredits(apiKey.id, estimatedCredits); - - if (!hasCredits) { - throw new HttpException( - { - statusCode: HttpStatus.PAYMENT_REQUIRED, - message: 'Insufficient credits. Please upgrade your plan or wait for monthly reset.', - creditsRequired: estimatedCredits, - creditsUsed: apiKey.creditsUsed, - monthlyLimit: apiKey.monthlyCredits, - }, - HttpStatus.PAYMENT_REQUIRED - ); - } - - return true; - } - - private extractEndpoint(path: string): string { - const match = path.match(/\/v1\/(\w+)/); - return match ? match[1] : 'unknown'; - } - - private estimateCredits(endpoint: string, request: any): number { - switch (endpoint) { - case 'search': - return CREDIT_COSTS.search; - case 'tts': - const text = request.body?.text || ''; - return Math.max(1, Math.ceil(text.length / 1000) * CREDIT_COSTS.tts.per1000Chars); - case 'stt': - // Estimate based on file size or default to 1 minute - return CREDIT_COSTS.stt.perMinute; - default: - return 0; - } - } -} diff --git a/services/mana-api-gateway/src/guards/index.ts b/services/mana-api-gateway/src/guards/index.ts deleted file mode 100644 index 2279e9341..000000000 --- a/services/mana-api-gateway/src/guards/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './api-key.guard'; -export * from './rate-limit.guard'; -export * from './credits.guard'; diff --git a/services/mana-api-gateway/src/guards/rate-limit.guard.ts b/services/mana-api-gateway/src/guards/rate-limit.guard.ts deleted file mode 100644 index 6bc4c3990..000000000 --- a/services/mana-api-gateway/src/guards/rate-limit.guard.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { - Injectable, - CanActivate, - ExecutionContext, - HttpException, - HttpStatus, - Inject, -} from '@nestjs/common'; -import Redis from 'ioredis'; -import { ApiKeyData } from '../api-keys/api-keys.service'; - -export const REDIS_CLIENT = 'REDIS_CLIENT'; - -@Injectable() -export class RateLimitGuard implements CanActivate { - constructor(@Inject(REDIS_CLIENT) private readonly redis: Redis) {} - - async canActivate(context: ExecutionContext): Promise { - const request = context.switchToHttp().getRequest(); - const apiKey = request.apiKey as ApiKeyData; - - if (!apiKey) { - return true; // Let ApiKeyGuard handle missing key - } - - const key = `ratelimit:${apiKey.id}`; - const limit = apiKey.rateLimit; - const window = 60; // 60 seconds - - // Sliding window rate limiting using sorted set - const now = Date.now(); - const windowStart = now - window * 1000; - - // Remove old entries - await this.redis.zremrangebyscore(key, 0, windowStart); - - // Count current requests - const count = await this.redis.zcard(key); - - if (count >= limit) { - // Get the oldest entry to calculate retry-after - const oldestEntries = await this.redis.zrange(key, 0, 0, 'WITHSCORES'); - const oldestTimestamp = oldestEntries.length > 1 ? parseInt(oldestEntries[1], 10) : now; - const retryAfter = Math.ceil((oldestTimestamp + window * 1000 - now) / 1000); - - throw new HttpException( - { - statusCode: HttpStatus.TOO_MANY_REQUESTS, - message: 'Rate limit exceeded', - retryAfter, - limit, - remaining: 0, - }, - HttpStatus.TOO_MANY_REQUESTS - ); - } - - // Add current request - await this.redis.zadd(key, now, `${now}`); - await this.redis.expire(key, window); - - // Add rate limit headers to response - const response = context.switchToHttp().getResponse(); - response.setHeader('X-RateLimit-Limit', limit); - response.setHeader('X-RateLimit-Remaining', Math.max(0, limit - count - 1)); - response.setHeader('X-RateLimit-Reset', Math.ceil(now / 1000) + window); - - return true; - } -} diff --git a/services/mana-api-gateway/src/health/health.controller.ts b/services/mana-api-gateway/src/health/health.controller.ts deleted file mode 100644 index 2cde48aac..000000000 --- a/services/mana-api-gateway/src/health/health.controller.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { Controller, Get } from '@nestjs/common'; -import { ApiTags, ApiOperation, ApiResponse } from '@nestjs/swagger'; - -@ApiTags('System') -@Controller('health') -export class HealthController { - @Get() - @ApiOperation({ - summary: 'Health check', - description: 'Returns service health status. No authentication required.', - }) - @ApiResponse({ - status: 200, - description: 'Service is healthy', - schema: { - type: 'object', - properties: { - status: { type: 'string', example: 'ok' }, - service: { type: 'string', example: 'api-gateway' }, - timestamp: { type: 'string', example: '2025-01-29T10:30:00.000Z' }, - }, - }, - }) - check() { - return { - status: 'ok', - service: 'api-gateway', - timestamp: new Date().toISOString(), - }; - } -} diff --git a/services/mana-api-gateway/src/health/health.module.ts b/services/mana-api-gateway/src/health/health.module.ts deleted file mode 100644 index a61d8b044..000000000 --- a/services/mana-api-gateway/src/health/health.module.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { Module } from '@nestjs/common'; -import { HealthController } from './health.controller'; - -@Module({ - controllers: [HealthController], -}) -export class HealthModule {} diff --git a/services/mana-api-gateway/src/main.ts b/services/mana-api-gateway/src/main.ts deleted file mode 100644 index 062254cac..000000000 --- a/services/mana-api-gateway/src/main.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { NestFactory } from '@nestjs/core'; -import { ValidationPipe } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger'; -import { AppModule } from './app.module'; -import { HttpExceptionFilter } from './common/filters/http-exception.filter'; - -async function bootstrap() { - const app = await NestFactory.create(AppModule); - - const configService = app.get(ConfigService); - const port = configService.get('port') || 3030; - const corsOrigins = configService.get('cors.origins') || []; - - // Enable CORS - app.enableCors({ - origin: corsOrigins, - credentials: true, - }); - - // Global validation pipe - app.useGlobalPipes( - new ValidationPipe({ - whitelist: true, - transform: true, - forbidNonWhitelisted: true, - }) - ); - - // Global exception filter - app.useGlobalFilters(new HttpExceptionFilter()); - - // Swagger/OpenAPI documentation - const config = new DocumentBuilder() - .setTitle('ManaCore API Gateway') - .setDescription( - 'API Gateway for ManaCore services (Search, STT, TTS). ' + - 'Use X-API-Key header for public endpoints (/v1/*) and Bearer JWT for management endpoints (/api-keys/*).' - ) - .setVersion('1.0') - .addApiKey( - { - type: 'apiKey', - name: 'X-API-Key', - in: 'header', - description: 'API Key for accessing public endpoints', - }, - 'api-key' - ) - .addBearerAuth( - { - type: 'http', - scheme: 'bearer', - bearerFormat: 'JWT', - description: 'JWT token from mana-core-auth for management endpoints', - }, - 'jwt' - ) - .addTag('Search', 'Web search and content extraction') - .addTag('STT', 'Speech-to-Text transcription') - .addTag('TTS', 'Text-to-Speech synthesis') - .addTag('API Keys', 'API key management (requires JWT authentication)') - .addTag('System', 'Health checks and metrics') - .build(); - - const document = SwaggerModule.createDocument(app, config); - SwaggerModule.setup('docs', app, document, { - swaggerOptions: { - persistAuthorization: true, - }, - }); - - await app.listen(port); - console.log(`API Gateway running on port ${port}`); - console.log(`Swagger docs available at http://localhost:${port}/docs`); -} - -bootstrap(); diff --git a/services/mana-api-gateway/src/metrics/metrics.controller.ts b/services/mana-api-gateway/src/metrics/metrics.controller.ts deleted file mode 100644 index dcb015220..000000000 --- a/services/mana-api-gateway/src/metrics/metrics.controller.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { Controller, Get, Res } from '@nestjs/common'; -import { ApiTags, ApiOperation, ApiResponse, ApiProduces } from '@nestjs/swagger'; -import { Response } from 'express'; -import { MetricsService } from './metrics.service'; - -@ApiTags('System') -@Controller('metrics') -export class MetricsController { - constructor(private readonly metricsService: MetricsService) {} - - @Get() - @ApiOperation({ - summary: 'Prometheus metrics', - description: 'Returns Prometheus-formatted metrics for monitoring. No authentication required.', - }) - @ApiProduces('text/plain') - @ApiResponse({ - status: 200, - description: 'Prometheus metrics in text format', - }) - async getMetrics(@Res() res: Response) { - const metrics = await this.metricsService.getMetrics(); - res.setHeader('Content-Type', this.metricsService.getContentType()); - res.send(metrics); - } -} diff --git a/services/mana-api-gateway/src/metrics/metrics.module.ts b/services/mana-api-gateway/src/metrics/metrics.module.ts deleted file mode 100644 index d52db9f12..000000000 --- a/services/mana-api-gateway/src/metrics/metrics.module.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { Global, Module } from '@nestjs/common'; -import { MetricsController } from './metrics.controller'; -import { MetricsService } from './metrics.service'; - -@Global() -@Module({ - controllers: [MetricsController], - providers: [MetricsService], - exports: [MetricsService], -}) -export class MetricsModule {} diff --git a/services/mana-api-gateway/src/metrics/metrics.service.ts b/services/mana-api-gateway/src/metrics/metrics.service.ts deleted file mode 100644 index 7cd98aee9..000000000 --- a/services/mana-api-gateway/src/metrics/metrics.service.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { Injectable, OnModuleInit } from '@nestjs/common'; -import * as client from 'prom-client'; - -@Injectable() -export class MetricsService implements OnModuleInit { - private readonly register: client.Registry; - - // Counters - public readonly requestsTotal: client.Counter; - public readonly creditsUsedTotal: client.Counter; - public readonly errorsTotal: client.Counter; - - // Histograms - public readonly requestDuration: client.Histogram; - - // Gauges - public readonly activeApiKeys: client.Gauge; - public readonly rateLimitExceeded: client.Counter; - - constructor() { - this.register = new client.Registry(); - - // Add default metrics - client.collectDefaultMetrics({ register: this.register }); - - // Custom metrics - this.requestsTotal = new client.Counter({ - name: 'api_gateway_requests_total', - help: 'Total number of API requests', - labelNames: ['endpoint', 'method', 'status', 'tier'], - registers: [this.register], - }); - - this.creditsUsedTotal = new client.Counter({ - name: 'api_gateway_credits_used_total', - help: 'Total credits consumed', - labelNames: ['endpoint', 'tier'], - registers: [this.register], - }); - - this.errorsTotal = new client.Counter({ - name: 'api_gateway_errors_total', - help: 'Total number of errors', - labelNames: ['endpoint', 'error_type'], - registers: [this.register], - }); - - this.requestDuration = new client.Histogram({ - name: 'api_gateway_request_duration_seconds', - help: 'Request duration in seconds', - labelNames: ['endpoint', 'method'], - buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10], - registers: [this.register], - }); - - this.activeApiKeys = new client.Gauge({ - name: 'api_gateway_active_api_keys', - help: 'Number of active API keys', - labelNames: ['tier'], - registers: [this.register], - }); - - this.rateLimitExceeded = new client.Counter({ - name: 'api_gateway_rate_limit_exceeded_total', - help: 'Total number of rate limit exceeded events', - labelNames: ['tier'], - registers: [this.register], - }); - } - - onModuleInit() { - // Initial setup if needed - } - - async getMetrics(): Promise { - return this.register.metrics(); - } - - getContentType(): string { - return this.register.contentType; - } -} diff --git a/services/mana-api-gateway/src/proxy/proxy.controller.ts b/services/mana-api-gateway/src/proxy/proxy.controller.ts deleted file mode 100644 index e3bb68ea8..000000000 --- a/services/mana-api-gateway/src/proxy/proxy.controller.ts +++ /dev/null @@ -1,275 +0,0 @@ -import { - Controller, - Post, - Get, - Body, - UseGuards, - UseInterceptors, - UploadedFile, - Res, - Query, -} from '@nestjs/common'; -import { - ApiTags, - ApiSecurity, - ApiOperation, - ApiResponse, - ApiConsumes, - ApiBody, - ApiHeader, -} from '@nestjs/swagger'; -import { FileInterceptor } from '@nestjs/platform-express'; -import { Response } from 'express'; -import { ApiKeyGuard } from '../guards/api-key.guard'; -import { RateLimitGuard } from '../guards/rate-limit.guard'; -import { CreditsGuard } from '../guards/credits.guard'; -import { UsageTrackingInterceptor } from '../common/interceptors/usage-tracking.interceptor'; -import { ApiKeyParam } from '../common/decorators/api-key.decorator'; -import { ApiKeyData } from '../api-keys/api-keys.service'; -import { - SearchProxyService, - SearchRequestDto, - ExtractRequestDto, - BulkExtractRequestDto, -} from './services/search-proxy.service'; -import { SttProxyService, TranscribeRequestDto } from './services/stt-proxy.service'; -import { TtsProxyService, SynthesizeRequestDto } from './services/tts-proxy.service'; - -@ApiSecurity('api-key') -@ApiHeader({ - name: 'X-API-Key', - description: 'Your API key (sk_live_xxx or sk_test_xxx)', - required: true, -}) -@Controller('v1') -@UseGuards(ApiKeyGuard, RateLimitGuard, CreditsGuard) -@UseInterceptors(UsageTrackingInterceptor) -export class ProxyController { - constructor( - private readonly searchProxy: SearchProxyService, - private readonly sttProxy: SttProxyService, - private readonly ttsProxy: TtsProxyService - ) {} - - // === SEARCH === - - @Post('search') - @ApiTags('Search') - @ApiOperation({ - summary: 'Web search', - description: 'Search the web using multiple search engines. Costs 1 credit per request.', - }) - @ApiBody({ - schema: { - type: 'object', - required: ['query'], - properties: { - query: { type: 'string', example: 'quantum computing' }, - options: { - type: 'object', - properties: { - categories: { - type: 'array', - items: { type: 'string' }, - example: ['general', 'science'], - }, - engines: { type: 'array', items: { type: 'string' }, example: ['google', 'bing'] }, - language: { type: 'string', example: 'en' }, - limit: { type: 'number', example: 10 }, - }, - }, - }, - }, - }) - @ApiResponse({ status: 200, description: 'Search results' }) - @ApiResponse({ status: 429, description: 'Rate limit exceeded' }) - @ApiResponse({ status: 402, description: 'Insufficient credits' }) - async search(@Body() body: SearchRequestDto, @ApiKeyParam() apiKey: ApiKeyData) { - return this.searchProxy.search(body); - } - - @Get('search/engines') - @ApiTags('Search') - @ApiOperation({ - summary: 'Get available search engines', - description: 'Returns a list of available search engines and categories. Free endpoint.', - }) - @ApiResponse({ status: 200, description: 'List of search engines' }) - async getEngines() { - return this.searchProxy.getEngines(); - } - - @Post('extract') - @ApiTags('Search') - @ApiOperation({ - summary: 'Extract content from URL', - description: 'Extracts main content from a webpage. Costs 1 credit per request.', - }) - @ApiBody({ - schema: { - type: 'object', - required: ['url'], - properties: { - url: { type: 'string', example: 'https://example.com/article' }, - options: { - type: 'object', - properties: { - includeMarkdown: { type: 'boolean', example: true }, - maxLength: { type: 'number', example: 5000 }, - }, - }, - }, - }, - }) - @ApiResponse({ status: 200, description: 'Extracted content' }) - async extract(@Body() body: ExtractRequestDto) { - return this.searchProxy.extract(body); - } - - @Post('extract/bulk') - @ApiTags('Search') - @ApiOperation({ - summary: 'Bulk extract content', - description: 'Extracts content from multiple URLs (max 20). Costs 1 credit per URL.', - }) - @ApiBody({ - schema: { - type: 'object', - required: ['urls'], - properties: { - urls: { - type: 'array', - items: { type: 'string' }, - example: ['https://example.com/article1', 'https://example.com/article2'], - }, - options: { - type: 'object', - properties: { - includeMarkdown: { type: 'boolean', example: true }, - maxLength: { type: 'number', example: 5000 }, - }, - }, - concurrency: { type: 'number', example: 5 }, - }, - }, - }) - @ApiResponse({ status: 200, description: 'Extracted content from all URLs' }) - async bulkExtract(@Body() body: BulkExtractRequestDto) { - return this.searchProxy.bulkExtract(body); - } - - // === STT === - - @Post('stt/transcribe') - @ApiTags('STT') - @ApiOperation({ - summary: 'Transcribe audio to text', - description: - 'Converts audio file to text. Costs 10 credits per minute of audio. Supports WAV, MP3, OGG, FLAC.', - }) - @ApiConsumes('multipart/form-data') - @ApiBody({ - schema: { - type: 'object', - required: ['file'], - properties: { - file: { type: 'string', format: 'binary', description: 'Audio file' }, - language: { type: 'string', example: 'en', description: 'Language code (optional)' }, - model: { type: 'string', example: 'base', description: 'Model name (optional)' }, - }, - }, - }) - @ApiResponse({ status: 200, description: 'Transcription result' }) - @ApiResponse({ status: 400, description: 'Invalid audio file' }) - @UseInterceptors(FileInterceptor('file')) - async transcribe(@UploadedFile() file: Express.Multer.File, @Body() body: TranscribeRequestDto) { - return this.sttProxy.transcribe(file, body); - } - - @Get('stt/models') - @ApiTags('STT') - @ApiOperation({ - summary: 'Get available STT models', - description: 'Returns a list of available speech-to-text models. Free endpoint.', - }) - @ApiResponse({ status: 200, description: 'List of STT models' }) - async getSttModels() { - return this.sttProxy.getModels(); - } - - @Get('stt/languages') - @ApiTags('STT') - @ApiOperation({ - summary: 'Get supported languages', - description: 'Returns a list of languages supported by STT. Free endpoint.', - }) - @ApiResponse({ status: 200, description: 'List of supported languages' }) - async getSttLanguages() { - return this.sttProxy.getLanguages(); - } - - // === TTS === - - @Post('tts/synthesize') - @ApiTags('TTS') - @ApiOperation({ - summary: 'Synthesize text to speech', - description: - 'Converts text to audio. Costs 1 credit per 1000 characters. Returns audio file (MP3, WAV, or OGG).', - }) - @ApiBody({ - schema: { - type: 'object', - required: ['text'], - properties: { - text: { type: 'string', example: 'Hello, world! This is a test.' }, - voice: { type: 'string', example: 'en-US-1', description: 'Voice ID' }, - language: { type: 'string', example: 'en-US', description: 'Language code' }, - speed: { type: 'number', example: 1.0, minimum: 0.5, maximum: 2.0 }, - format: { type: 'string', enum: ['mp3', 'wav', 'ogg'], default: 'mp3' }, - }, - }, - }) - @ApiResponse({ - status: 200, - description: 'Audio file', - content: { - 'audio/mpeg': { schema: { type: 'string', format: 'binary' } }, - 'audio/wav': { schema: { type: 'string', format: 'binary' } }, - 'audio/ogg': { schema: { type: 'string', format: 'binary' } }, - }, - }) - async synthesize(@Body() body: SynthesizeRequestDto, @Res() res: Response) { - const audio = await this.ttsProxy.synthesize(body); - - const format = body.format || 'mp3'; - const contentType = - format === 'wav' ? 'audio/wav' : format === 'ogg' ? 'audio/ogg' : 'audio/mpeg'; - - res.setHeader('Content-Type', contentType); - res.setHeader('Content-Length', audio.length); - res.send(audio); - } - - @Get('tts/voices') - @ApiTags('TTS') - @ApiOperation({ - summary: 'Get available voices', - description: 'Returns a list of available TTS voices. Free endpoint.', - }) - @ApiResponse({ status: 200, description: 'List of available voices' }) - async getTtsVoices() { - return this.ttsProxy.getVoices(); - } - - @Get('tts/languages') - @ApiTags('TTS') - @ApiOperation({ - summary: 'Get supported TTS languages', - description: 'Returns a list of languages supported by TTS. Free endpoint.', - }) - @ApiResponse({ status: 200, description: 'List of supported languages' }) - async getTtsLanguages() { - return this.ttsProxy.getLanguages(); - } -} diff --git a/services/mana-api-gateway/src/proxy/proxy.module.ts b/services/mana-api-gateway/src/proxy/proxy.module.ts deleted file mode 100644 index 0c53a77dc..000000000 --- a/services/mana-api-gateway/src/proxy/proxy.module.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { Module } from '@nestjs/common'; -import { MulterModule } from '@nestjs/platform-express'; -import { memoryStorage } from 'multer'; -import { ConfigService } from '@nestjs/config'; -import Redis from 'ioredis'; -import { ProxyController } from './proxy.controller'; -import { SearchProxyService, SttProxyService, TtsProxyService } from './services'; -import { ApiKeysModule } from '../api-keys/api-keys.module'; -import { UsageModule } from '../usage/usage.module'; -import { CreditsModule } from '../credits/credits.module'; -import { ApiKeyGuard } from '../guards/api-key.guard'; -import { RateLimitGuard, REDIS_CLIENT } from '../guards/rate-limit.guard'; -import { CreditsGuard } from '../guards/credits.guard'; -import { UsageTrackingInterceptor } from '../common/interceptors/usage-tracking.interceptor'; - -@Module({ - imports: [ - MulterModule.register({ - storage: memoryStorage(), - limits: { - fileSize: 100 * 1024 * 1024, // 100MB max file size - }, - }), - ApiKeysModule, - UsageModule, - CreditsModule, - ], - controllers: [ProxyController], - providers: [ - SearchProxyService, - SttProxyService, - TtsProxyService, - ApiKeyGuard, - RateLimitGuard, - CreditsGuard, - UsageTrackingInterceptor, - { - provide: REDIS_CLIENT, - useFactory: (configService: ConfigService) => { - const host = configService.get('redis.host') || 'localhost'; - const port = configService.get('redis.port') || 6379; - const password = configService.get('redis.password'); - - return new Redis({ - host, - port, - password: password || undefined, - keyPrefix: configService.get('redis.keyPrefix') || 'api-gateway:', - }); - }, - inject: [ConfigService], - }, - ], - exports: [REDIS_CLIENT], -}) -export class ProxyModule {} diff --git a/services/mana-api-gateway/src/proxy/services/index.ts b/services/mana-api-gateway/src/proxy/services/index.ts deleted file mode 100644 index de4b49657..000000000 --- a/services/mana-api-gateway/src/proxy/services/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './search-proxy.service'; -export * from './stt-proxy.service'; -export * from './tts-proxy.service'; diff --git a/services/mana-api-gateway/src/proxy/services/search-proxy.service.ts b/services/mana-api-gateway/src/proxy/services/search-proxy.service.ts deleted file mode 100644 index d4f52afe2..000000000 --- a/services/mana-api-gateway/src/proxy/services/search-proxy.service.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { Injectable, HttpException, HttpStatus } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; - -export interface SearchRequestDto { - query: string; - options?: { - categories?: string[]; - engines?: string[]; - language?: string; - limit?: number; - }; -} - -export interface ExtractRequestDto { - url: string; - options?: { - includeMarkdown?: boolean; - maxLength?: number; - }; -} - -export interface BulkExtractRequestDto { - urls: string[]; - options?: { - includeMarkdown?: boolean; - maxLength?: number; - }; - concurrency?: number; -} - -@Injectable() -export class SearchProxyService { - private readonly searchUrl: string; - - constructor(private readonly configService: ConfigService) { - this.searchUrl = this.configService.get('services.search') || 'http://localhost:3021'; - } - - async search(body: SearchRequestDto): Promise { - const response = await fetch(`${this.searchUrl}/api/v1/search`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(body), - }); - - if (!response.ok) { - const error = await response.text(); - throw new HttpException( - `Search service error: ${error}`, - response.status || HttpStatus.BAD_GATEWAY - ); - } - - return response.json(); - } - - async extract(body: ExtractRequestDto): Promise { - const response = await fetch(`${this.searchUrl}/api/v1/extract`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(body), - }); - - if (!response.ok) { - const error = await response.text(); - throw new HttpException( - `Extract service error: ${error}`, - response.status || HttpStatus.BAD_GATEWAY - ); - } - - return response.json(); - } - - async bulkExtract(body: BulkExtractRequestDto): Promise { - const response = await fetch(`${this.searchUrl}/api/v1/extract/bulk`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(body), - }); - - if (!response.ok) { - const error = await response.text(); - throw new HttpException( - `Bulk extract service error: ${error}`, - response.status || HttpStatus.BAD_GATEWAY - ); - } - - return response.json(); - } - - async getEngines(): Promise { - const response = await fetch(`${this.searchUrl}/api/v1/search/engines`); - - if (!response.ok) { - throw new HttpException('Failed to get search engines', HttpStatus.BAD_GATEWAY); - } - - return response.json(); - } -} diff --git a/services/mana-api-gateway/src/proxy/services/stt-proxy.service.ts b/services/mana-api-gateway/src/proxy/services/stt-proxy.service.ts deleted file mode 100644 index 502ff43c2..000000000 --- a/services/mana-api-gateway/src/proxy/services/stt-proxy.service.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { Injectable, HttpException, HttpStatus } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; - -export interface TranscribeRequestDto { - language?: string; - model?: string; -} - -@Injectable() -export class SttProxyService { - private readonly sttUrl: string; - - constructor(private readonly configService: ConfigService) { - this.sttUrl = this.configService.get('services.stt') || 'http://localhost:3020'; - } - - async transcribe(file: Express.Multer.File, options: TranscribeRequestDto): Promise { - const formData = new FormData(); - const uint8Array = new Uint8Array(file.buffer); - formData.append('file', new Blob([uint8Array], { type: file.mimetype }), file.originalname); - - if (options.language) { - formData.append('language', options.language); - } - if (options.model) { - formData.append('model', options.model); - } - - const response = await fetch(`${this.sttUrl}/api/v1/transcribe`, { - method: 'POST', - body: formData, - }); - - if (!response.ok) { - const error = await response.text(); - throw new HttpException( - `STT service error: ${error}`, - response.status || HttpStatus.BAD_GATEWAY - ); - } - - return response.json(); - } - - async getModels(): Promise { - const response = await fetch(`${this.sttUrl}/api/v1/models`); - - if (!response.ok) { - throw new HttpException('Failed to get STT models', HttpStatus.BAD_GATEWAY); - } - - return response.json(); - } - - async getLanguages(): Promise { - const response = await fetch(`${this.sttUrl}/api/v1/languages`); - - if (!response.ok) { - throw new HttpException('Failed to get STT languages', HttpStatus.BAD_GATEWAY); - } - - return response.json(); - } -} diff --git a/services/mana-api-gateway/src/proxy/services/tts-proxy.service.ts b/services/mana-api-gateway/src/proxy/services/tts-proxy.service.ts deleted file mode 100644 index b4e23bf48..000000000 --- a/services/mana-api-gateway/src/proxy/services/tts-proxy.service.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { Injectable, HttpException, HttpStatus } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; - -export interface SynthesizeRequestDto { - text: string; - voice?: string; - language?: string; - speed?: number; - format?: 'mp3' | 'wav' | 'ogg'; -} - -@Injectable() -export class TtsProxyService { - private readonly ttsUrl: string; - - constructor(private readonly configService: ConfigService) { - this.ttsUrl = this.configService.get('services.tts') || 'http://localhost:3022'; - } - - async synthesize(body: SynthesizeRequestDto): Promise { - const response = await fetch(`${this.ttsUrl}/api/v1/synthesize`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(body), - }); - - if (!response.ok) { - const error = await response.text(); - throw new HttpException( - `TTS service error: ${error}`, - response.status || HttpStatus.BAD_GATEWAY - ); - } - - const arrayBuffer = await response.arrayBuffer(); - return Buffer.from(arrayBuffer); - } - - async getVoices(): Promise { - const response = await fetch(`${this.ttsUrl}/api/v1/voices`); - - if (!response.ok) { - throw new HttpException('Failed to get TTS voices', HttpStatus.BAD_GATEWAY); - } - - return response.json(); - } - - async getLanguages(): Promise { - const response = await fetch(`${this.ttsUrl}/api/v1/languages`); - - if (!response.ok) { - throw new HttpException('Failed to get TTS languages', HttpStatus.BAD_GATEWAY); - } - - return response.json(); - } -} diff --git a/services/mana-api-gateway/src/scheduler/scheduler.module.ts b/services/mana-api-gateway/src/scheduler/scheduler.module.ts deleted file mode 100644 index d9da4b9e4..000000000 --- a/services/mana-api-gateway/src/scheduler/scheduler.module.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { Module } from '@nestjs/common'; -import { ScheduleModule } from '@nestjs/schedule'; -import { SchedulerService } from './scheduler.service'; - -@Module({ - imports: [ScheduleModule.forRoot()], - providers: [SchedulerService], -}) -export class SchedulerModule {} diff --git a/services/mana-api-gateway/src/scheduler/scheduler.service.ts b/services/mana-api-gateway/src/scheduler/scheduler.service.ts deleted file mode 100644 index 58e31b7d3..000000000 --- a/services/mana-api-gateway/src/scheduler/scheduler.service.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { Injectable, Inject } from '@nestjs/common'; -import { Cron, CronExpression } from '@nestjs/schedule'; -import { sql } from 'drizzle-orm'; -import { DATABASE_CONNECTION } from '../db/database.module'; -import { apiKeys } from '../db/schema'; - -@Injectable() -export class SchedulerService { - constructor( - @Inject(DATABASE_CONNECTION) - private readonly db: ReturnType - ) {} - - /** - * Reset monthly credits on the 1st of each month at 00:00 UTC - */ - @Cron('0 0 1 * *') - async resetMonthlyCredits() { - console.log('[Scheduler] Running monthly credit reset...'); - - try { - const result = await this.db - .update(apiKeys) - .set({ - creditsUsed: 0, - creditsResetAt: this.getNextMonthReset(), - updatedAt: new Date(), - }) - .returning({ id: apiKeys.id }); - - console.log(`[Scheduler] Reset credits for ${result.length} API keys`); - } catch (error) { - console.error('[Scheduler] Failed to reset monthly credits:', error); - } - } - - /** - * Clean up old usage logs (older than 90 days) - runs weekly - */ - @Cron(CronExpression.EVERY_WEEK) - async cleanupOldUsageLogs() { - console.log('[Scheduler] Cleaning up old usage logs...'); - - try { - const cutoffDate = new Date(); - cutoffDate.setDate(cutoffDate.getDate() - 90); - - await this.db.execute( - sql`DELETE FROM api_gateway.api_usage WHERE created_at < ${cutoffDate.toISOString()}` - ); - - console.log('[Scheduler] Cleaned up usage logs older than 90 days'); - } catch (error) { - console.error('[Scheduler] Failed to cleanup usage logs:', error); - } - } - - /** - * Aggregate daily usage stats - runs at 1:00 AM UTC - */ - @Cron('0 1 * * *') - async aggregateDailyUsage() { - console.log('[Scheduler] Daily usage aggregation completed (handled by interceptor)'); - // Note: Daily aggregation is already handled in real-time by UsageTrackingInterceptor - // This cron is a placeholder for any additional daily processing - } - - private getNextMonthReset(): Date { - const now = new Date(); - return new Date(now.getFullYear(), now.getMonth() + 1, 1); - } -} diff --git a/services/mana-api-gateway/src/usage/dto/usage-query.dto.ts b/services/mana-api-gateway/src/usage/dto/usage-query.dto.ts deleted file mode 100644 index 0d98d57d9..000000000 --- a/services/mana-api-gateway/src/usage/dto/usage-query.dto.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { IsOptional, IsString, IsDateString, IsInt, Min, Max } from 'class-validator'; -import { Transform } from 'class-transformer'; -import { ApiPropertyOptional } from '@nestjs/swagger'; - -export class UsageQueryDto { - @ApiPropertyOptional({ - description: 'Number of days to query (1-365)', - minimum: 1, - maximum: 365, - default: 30, - }) - @IsOptional() - @Transform(({ value }) => parseInt(value, 10)) - @IsInt() - @Min(1) - @Max(365) - days?: number = 30; - - @ApiPropertyOptional({ - description: 'Start date for custom range (ISO 8601)', - example: '2025-01-01', - }) - @IsOptional() - @IsDateString() - startDate?: string; - - @ApiPropertyOptional({ - description: 'End date for custom range (ISO 8601)', - example: '2025-01-31', - }) - @IsOptional() - @IsDateString() - endDate?: string; - - @ApiPropertyOptional({ - description: 'Filter by endpoint', - example: 'search', - }) - @IsOptional() - @IsString() - endpoint?: string; -} diff --git a/services/mana-api-gateway/src/usage/usage.module.ts b/services/mana-api-gateway/src/usage/usage.module.ts deleted file mode 100644 index 992ea5821..000000000 --- a/services/mana-api-gateway/src/usage/usage.module.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { Module, forwardRef } from '@nestjs/common'; -import { UsageService } from './usage.service'; -import { ApiKeysModule } from '../api-keys/api-keys.module'; - -@Module({ - imports: [forwardRef(() => ApiKeysModule)], - providers: [UsageService], - exports: [UsageService], -}) -export class UsageModule {} diff --git a/services/mana-api-gateway/src/usage/usage.service.ts b/services/mana-api-gateway/src/usage/usage.service.ts deleted file mode 100644 index 37448d4fb..000000000 --- a/services/mana-api-gateway/src/usage/usage.service.ts +++ /dev/null @@ -1,187 +0,0 @@ -import { Injectable, Inject } from '@nestjs/common'; -import { eq, sql, gte, and, desc } from 'drizzle-orm'; -import { DATABASE_CONNECTION } from '../db/database.module'; -import { apiUsage, apiUsageDaily, NewApiUsage } from '../db/schema'; - -export interface TrackUsageParams { - apiKeyId: string; - endpoint: string; - method: string; - path: string; - latencyMs: number; - statusCode: number; - creditsUsed: number; - requestSize?: number; - responseSize?: number; - creditReason?: string; - metadata?: Record; -} - -export interface UsageSummary { - totalRequests: number; - totalCreditsUsed: number; - avgLatencyMs: number; - errorCount: number; - byEndpoint: Record< - string, - { - requests: number; - credits: number; - avgLatencyMs: number; - errors: number; - } - >; -} - -@Injectable() -export class UsageService { - constructor( - @Inject(DATABASE_CONNECTION) - private readonly db: ReturnType - ) {} - - /** - * Track a single API usage event - */ - async track(params: TrackUsageParams): Promise { - const usage: NewApiUsage = { - apiKeyId: params.apiKeyId, - endpoint: params.endpoint, - method: params.method, - path: params.path, - latencyMs: params.latencyMs, - statusCode: params.statusCode, - creditsUsed: params.creditsUsed, - requestSize: params.requestSize, - responseSize: params.responseSize, - creditReason: params.creditReason, - metadata: params.metadata, - }; - - await this.db.insert(apiUsage).values(usage); - - // Also update daily aggregates - await this.updateDailyAggregate(params); - } - - /** - * Update daily usage aggregate - */ - private async updateDailyAggregate(params: TrackUsageParams): Promise { - const today = new Date().toISOString().split('T')[0]; - const isError = params.statusCode >= 400; - - // Upsert daily aggregate - await this.db - .insert(apiUsageDaily) - .values({ - apiKeyId: params.apiKeyId, - date: today, - endpoint: params.endpoint, - requestCount: 1, - creditsUsed: params.creditsUsed, - totalLatencyMs: params.latencyMs, - errorCount: isError ? 1 : 0, - }) - .onConflictDoUpdate({ - target: [apiUsageDaily.apiKeyId, apiUsageDaily.date, apiUsageDaily.endpoint], - set: { - requestCount: sql`${apiUsageDaily.requestCount} + 1`, - creditsUsed: sql`${apiUsageDaily.creditsUsed} + ${params.creditsUsed}`, - totalLatencyMs: sql`${apiUsageDaily.totalLatencyMs} + ${params.latencyMs}`, - errorCount: isError ? sql`${apiUsageDaily.errorCount} + 1` : apiUsageDaily.errorCount, - }, - }); - } - - /** - * Get daily usage for an API key - */ - async getDailyUsage(apiKeyId: string, days: number = 30) { - const startDate = new Date(); - startDate.setDate(startDate.getDate() - days); - - const usage = await this.db - .select() - .from(apiUsageDaily) - .where( - and( - eq(apiUsageDaily.apiKeyId, apiKeyId), - gte(apiUsageDaily.date, startDate.toISOString().split('T')[0]) - ) - ) - .orderBy(desc(apiUsageDaily.date)); - - return usage; - } - - /** - * Get usage summary for an API key - */ - async getUsageSummary(apiKeyId: string, days: number = 30): Promise { - const startDate = new Date(); - startDate.setDate(startDate.getDate() - days); - - const dailyUsage = await this.getDailyUsage(apiKeyId, days); - - const summary: UsageSummary = { - totalRequests: 0, - totalCreditsUsed: 0, - avgLatencyMs: 0, - errorCount: 0, - byEndpoint: {}, - }; - - let totalLatency = 0; - - for (const day of dailyUsage) { - summary.totalRequests += day.requestCount; - summary.totalCreditsUsed += day.creditsUsed; - totalLatency += day.totalLatencyMs; - summary.errorCount += day.errorCount; - - if (!summary.byEndpoint[day.endpoint]) { - summary.byEndpoint[day.endpoint] = { - requests: 0, - credits: 0, - avgLatencyMs: 0, - errors: 0, - }; - } - - const ep = summary.byEndpoint[day.endpoint]; - ep.requests += day.requestCount; - ep.credits += day.creditsUsed; - ep.avgLatencyMs += day.totalLatencyMs; - ep.errors += day.errorCount; - } - - if (summary.totalRequests > 0) { - summary.avgLatencyMs = Math.round(totalLatency / summary.totalRequests); - } - - // Calculate average latency per endpoint - for (const endpoint of Object.keys(summary.byEndpoint)) { - const ep = summary.byEndpoint[endpoint]; - if (ep.requests > 0) { - ep.avgLatencyMs = Math.round(ep.avgLatencyMs / ep.requests); - } - } - - return summary; - } - - /** - * Get recent usage logs for an API key - */ - async getRecentLogs(apiKeyId: string, limit: number = 100) { - const logs = await this.db - .select() - .from(apiUsage) - .where(eq(apiUsage.apiKeyId, apiKeyId)) - .orderBy(desc(apiUsage.createdAt)) - .limit(limit); - - return logs; - } -} diff --git a/services/mana-api-gateway/tsconfig.build.json b/services/mana-api-gateway/tsconfig.build.json deleted file mode 100644 index 045c9529c..000000000 --- a/services/mana-api-gateway/tsconfig.build.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "extends": "./tsconfig.json", - "exclude": ["node_modules", "dist", "test", "**/*spec.ts"] -} diff --git a/services/mana-api-gateway/tsconfig.json b/services/mana-api-gateway/tsconfig.json deleted file mode 100644 index f02c2417e..000000000 --- a/services/mana-api-gateway/tsconfig.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "compilerOptions": { - "module": "commonjs", - "declaration": true, - "removeComments": true, - "emitDecoratorMetadata": true, - "experimentalDecorators": true, - "allowSyntheticDefaultImports": true, - "target": "ES2022", - "sourceMap": true, - "outDir": "./dist", - "baseUrl": "./", - "incremental": true, - "skipLibCheck": true, - "strictNullChecks": true, - "noImplicitAny": true, - "strictBindCallApply": true, - "forceConsistentCasingInFileNames": true, - "noFallthroughCasesInSwitch": true, - "esModuleInterop": true, - "resolveJsonModule": true - }, - "include": ["src/**/*"], - "exclude": ["node_modules", "dist"] -} diff --git a/services/mana-search-go/.gitignore b/services/mana-search-go/.gitignore new file mode 100644 index 000000000..e660fd93d --- /dev/null +++ b/services/mana-search-go/.gitignore @@ -0,0 +1 @@ +bin/ diff --git a/services/mana-search-go/CLAUDE.md b/services/mana-search-go/CLAUDE.md new file mode 100644 index 000000000..6fa52d9b8 --- /dev/null +++ b/services/mana-search-go/CLAUDE.md @@ -0,0 +1,73 @@ +# mana-search (Go) + +Go replacement for the NestJS mana-search service. Unified web search and content extraction microservice using SearXNG + Redis. + +## Architecture + +- **Language:** Go 1.25 +- **Search Engine:** SearXNG (meta-search) +- **Cache:** Redis (graceful degradation if unavailable) +- **Metrics:** Prometheus +- **Port:** 3021 + +## Endpoints + +### Search +- `POST /api/v1/search` — Web search via SearXNG +- `GET /api/v1/search/engines` — List available engines +- `GET /api/v1/search/health` — Search service health + cache stats +- `DELETE /api/v1/search/cache` — Clear all cached results + +### Extract +- `POST /api/v1/extract` — Extract content from URL (readability + optional markdown) +- `POST /api/v1/extract/bulk` — Bulk extract (max 20 URLs, configurable concurrency) + +### System +- `GET /health` — Health check (SearXNG + Redis) +- `GET /metrics` — Prometheus metrics + +## Commands + +```bash +go run ./cmd/server # Dev +go build -o bin/mana-search ./cmd/server # Build +go test ./... # Test +``` + +## Environment Variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `PORT` | 3021 | Server port | +| `SEARXNG_URL` | http://localhost:8080 | SearXNG base URL | +| `SEARXNG_TIMEOUT` | 15000 | SearXNG timeout (ms) | +| `SEARXNG_DEFAULT_LANGUAGE` | de-DE | Default search language | +| `REDIS_HOST` | localhost | Redis host | +| `REDIS_PORT` | 6379 | Redis port | +| `REDIS_PASSWORD` | | Redis password | +| `CACHE_SEARCH_TTL` | 3600 | Search cache TTL (seconds) | +| `CACHE_EXTRACT_TTL` | 86400 | Extract cache TTL (seconds) | +| `EXTRACT_TIMEOUT` | 10000 | Content extraction timeout (ms) | +| `EXTRACT_MAX_LENGTH` | 50000 | Max extracted text length (chars) | +| `CORS_ORIGINS` | localhost:3000,5173,8081 | Allowed CORS origins | + +## Search Categories + +| Category | Engines | +|----------|---------| +| `general` | Google, Bing, DuckDuckGo, Brave, Wikipedia | +| `news` | Google News, Bing News | +| `science` | arXiv, Google Scholar, PubMed | +| `it` | GitHub, StackOverflow, NPM, MDN | + +## Docker + +Uses the same `docker-compose.dev.yml` from `services/mana-search/` for SearXNG + Redis. + +```bash +# Start SearXNG + Redis +cd services/mana-search && docker-compose -f docker-compose.dev.yml up -d + +# Run Go service +cd services/mana-search-go && go run ./cmd/server +``` diff --git a/services/mana-search-go/Dockerfile b/services/mana-search-go/Dockerfile new file mode 100644 index 000000000..c545e1a8f --- /dev/null +++ b/services/mana-search-go/Dockerfile @@ -0,0 +1,21 @@ +FROM golang:1.25-alpine AS builder + +WORKDIR /app +COPY services/mana-search-go/go.mod services/mana-search-go/go.sum ./ +RUN go mod download + +COPY services/mana-search-go/ . +RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /mana-search ./cmd/server + +FROM alpine:3.21 + +RUN apk --no-cache add ca-certificates tzdata + +COPY --from=builder /mana-search /usr/local/bin/mana-search + +EXPOSE 3021 + +HEALTHCHECK --interval=30s --timeout=5s --start-period=5s --retries=3 \ + CMD wget -q --spider http://localhost:3021/health || exit 1 + +ENTRYPOINT ["mana-search"] diff --git a/services/mana-search-go/cmd/server/main.go b/services/mana-search-go/cmd/server/main.go new file mode 100644 index 000000000..6b67d4b5f --- /dev/null +++ b/services/mana-search-go/cmd/server/main.go @@ -0,0 +1,93 @@ +package main + +import ( + "context" + "fmt" + "log/slog" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/rs/cors" + + "github.com/manacore/mana-search/internal/cache" + "github.com/manacore/mana-search/internal/config" + "github.com/manacore/mana-search/internal/extract" + "github.com/manacore/mana-search/internal/handler" + "github.com/manacore/mana-search/internal/metrics" + "github.com/manacore/mana-search/internal/search" +) + +func main() { + slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ + Level: slog.LevelInfo, + }))) + + cfg := config.Load() + m := metrics.New() + c := cache.New(cfg, m) + defer c.Close() + + provider := search.NewSearxngProvider(cfg) + extractor := extract.New(cfg) + + searchHandler := handler.NewSearchHandler(provider, c, m, cfg) + extractHandler := handler.NewExtractHandler(extractor, c, m, cfg) + healthHandler := handler.NewHealthHandler(provider, c) + + mux := http.NewServeMux() + + // Health & metrics + mux.HandleFunc("GET /health", healthHandler.Health) + mux.HandleFunc("GET /api/v1/health", healthHandler.Health) + mux.Handle("GET /metrics", promhttp.Handler()) + + // Search endpoints + mux.HandleFunc("POST /api/v1/search", searchHandler.Search) + mux.HandleFunc("GET /api/v1/search/engines", searchHandler.Engines) + mux.HandleFunc("GET /api/v1/search/health", searchHandler.Health) + mux.HandleFunc("DELETE /api/v1/search/cache", searchHandler.ClearCache) + + // Extract endpoints + mux.HandleFunc("POST /api/v1/extract", extractHandler.Extract) + mux.HandleFunc("POST /api/v1/extract/bulk", extractHandler.BulkExtract) + + c_handler := cors.New(cors.Options{ + AllowedOrigins: cfg.CORSOrigins, + AllowedMethods: []string{"GET", "POST", "DELETE", "OPTIONS"}, + AllowedHeaders: []string{"Content-Type", "Authorization"}, + AllowCredentials: true, + }).Handler(mux) + + server := &http.Server{ + Addr: fmt.Sprintf(":%d", cfg.Port), + Handler: c_handler, + ReadTimeout: 30 * time.Second, + WriteTimeout: 60 * time.Second, + IdleTimeout: 120 * time.Second, + } + + // Graceful shutdown + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + + go func() { + slog.Info("mana-search started", "port", cfg.Port, "searxng", cfg.SearxngURL) + if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { + slog.Error("server error", "error", err) + os.Exit(1) + } + }() + + <-sigCh + slog.Info("shutting down...") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + server.Shutdown(ctx) + + slog.Info("server stopped") +} diff --git a/services/mana-search-go/go.mod b/services/mana-search-go/go.mod new file mode 100644 index 000000000..ae6d4a8cf --- /dev/null +++ b/services/mana-search-go/go.mod @@ -0,0 +1,31 @@ +module github.com/manacore/mana-search + +go 1.25.0 + +require ( + github.com/JohannesKaufmann/html-to-markdown/v2 v2.3.3 + github.com/go-shiori/go-readability v0.0.0-20251205110129-5db1dc9836f0 + github.com/prometheus/client_golang v1.22.0 + github.com/redis/go-redis/v9 v9.18.0 + github.com/rs/cors v1.11.1 +) + +require ( + github.com/JohannesKaufmann/dom v0.2.0 // indirect + github.com/andybalholm/cascadia v1.3.3 // indirect + github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c // indirect + github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.62.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + go.uber.org/atomic v1.11.0 // indirect + golang.org/x/net v0.39.0 // indirect + golang.org/x/sys v0.32.0 // indirect + golang.org/x/text v0.24.0 // indirect + google.golang.org/protobuf v1.36.5 // indirect +) diff --git a/services/mana-search-go/go.sum b/services/mana-search-go/go.sum new file mode 100644 index 000000000..14c3b3fdd --- /dev/null +++ b/services/mana-search-go/go.sum @@ -0,0 +1,145 @@ +github.com/JohannesKaufmann/dom v0.2.0 h1:1bragmEb19K8lHAqgFgqCpiPCFEZMTXzOIEjuxkUfLQ= +github.com/JohannesKaufmann/dom v0.2.0/go.mod h1:57iSUl5RKric4bUkgos4zu6Xt5LMHUnw3TF1l5CbGZo= +github.com/JohannesKaufmann/html-to-markdown/v2 v2.3.3 h1:r3fokGFRDk/8pHmwLwJ8zsX4qiqfS1/1TZm2BH8ueY8= +github.com/JohannesKaufmann/html-to-markdown/v2 v2.3.3/go.mod h1:HtsP+1Fchp4dVvaiIsLHAl/yqL3H1YLwqLC9kNwqQEg= +github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= +github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= +github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA= +github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c h1:wpkoddUomPfHiOziHZixGO5ZBS73cKqVzZipfrLmO1w= +github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c/go.mod h1:oVDCh3qjJMLVUSILBRwrm+Bc6RNXGZYtoh9xdvf1ffM= +github.com/go-shiori/go-readability v0.0.0-20251205110129-5db1dc9836f0 h1:A3B75Yp163FAIf9nLlFMl4pwIj+T3uKxfI7mbvvY2Ls= +github.com/go-shiori/go-readability v0.0.0-20251205110129-5db1dc9836f0/go.mod h1:suxK0Wpz4BM3/2+z1mnOVTIWHDiMCIOGoKDCRumSsk0= +github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs= +github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs= +github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA= +github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= +github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= +github.com/sebdah/goldie/v2 v2.5.5 h1:rx1mwF95RxZ3/83sdS4Yp7t2C5TCokvWP4TBRbAyEWY= +github.com/sebdah/goldie/v2 v2.5.5/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= +github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= +github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark v1.7.11 h1:ZCxLyDMtz0nT2HFfsYG8WZ47Trip2+JyLysKcMYE5bo= +github.com/yuin/goldmark v1.7.11/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= +golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/services/mana-search-go/internal/cache/cache.go b/services/mana-search-go/internal/cache/cache.go new file mode 100644 index 000000000..37ce43f76 --- /dev/null +++ b/services/mana-search-go/internal/cache/cache.go @@ -0,0 +1,159 @@ +package cache + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "sync/atomic" + "time" + + "github.com/redis/go-redis/v9" + + "github.com/manacore/mana-search/internal/config" + "github.com/manacore/mana-search/internal/metrics" +) + +type Cache struct { + client *redis.Client + prefix string + metrics *metrics.Metrics + hits atomic.Int64 + misses atomic.Int64 +} + +func New(cfg *config.Config, m *metrics.Metrics) *Cache { + c := &Cache{ + prefix: cfg.RedisPrefix, + metrics: m, + } + + client := redis.NewClient(&redis.Options{ + Addr: fmt.Sprintf("%s:%d", cfg.RedisHost, cfg.RedisPort), + Password: cfg.RedisPassword, + MaxRetries: 3, + MinRetryBackoff: 200 * time.Millisecond, + MaxRetryBackoff: 2 * time.Second, + }) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + if err := client.Ping(ctx).Err(); err != nil { + slog.Warn("redis unavailable, running without cache", "error", err) + return c + } + + slog.Info("redis connected", "addr", fmt.Sprintf("%s:%d", cfg.RedisHost, cfg.RedisPort)) + c.client = client + return c +} + +func (c *Cache) Get(ctx context.Context, key string) ([]byte, bool) { + if c.client == nil { + return nil, false + } + + val, err := c.client.Get(ctx, c.prefix+key).Bytes() + if err != nil { + c.misses.Add(1) + if c.metrics != nil { + c.metrics.CacheMisses.Inc() + } + return nil, false + } + + c.hits.Add(1) + if c.metrics != nil { + c.metrics.CacheHits.Inc() + } + return val, true +} + +func (c *Cache) Set(ctx context.Context, key string, value any, ttl time.Duration) { + if c.client == nil { + return + } + + data, err := json.Marshal(value) + if err != nil { + slog.Error("cache marshal error", "error", err) + return + } + + if err := c.client.Set(ctx, c.prefix+key, data, ttl).Err(); err != nil { + slog.Error("cache set error", "error", err) + } +} + +func (c *Cache) Delete(ctx context.Context, key string) { + if c.client == nil { + return + } + c.client.Del(ctx, c.prefix+key) +} + +func (c *Cache) Clear(ctx context.Context) (int64, error) { + if c.client == nil { + return 0, nil + } + + keys, err := c.client.Keys(ctx, c.prefix+"*").Result() + if err != nil { + return 0, err + } + if len(keys) == 0 { + return 0, nil + } + + deleted, err := c.client.Del(ctx, keys...).Result() + return deleted, err +} + +type HealthStatus struct { + Status string `json:"status"` + Latency int64 `json:"latency"` +} + +func (c *Cache) HealthCheck(ctx context.Context) HealthStatus { + if c.client == nil { + return HealthStatus{Status: "disabled", Latency: 0} + } + + start := time.Now() + err := c.client.Ping(ctx).Err() + latency := time.Since(start).Milliseconds() + + if err != nil { + return HealthStatus{Status: "error", Latency: latency} + } + return HealthStatus{Status: "ok", Latency: latency} +} + +func (c *Cache) IsConnected() bool { + return c.client != nil +} + +type Stats struct { + Hits int64 `json:"hits"` + Misses int64 `json:"misses"` + HitRate float64 `json:"hitRate"` +} + +func (c *Cache) Stats() Stats { + hits := c.hits.Load() + misses := c.misses.Load() + total := hits + misses + var rate float64 + if total > 0 { + rate = float64(hits) / float64(total) + } + return Stats{Hits: hits, Misses: misses, HitRate: rate} +} + +func (c *Cache) Close() error { + if c.client != nil { + return c.client.Close() + } + return nil +} diff --git a/services/mana-search-go/internal/config/config.go b/services/mana-search-go/internal/config/config.go new file mode 100644 index 000000000..b96a33466 --- /dev/null +++ b/services/mana-search-go/internal/config/config.go @@ -0,0 +1,81 @@ +package config + +import ( + "os" + "strconv" + "strings" +) + +type Config struct { + Port int + + // SearXNG + SearxngURL string + SearxngTimeout int // ms + SearxngDefaultLanguage string + + // Redis + RedisHost string + RedisPort int + RedisPassword string + RedisPrefix string + + // Cache TTLs (seconds) + CacheSearchTTL int + CacheExtractTTL int + + // Extract + ExtractTimeout int // ms + ExtractMaxLength int + ExtractUserAgent string + + // CORS + CORSOrigins []string +} + +func Load() *Config { + return &Config{ + Port: getEnvInt("PORT", 3021), + + SearxngURL: getEnv("SEARXNG_URL", "http://localhost:8080"), + SearxngTimeout: getEnvInt("SEARXNG_TIMEOUT", 15000), + SearxngDefaultLanguage: getEnv("SEARXNG_DEFAULT_LANGUAGE", "de-DE"), + + RedisHost: getEnv("REDIS_HOST", "localhost"), + RedisPort: getEnvInt("REDIS_PORT", 6379), + RedisPassword: getEnv("REDIS_PASSWORD", ""), + RedisPrefix: "mana-search:", + + CacheSearchTTL: getEnvInt("CACHE_SEARCH_TTL", 3600), + CacheExtractTTL: getEnvInt("CACHE_EXTRACT_TTL", 86400), + + ExtractTimeout: getEnvInt("EXTRACT_TIMEOUT", 10000), + ExtractMaxLength: getEnvInt("EXTRACT_MAX_LENGTH", 50000), + ExtractUserAgent: getEnv("EXTRACT_USER_AGENT", "Mozilla/5.0 (compatible; ManaSearchBot/1.0; +https://mana.how)"), + + CORSOrigins: getEnvSlice("CORS_ORIGINS", []string{"http://localhost:3000", "http://localhost:5173", "http://localhost:8081"}), + } +} + +func getEnv(key, fallback string) string { + if v := os.Getenv(key); v != "" { + return v + } + return fallback +} + +func getEnvInt(key string, fallback int) int { + if v := os.Getenv(key); v != "" { + if i, err := strconv.Atoi(v); err == nil { + return i + } + } + return fallback +} + +func getEnvSlice(key string, fallback []string) []string { + if v := os.Getenv(key); v != "" { + return strings.Split(v, ",") + } + return fallback +} diff --git a/services/mana-search-go/internal/extract/extractor.go b/services/mana-search-go/internal/extract/extractor.go new file mode 100644 index 000000000..1be9468b7 --- /dev/null +++ b/services/mana-search-go/internal/extract/extractor.go @@ -0,0 +1,278 @@ +package extract + +import ( + "context" + "fmt" + "log/slog" + "math" + "net/http" + "net/url" + "regexp" + "strings" + "time" + + htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown/v2" + readability "github.com/go-shiori/go-readability" + + "github.com/manacore/mana-search/internal/config" +) + +type Extractor struct { + timeout time.Duration + maxLength int + userAgent string +} + +func New(cfg *config.Config) *Extractor { + return &Extractor{ + timeout: time.Duration(cfg.ExtractTimeout) * time.Millisecond, + maxLength: cfg.ExtractMaxLength, + userAgent: cfg.ExtractUserAgent, + } +} + +// ExtractRequest from the client. +type ExtractRequest struct { + URL string `json:"url"` + Options *ExtractOptions `json:"options,omitempty"` +} + +type ExtractOptions struct { + IncludeHTML bool `json:"includeHtml,omitempty"` + IncludeMarkdown bool `json:"includeMarkdown,omitempty"` + MaxLength int `json:"maxLength,omitempty"` + Timeout int `json:"timeout,omitempty"` +} + +// BulkExtractRequest for multiple URLs. +type BulkExtractRequest struct { + URLs []string `json:"urls"` + Options *ExtractOptions `json:"options,omitempty"` + Concurrency int `json:"concurrency,omitempty"` +} + +// ExtractResponse returned to the client. +type ExtractResponse struct { + Success bool `json:"success"` + Content *ExtractedContent `json:"content,omitempty"` + Error string `json:"error,omitempty"` + Meta ExtractMeta `json:"meta"` +} + +type ExtractedContent struct { + Title string `json:"title"` + Description string `json:"description,omitempty"` + Author string `json:"author,omitempty"` + PublishedDate string `json:"publishedDate,omitempty"` + SiteName string `json:"siteName,omitempty"` + Text string `json:"text"` + Markdown string `json:"markdown,omitempty"` + HTML string `json:"html,omitempty"` + WordCount int `json:"wordCount"` + ReadingTime int `json:"readingTime"` + OgImage string `json:"ogImage,omitempty"` +} + +type ExtractMeta struct { + URL string `json:"url"` + Duration int64 `json:"duration"` + Cached bool `json:"cached"` + ContentType string `json:"contentType"` +} + +type BulkExtractResponse struct { + Results []BulkExtractResult `json:"results"` + Meta BulkMeta `json:"meta"` +} + +type BulkExtractResult struct { + URL string `json:"url"` + Success bool `json:"success"` + Content *ExtractedContent `json:"content,omitempty"` + Error string `json:"error,omitempty"` +} + +type BulkMeta struct { + Total int `json:"total"` + Successful int `json:"successful"` + Failed int `json:"failed"` + Duration int64 `json:"duration"` +} + +// Extract fetches a URL and extracts its content using readability. +func (e *Extractor) Extract(ctx context.Context, req *ExtractRequest) *ExtractResponse { + start := time.Now() + + timeout := e.timeout + if req.Options != nil && req.Options.Timeout > 0 { + timeout = time.Duration(req.Options.Timeout) * time.Millisecond + } + + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + parsedURL, err := url.Parse(req.URL) + if err != nil { + return errorResponse(req.URL, "invalid URL", start) + } + + article, err := readability.FromURL(parsedURL.String(), timeout, func(req *http.Request) { + req.Header.Set("User-Agent", e.userAgent) + }) + if err != nil { + slog.Warn("extraction failed", "url", req.URL, "error", err) + return errorResponse(req.URL, fmt.Sprintf("extraction failed: %v", err), start) + } + + text := cleanText(article.TextContent) + maxLen := e.maxLength + if req.Options != nil && req.Options.MaxLength > 0 { + maxLen = req.Options.MaxLength + } + if len(text) > maxLen { + text = text[:maxLen] + } + + wordCount := countWords(text) + readingTime := int(math.Ceil(float64(wordCount) / 200.0)) + + content := &ExtractedContent{ + Title: article.Title, + Description: article.Excerpt, + Author: article.Byline, + PublishedDate: formatTime(article.PublishedTime), + SiteName: article.SiteName, + Text: text, + WordCount: wordCount, + ReadingTime: readingTime, + OgImage: article.Image, + } + + if req.Options != nil && req.Options.IncludeMarkdown && article.Content != "" { + md, err := htmltomarkdown.ConvertString(article.Content) + if err == nil { + content.Markdown = md + } + } + + if req.Options != nil && req.Options.IncludeHTML { + content.HTML = article.Content + } + + return &ExtractResponse{ + Success: true, + Content: content, + Meta: ExtractMeta{ + URL: req.URL, + Duration: time.Since(start).Milliseconds(), + Cached: false, + ContentType: "text/html", + }, + } +} + +// BulkExtract processes multiple URLs with limited concurrency. +func (e *Extractor) BulkExtract(ctx context.Context, req *BulkExtractRequest) *BulkExtractResponse { + start := time.Now() + concurrency := 5 + if req.Concurrency > 0 && req.Concurrency <= 10 { + concurrency = req.Concurrency + } + + results := make([]BulkExtractResult, len(req.URLs)) + + // Process in batches + for i := 0; i < len(req.URLs); i += concurrency { + end := i + concurrency + if end > len(req.URLs) { + end = len(req.URLs) + } + + type indexedResult struct { + index int + result *ExtractResponse + } + + ch := make(chan indexedResult, end-i) + for j := i; j < end; j++ { + go func(idx int, u string) { + r := e.Extract(ctx, &ExtractRequest{URL: u, Options: req.Options}) + ch <- indexedResult{index: idx, result: r} + }(j, req.URLs[j]) + } + + for j := i; j < end; j++ { + ir := <-ch + results[ir.index] = BulkExtractResult{ + URL: req.URLs[ir.index], + Success: ir.result.Success, + Content: ir.result.Content, + Error: ir.result.Error, + } + } + } + + successful := 0 + failed := 0 + for _, r := range results { + if r.Success { + successful++ + } else { + failed++ + } + } + + return &BulkExtractResponse{ + Results: results, + Meta: BulkMeta{ + Total: len(req.URLs), + Successful: successful, + Failed: failed, + Duration: time.Since(start).Milliseconds(), + }, + } +} + +// BuildCacheKey creates a cache key for extraction results. +func BuildCacheKey(rawURL string) string { + return "extract:" + rawURL +} + +func errorResponse(rawURL, errMsg string, start time.Time) *ExtractResponse { + return &ExtractResponse{ + Success: false, + Error: errMsg, + Meta: ExtractMeta{ + URL: rawURL, + Duration: time.Since(start).Milliseconds(), + }, + } +} + +var ( + reScript = regexp.MustCompile(`(?is)]*>.*?`) + reStyle = regexp.MustCompile(`(?is)]*>.*?`) + reTags = regexp.MustCompile(`<[^>]+>`) + reWhitespace = regexp.MustCompile(`\s+`) +) + +func cleanText(html string) string { + text := reScript.ReplaceAllString(html, "") + text = reStyle.ReplaceAllString(text, "") + text = reTags.ReplaceAllString(text, "") + text = reWhitespace.ReplaceAllString(text, " ") + return strings.TrimSpace(text) +} + +func countWords(text string) int { + fields := strings.Fields(text) + return len(fields) +} + +func formatTime(t *time.Time) string { + if t == nil || t.IsZero() { + return "" + } + return t.Format(time.RFC3339) +} + diff --git a/services/mana-search-go/internal/handler/common.go b/services/mana-search-go/internal/handler/common.go new file mode 100644 index 000000000..69d3f747e --- /dev/null +++ b/services/mana-search-go/internal/handler/common.go @@ -0,0 +1,24 @@ +package handler + +import ( + "encoding/json" + "net/http" + "time" +) + +func writeJSON(w http.ResponseWriter, status int, data any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + json.NewEncoder(w).Encode(data) +} + +func writeError(w http.ResponseWriter, status int, message string) { + writeJSON(w, status, map[string]any{ + "success": false, + "error": map[string]any{ + "statusCode": status, + "message": message, + "timestamp": time.Now().UTC().Format(time.RFC3339), + }, + }) +} diff --git a/services/mana-search-go/internal/handler/extract.go b/services/mana-search-go/internal/handler/extract.go new file mode 100644 index 000000000..150812078 --- /dev/null +++ b/services/mana-search-go/internal/handler/extract.go @@ -0,0 +1,127 @@ +package handler + +import ( + "encoding/json" + "net/http" + "net/url" + "time" + + "github.com/manacore/mana-search/internal/cache" + "github.com/manacore/mana-search/internal/config" + "github.com/manacore/mana-search/internal/extract" + "github.com/manacore/mana-search/internal/metrics" +) + +type ExtractHandler struct { + extractor *extract.Extractor + cache *cache.Cache + metrics *metrics.Metrics + cfg *config.Config +} + +func NewExtractHandler(extractor *extract.Extractor, c *cache.Cache, m *metrics.Metrics, cfg *config.Config) *ExtractHandler { + return &ExtractHandler{ + extractor: extractor, + cache: c, + metrics: m, + cfg: cfg, + } +} + +// Extract handles POST /api/v1/extract +func (h *ExtractHandler) Extract(w http.ResponseWriter, r *http.Request) { + start := time.Now() + + var req extract.ExtractRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "invalid request body") + return + } + + if req.URL == "" { + writeError(w, http.StatusBadRequest, "url is required") + return + } + if _, err := url.ParseRequestURI(req.URL); err != nil { + writeError(w, http.StatusBadRequest, "url must be a valid URL") + return + } + + // Validate options + if req.Options != nil { + if req.Options.MaxLength > 0 && (req.Options.MaxLength < 100 || req.Options.MaxLength > 100000) { + writeError(w, http.StatusBadRequest, "maxLength must be between 100 and 100000") + return + } + if req.Options.Timeout > 0 && (req.Options.Timeout < 1000 || req.Options.Timeout > 30000) { + writeError(w, http.StatusBadRequest, "timeout must be between 1000 and 30000") + return + } + } + + cacheKey := extract.BuildCacheKey(req.URL) + + // Check cache + if data, ok := h.cache.Get(r.Context(), cacheKey); ok { + var cached extract.ExtractResponse + if err := json.Unmarshal(data, &cached); err == nil { + cached.Meta.Cached = true + duration := time.Since(start).Seconds() + h.metrics.RecordRequest("extract", "200", duration) + writeJSON(w, http.StatusOK, cached) + return + } + } + + // Extract content + resp := h.extractor.Extract(r.Context(), &req) + + // Cache successful results + if resp.Success { + ttl := time.Duration(h.cfg.CacheExtractTTL) * time.Second + h.cache.Set(r.Context(), cacheKey, resp, ttl) + } + + status := "200" + if !resp.Success { + status = "500" + } + duration := time.Since(start).Seconds() + h.metrics.RecordRequest("extract", status, duration) + + writeJSON(w, http.StatusOK, resp) +} + +// BulkExtract handles POST /api/v1/extract/bulk +func (h *ExtractHandler) BulkExtract(w http.ResponseWriter, r *http.Request) { + start := time.Now() + + var req extract.BulkExtractRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "invalid request body") + return + } + + if len(req.URLs) == 0 { + writeError(w, http.StatusBadRequest, "urls is required") + return + } + if len(req.URLs) > 20 { + writeError(w, http.StatusBadRequest, "maximum 20 URLs allowed") + return + } + + for _, u := range req.URLs { + if _, err := url.ParseRequestURI(u); err != nil { + writeError(w, http.StatusBadRequest, "invalid URL: "+u) + return + } + } + + resp := h.extractor.BulkExtract(r.Context(), &req) + + duration := time.Since(start).Seconds() + h.metrics.RecordRequest("extract_bulk", "200", duration) + + writeJSON(w, http.StatusOK, resp) +} diff --git a/services/mana-search-go/internal/handler/health.go b/services/mana-search-go/internal/handler/health.go new file mode 100644 index 000000000..6afdfc68b --- /dev/null +++ b/services/mana-search-go/internal/handler/health.go @@ -0,0 +1,51 @@ +package handler + +import ( + "net/http" + "time" + + "github.com/manacore/mana-search/internal/cache" + "github.com/manacore/mana-search/internal/search" +) + +type HealthHandler struct { + provider *search.SearxngProvider + cache *cache.Cache + startTime time.Time +} + +func NewHealthHandler(provider *search.SearxngProvider, c *cache.Cache) *HealthHandler { + return &HealthHandler{ + provider: provider, + cache: c, + startTime: time.Now(), + } +} + +// Health handles GET /health +func (h *HealthHandler) Health(w http.ResponseWriter, r *http.Request) { + sxStatus, sxLatency := h.provider.HealthCheck(r.Context()) + redisHealth := h.cache.HealthCheck(r.Context()) + + overall := "ok" + if sxStatus == "error" && redisHealth.Status == "error" { + overall = "error" + } else if sxStatus == "error" || redisHealth.Status == "error" || redisHealth.Status == "disabled" { + overall = "degraded" + } + + writeJSON(w, http.StatusOK, map[string]any{ + "status": overall, + "service": "mana-search", + "version": "1.0.0", + "uptime": time.Since(h.startTime).Seconds(), + "timestamp": time.Now().UTC().Format(time.RFC3339), + "components": map[string]any{ + "searxng": map[string]any{ + "status": sxStatus, + "latency": sxLatency, + }, + "redis": redisHealth, + }, + }) +} diff --git a/services/mana-search-go/internal/handler/search.go b/services/mana-search-go/internal/handler/search.go new file mode 100644 index 000000000..2ad65d3cb --- /dev/null +++ b/services/mana-search-go/internal/handler/search.go @@ -0,0 +1,156 @@ +package handler + +import ( + "encoding/json" + "log/slog" + "net/http" + "time" + + "github.com/manacore/mana-search/internal/cache" + "github.com/manacore/mana-search/internal/config" + "github.com/manacore/mana-search/internal/metrics" + "github.com/manacore/mana-search/internal/search" +) + +type SearchHandler struct { + provider *search.SearxngProvider + cache *cache.Cache + metrics *metrics.Metrics + cfg *config.Config +} + +func NewSearchHandler(provider *search.SearxngProvider, c *cache.Cache, m *metrics.Metrics, cfg *config.Config) *SearchHandler { + return &SearchHandler{ + provider: provider, + cache: c, + metrics: m, + cfg: cfg, + } +} + +// Search handles POST /api/v1/search +func (h *SearchHandler) Search(w http.ResponseWriter, r *http.Request) { + start := time.Now() + h.metrics.ActiveSearches.Inc() + defer h.metrics.ActiveSearches.Dec() + + var req search.SearchRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "invalid request body") + return + } + + if req.Query == "" { + writeError(w, http.StatusBadRequest, "query is required") + return + } + + // Validate options + if req.Options != nil { + if req.Options.Limit < 0 || req.Options.Limit > 50 { + writeError(w, http.StatusBadRequest, "limit must be between 1 and 50") + return + } + if req.Options.SafeSearch < 0 || req.Options.SafeSearch > 2 { + writeError(w, http.StatusBadRequest, "safeSearch must be 0, 1, or 2") + return + } + } + + cacheKey := search.BuildCacheKey(&req) + + // Check cache + if req.Cache.IsEnabled() { + if data, ok := h.cache.Get(r.Context(), cacheKey); ok { + var cached search.SearchResponse + if err := json.Unmarshal(data, &cached); err == nil { + cached.Meta.Cached = true + cached.Meta.CacheKey = cacheKey + duration := time.Since(start).Seconds() + h.metrics.RecordRequest("search", "200", duration) + writeJSON(w, http.StatusOK, cached) + return + } + } + } + + // Query SearXNG + results, err := h.provider.Search(r.Context(), &req) + if err != nil { + slog.Error("search failed", "error", err, "query", req.Query) + duration := time.Since(start).Seconds() + h.metrics.RecordRequest("search", "502", duration) + writeError(w, http.StatusBadGateway, err.Error()) + return + } + + // Collect unique engines + engineSet := make(map[string]bool) + for _, r := range results { + engineSet[r.Engine] = true + } + engines := make([]string, 0, len(engineSet)) + for e := range engineSet { + engines = append(engines, e) + } + + resp := search.SearchResponse{ + Results: results, + Meta: search.SearchMeta{ + Query: req.Query, + TotalResults: len(results), + Engines: engines, + Duration: time.Since(start).Milliseconds(), + Cached: false, + CacheKey: cacheKey, + }, + } + + // Cache result + if req.Cache.IsEnabled() { + ttl := time.Duration(h.cfg.CacheSearchTTL) * time.Second + if req.Cache != nil && req.Cache.TTL > 0 { + ttl = time.Duration(req.Cache.TTL) * time.Second + } + h.cache.Set(r.Context(), cacheKey, resp, ttl) + } + + duration := time.Since(start).Seconds() + h.metrics.RecordRequest("search", "200", duration) + writeJSON(w, http.StatusOK, resp) +} + +// Engines handles GET /api/v1/search/engines +func (h *SearchHandler) Engines(w http.ResponseWriter, r *http.Request) { + engines := h.provider.GetEngines(r.Context()) + writeJSON(w, http.StatusOK, map[string]any{"engines": engines}) +} + +// Health handles GET /api/v1/search/health +func (h *SearchHandler) Health(w http.ResponseWriter, r *http.Request) { + sxStatus, sxLatency := h.provider.HealthCheck(r.Context()) + redisHealth := h.cache.HealthCheck(r.Context()) + cacheStats := h.cache.Stats() + + writeJSON(w, http.StatusOK, map[string]any{ + "searxng": map[string]any{ + "status": sxStatus, + "latency": sxLatency, + }, + "redis": redisHealth, + "cache": cacheStats, + }) +} + +// ClearCache handles DELETE /api/v1/search/cache +func (h *SearchHandler) ClearCache(w http.ResponseWriter, r *http.Request) { + deleted, err := h.cache.Clear(r.Context()) + if err != nil { + writeError(w, http.StatusInternalServerError, "failed to clear cache") + return + } + writeJSON(w, http.StatusOK, map[string]any{ + "cleared": true, + "keysRemoved": deleted, + }) +} diff --git a/services/mana-search-go/internal/metrics/metrics.go b/services/mana-search-go/internal/metrics/metrics.go new file mode 100644 index 000000000..de6b1903c --- /dev/null +++ b/services/mana-search-go/internal/metrics/metrics.go @@ -0,0 +1,55 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +type Metrics struct { + Requests *prometheus.CounterVec + Latency *prometheus.HistogramVec + CacheHits prometheus.Counter + CacheMisses prometheus.Counter + EngineStatus *prometheus.GaugeVec + ActiveSearches prometheus.Gauge +} + +func New() *Metrics { + return &Metrics{ + Requests: promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "mana_search_requests_total", + Help: "Total number of requests", + }, []string{"endpoint", "status"}), + + Latency: promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "mana_search_latency_seconds", + Help: "Request latency in seconds", + Buckets: []float64{0.1, 0.25, 0.5, 1, 2, 5, 10}, + }, []string{"endpoint"}), + + CacheHits: promauto.NewCounter(prometheus.CounterOpts{ + Name: "mana_search_cache_hits_total", + Help: "Total cache hits", + }), + + CacheMisses: promauto.NewCounter(prometheus.CounterOpts{ + Name: "mana_search_cache_misses_total", + Help: "Total cache misses", + }), + + EngineStatus: promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "mana_search_engine_status", + Help: "Search engine status (1=ok, 0=error)", + }, []string{"engine"}), + + ActiveSearches: promauto.NewGauge(prometheus.GaugeOpts{ + Name: "mana_search_active_searches", + Help: "Number of active search requests", + }), + } +} + +func (m *Metrics) RecordRequest(endpoint string, status string, durationSeconds float64) { + m.Requests.WithLabelValues(endpoint, status).Inc() + m.Latency.WithLabelValues(endpoint).Observe(durationSeconds) +} diff --git a/services/mana-search-go/internal/search/searxng.go b/services/mana-search-go/internal/search/searxng.go new file mode 100644 index 000000000..86534fdc6 --- /dev/null +++ b/services/mana-search-go/internal/search/searxng.go @@ -0,0 +1,295 @@ +package search + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "math" + "net/http" + "net/url" + "sort" + "strings" + "time" + + "github.com/manacore/mana-search/internal/config" +) + +// SearXNG provider handles communication with the SearXNG meta-search engine. +type SearxngProvider struct { + baseURL string + timeout time.Duration + client *http.Client +} + +func NewSearxngProvider(cfg *config.Config) *SearxngProvider { + timeout := time.Duration(cfg.SearxngTimeout) * time.Millisecond + return &SearxngProvider{ + baseURL: cfg.SearxngURL, + timeout: timeout, + client: &http.Client{ + Timeout: timeout, + }, + } +} + +// SearchRequest represents the incoming search request. +type SearchRequest struct { + Query string `json:"query"` + Options *SearchOptions `json:"options,omitempty"` + Cache *CacheOptions `json:"cache,omitempty"` +} + +type SearchOptions struct { + Categories []string `json:"categories,omitempty"` + Engines []string `json:"engines,omitempty"` + Language string `json:"language,omitempty"` + TimeRange string `json:"timeRange,omitempty"` + SafeSearch int `json:"safeSearch,omitempty"` + Limit int `json:"limit,omitempty"` +} + +type CacheOptions struct { + Enabled *bool `json:"enabled,omitempty"` + TTL int `json:"ttl,omitempty"` +} + +func (c *CacheOptions) IsEnabled() bool { + if c == nil || c.Enabled == nil { + return true + } + return *c.Enabled +} + +// SearchResponse is returned to the client. +type SearchResponse struct { + Results []SearchResult `json:"results"` + Meta SearchMeta `json:"meta"` +} + +type SearchResult struct { + URL string `json:"url"` + Title string `json:"title"` + Snippet string `json:"snippet"` + Engine string `json:"engine"` + Score float64 `json:"score"` + PublishedDate string `json:"publishedDate,omitempty"` + Thumbnail string `json:"thumbnail,omitempty"` + Category string `json:"category"` +} + +type SearchMeta struct { + Query string `json:"query"` + TotalResults int `json:"totalResults"` + Engines []string `json:"engines"` + Duration int64 `json:"duration"` + Cached bool `json:"cached"` + CacheKey string `json:"cacheKey,omitempty"` +} + +// searxngResponse is the raw response from SearXNG. +type searxngResponse struct { + Results []searxngResult `json:"results"` +} + +type searxngResult struct { + URL string `json:"url"` + Title string `json:"title"` + Content string `json:"content"` + Engine string `json:"engine"` + Score float64 `json:"score"` + PublishedDate string `json:"publishedDate,omitempty"` + Thumbnail string `json:"thumbnail,omitempty"` + Category string `json:"category"` +} + +// Search queries SearXNG and returns normalized, scored results. +func (p *SearxngProvider) Search(ctx context.Context, req *SearchRequest) ([]SearchResult, error) { + params := url.Values{} + params.Set("q", req.Query) + params.Set("format", "json") + + if req.Options != nil { + if len(req.Options.Categories) > 0 { + params.Set("categories", strings.Join(req.Options.Categories, ",")) + } + if len(req.Options.Engines) > 0 { + params.Set("engines", strings.Join(req.Options.Engines, ",")) + } + if req.Options.Language != "" { + params.Set("language", req.Options.Language) + } + if req.Options.TimeRange != "" { + params.Set("time_range", req.Options.TimeRange) + } + params.Set("safesearch", fmt.Sprintf("%d", req.Options.SafeSearch)) + } + + reqURL := fmt.Sprintf("%s/search?%s", p.baseURL, params.Encode()) + httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + + resp, err := p.client.Do(httpReq) + if err != nil { + if ctx.Err() != nil { + return nil, fmt.Errorf("searxng timeout") + } + return nil, fmt.Errorf("searxng unavailable: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("searxng returned %d: %s", resp.StatusCode, string(body)) + } + + var sxResp searxngResponse + if err := json.NewDecoder(resp.Body).Decode(&sxResp); err != nil { + return nil, fmt.Errorf("decode searxng response: %w", err) + } + + return normalizeResults(sxResp.Results, req.Options), nil +} + +// HealthCheck pings SearXNG. +func (p *SearxngProvider) HealthCheck(ctx context.Context) (string, int64) { + start := time.Now() + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + req, _ := http.NewRequestWithContext(ctx, http.MethodGet, p.baseURL+"/healthz", nil) + resp, err := p.client.Do(req) + latency := time.Since(start).Milliseconds() + + if err != nil || resp.StatusCode != http.StatusOK { + return "error", latency + } + resp.Body.Close() + return "ok", latency +} + +// GetEngines fetches available engines from SearXNG config. +func (p *SearxngProvider) GetEngines(ctx context.Context) []string { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + req, _ := http.NewRequestWithContext(ctx, http.MethodGet, p.baseURL+"/config", nil) + resp, err := p.client.Do(req) + if err != nil { + slog.Warn("failed to fetch searxng engines", "error", err) + return nil + } + defer resp.Body.Close() + + var cfg struct { + Engines map[string]any `json:"engines"` + } + if err := json.NewDecoder(resp.Body).Decode(&cfg); err != nil { + return nil + } + + engines := make([]string, 0, len(cfg.Engines)) + for name := range cfg.Engines { + engines = append(engines, name) + } + sort.Strings(engines) + return engines +} + +// BuildCacheKey creates a deterministic cache key from the search request. +func BuildCacheKey(req *SearchRequest) string { + var categories, engines, language, timeRange string + safeSearch := 0 + + if req.Options != nil { + cats := make([]string, len(req.Options.Categories)) + copy(cats, req.Options.Categories) + sort.Strings(cats) + categories = strings.Join(cats, ",") + + engs := make([]string, len(req.Options.Engines)) + copy(engs, req.Options.Engines) + sort.Strings(engs) + engines = strings.Join(engs, ",") + + language = strings.ToLower(req.Options.Language) + timeRange = req.Options.TimeRange + safeSearch = req.Options.SafeSearch + } + + return fmt.Sprintf("search:%s:%s:%s:%s:%s:%d", + strings.ToLower(req.Query), categories, engines, language, timeRange, safeSearch) +} + +// Trusted domains get a score boost. +var trustedDomains = map[string]bool{ + "wikipedia.org": true, + "github.com": true, + "stackoverflow.com": true, +} + +func normalizeResults(raw []searxngResult, opts *SearchOptions) []SearchResult { + seen := make(map[string]bool) + var results []SearchResult + + for _, r := range raw { + if seen[r.URL] { + continue + } + seen[r.URL] = true + + score := scoreResult(r) + results = append(results, SearchResult{ + URL: r.URL, + Title: r.Title, + Snippet: r.Content, + Engine: r.Engine, + Score: score, + PublishedDate: r.PublishedDate, + Thumbnail: r.Thumbnail, + Category: r.Category, + }) + } + + sort.Slice(results, func(i, j int) bool { + return results[i].Score > results[j].Score + }) + + limit := 10 + if opts != nil && opts.Limit > 0 && opts.Limit <= 50 { + limit = opts.Limit + } + if len(results) > limit { + results = results[:limit] + } + + return results +} + +func scoreResult(r searxngResult) float64 { + score := 0.5 + + if len(r.Content) > 100 { + score += 0.1 + } + + // Check if URL is from a trusted domain + if u, err := url.Parse(r.URL); err == nil { + host := strings.ToLower(u.Hostname()) + for domain := range trustedDomains { + if strings.HasSuffix(host, domain) { + score += 0.15 + break + } + } + } + + if len(r.URL) > 200 { + score -= 0.05 + } + + return math.Max(0, math.Min(1, score)) +} diff --git a/services/mana-search-go/package.json b/services/mana-search-go/package.json new file mode 100644 index 000000000..ece232d83 --- /dev/null +++ b/services/mana-search-go/package.json @@ -0,0 +1,10 @@ +{ + "name": "@manacore/mana-search-go", + "version": "1.0.0", + "private": true, + "scripts": { + "dev": "go run ./cmd/server", + "build": "go build -o bin/mana-search ./cmd/server", + "test": "go test ./..." + } +}