mirror of
https://github.com/Memo-2023/mana-monorepo.git
synced 2026-05-14 18:41:08 +02:00
feat(mana-search): rewrite search service from NestJS to Go
Replaces the NestJS mana-search service with a Go implementation for lower resource usage and faster startup. All 7 API endpoints are 1:1 compatible (search, extract, bulk extract, engines, health, metrics, cache clear). Uses go-readability for content extraction and html-to-markdown for Markdown conversion. Redis cache with graceful degradation, Prometheus metrics, and structured JSON logging. Binary: 22 MB vs ~200+ MB node_modules. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
c67ed0df14
commit
4b0f5a29fd
74 changed files with 1607 additions and 3594 deletions
41
.github/workflows/ci.yml
vendored
41
.github/workflows/ci.yml
vendored
|
|
@ -51,7 +51,6 @@ jobs:
|
|||
outputs:
|
||||
mana-core-auth: ${{ steps.changes.outputs.mana-core-auth }}
|
||||
mana-search: ${{ steps.changes.outputs.mana-search }}
|
||||
api-gateway: ${{ steps.changes.outputs.api-gateway }}
|
||||
manacore-web: ${{ steps.changes.outputs.manacore-web }}
|
||||
chat-backend: ${{ steps.changes.outputs.chat-backend }}
|
||||
chat-web: ${{ steps.changes.outputs.chat-web }}
|
||||
|
|
@ -89,7 +88,6 @@ jobs:
|
|||
echo "Force rebuild all services requested"
|
||||
echo "mana-core-auth=true" >> $GITHUB_OUTPUT
|
||||
echo "mana-search=true" >> $GITHUB_OUTPUT
|
||||
echo "api-gateway=true" >> $GITHUB_OUTPUT
|
||||
echo "manacore-web=true" >> $GITHUB_OUTPUT
|
||||
echo "chat-backend=true" >> $GITHUB_OUTPUT
|
||||
echo "chat-web=true" >> $GITHUB_OUTPUT
|
||||
|
|
@ -131,7 +129,6 @@ jobs:
|
|||
echo "Workflow dispatch without force_build_all - building all"
|
||||
echo "mana-core-auth=true" >> $GITHUB_OUTPUT
|
||||
echo "mana-search=true" >> $GITHUB_OUTPUT
|
||||
echo "api-gateway=true" >> $GITHUB_OUTPUT
|
||||
echo "manacore-web=true" >> $GITHUB_OUTPUT
|
||||
echo "chat-backend=true" >> $GITHUB_OUTPUT
|
||||
echo "chat-web=true" >> $GITHUB_OUTPUT
|
||||
|
|
@ -202,14 +199,6 @@ jobs:
|
|||
echo "mana-search=false" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
# api-gateway: services/mana-api-gateway + packages/shared-nestjs-auth
|
||||
GATEWAY_CHANGED=$(check_pattern "services/mana-api-gateway/|packages/shared-nestjs-auth/")
|
||||
if [ "$COMMON_CHANGED" == "true" ] || [ "$GATEWAY_CHANGED" == "true" ]; then
|
||||
echo "api-gateway=true" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "api-gateway=false" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
# manacore-web: apps/manacore/apps/web + shared packages
|
||||
MANACORE_WEB_CHANGED=$(check_pattern "apps/manacore/apps/web/|apps/manacore/packages/")
|
||||
if [ "$COMMON_CHANGED" == "true" ] || [ "$SHARED_AUTH_CHANGED" == "true" ] || [ "$SHARED_UI_CHANGED" == "true" ] || [ "$SHARED_WEB_CHANGED" == "true" ] || [ "$MANACORE_WEB_CHANGED" == "true" ]; then
|
||||
|
|
@ -401,7 +390,6 @@ jobs:
|
|||
echo "|---------|------------|" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| mana-core-auth | ${{ steps.changes.outputs.mana-core-auth }} |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| mana-search | ${{ steps.changes.outputs.mana-search }} |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| api-gateway | ${{ steps.changes.outputs.api-gateway }} |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| manacore-web | ${{ steps.changes.outputs.manacore-web }} |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| chat-backend | ${{ steps.changes.outputs.chat-backend }} |" >> $GITHUB_STEP_SUMMARY
|
||||
echo "| chat-web | ${{ steps.changes.outputs.chat-web }} |" >> $GITHUB_STEP_SUMMARY
|
||||
|
|
@ -546,35 +534,6 @@ jobs:
|
|||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
build-api-gateway:
|
||||
name: Build api-gateway
|
||||
runs-on: ubuntu-latest
|
||||
needs: detect-changes
|
||||
if: needs.detect-changes.outputs.api-gateway == 'true'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: docker/setup-qemu-action@v3
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
- uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
- uses: docker/metadata-action@v5
|
||||
id: meta
|
||||
with:
|
||||
images: ghcr.io/${{ github.repository_owner }}/api-gateway
|
||||
tags: type=raw,value=latest
|
||||
- uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
file: services/mana-api-gateway/Dockerfile
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
build-manacore-web:
|
||||
name: Build manacore-web
|
||||
runs-on: ubuntu-latest
|
||||
|
|
|
|||
|
|
@ -130,7 +130,8 @@ manacore-monorepo/
|
|||
│ └── {game-name}/ # Individual games
|
||||
├── services/ # Standalone microservices
|
||||
│ ├── mana-core-auth/ # Central authentication service
|
||||
│ ├── mana-search/ # Central search & content extraction service
|
||||
│ ├── mana-search/ # Central search & content extraction (NestJS, legacy)
|
||||
│ ├── mana-search-go/ # Central search & content extraction (Go, active)
|
||||
│ ├── mana-crawler/ # Web crawler service
|
||||
│ ├── mana-llm/ # Central LLM abstraction service
|
||||
│ ├── mana-landing-builder/# Org landing page builder (Astro → Cloudflare Pages)
|
||||
|
|
@ -932,7 +933,8 @@ Each project has its own `CLAUDE.md` with detailed information:
|
|||
- `apps/chat/CLAUDE.md` - Chat API endpoints, AI models
|
||||
- `apps/picture/CLAUDE.md` - AI image generation
|
||||
- `services/mana-core-auth/` - Central authentication service
|
||||
- `services/mana-search/CLAUDE.md` - Search & content extraction service
|
||||
- `services/mana-search/CLAUDE.md` - Search & content extraction service (NestJS, legacy)
|
||||
- `services/mana-search-go/CLAUDE.md` - Search & content extraction service (Go, active)
|
||||
- `services/mana-crawler/CLAUDE.md` - Web crawler service
|
||||
- `services/mana-llm/CLAUDE.md` - Central LLM abstraction service
|
||||
- `services/mana-landing-builder/CLAUDE.md` - Org landing page builder service
|
||||
|
|
|
|||
|
|
@ -302,16 +302,14 @@ services:
|
|||
mana-search:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: services/mana-search/Dockerfile
|
||||
dockerfile: services/mana-search-go/Dockerfile
|
||||
image: mana-search:local
|
||||
container_name: mana-core-search
|
||||
restart: always
|
||||
depends_on:
|
||||
searxng:
|
||||
condition: service_healthy
|
||||
# Removed: redis - lazy connect
|
||||
environment:
|
||||
NODE_ENV: production
|
||||
PORT: 3020
|
||||
SEARXNG_URL: http://searxng:8080
|
||||
SEARXNG_TIMEOUT: 15000
|
||||
|
|
@ -326,11 +324,11 @@ services:
|
|||
ports:
|
||||
- "3020:3020"
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://127.0.0.1:3020/api/v1/health"]
|
||||
test: ["CMD", "wget", "-q", "--spider", "http://localhost:3020/health"]
|
||||
interval: 120s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 40s
|
||||
start_period: 5s
|
||||
|
||||
mana-media:
|
||||
build:
|
||||
|
|
|
|||
|
|
@ -1,37 +0,0 @@
|
|||
# Server
|
||||
PORT=3030
|
||||
NODE_ENV=development
|
||||
|
||||
# Database (same as mana-core-auth)
|
||||
DATABASE_URL=postgresql://manacore:devpassword@localhost:5432/manacore
|
||||
|
||||
# Redis
|
||||
REDIS_HOST=localhost
|
||||
REDIS_PORT=6379
|
||||
REDIS_PASSWORD=
|
||||
|
||||
# Backend Services
|
||||
SEARCH_SERVICE_URL=http://localhost:3021
|
||||
STT_SERVICE_URL=http://localhost:3020
|
||||
TTS_SERVICE_URL=http://localhost:3022
|
||||
|
||||
# Auth Service (for JWT validation & credits)
|
||||
MANA_CORE_AUTH_URL=http://localhost:3001
|
||||
|
||||
# API Key Generation
|
||||
API_KEY_PREFIX_LIVE=sk_live_
|
||||
API_KEY_PREFIX_TEST=sk_test_
|
||||
|
||||
# Rate Limiting Defaults
|
||||
DEFAULT_RATE_LIMIT=10
|
||||
DEFAULT_MONTHLY_CREDITS=100
|
||||
|
||||
# CORS
|
||||
CORS_ORIGINS=http://localhost:3000,http://localhost:5173
|
||||
|
||||
# Development Auth Bypass
|
||||
DEV_BYPASS_AUTH=true
|
||||
DEV_USER_ID=00000000-0000-0000-0000-000000000000
|
||||
|
||||
# Admin Access (comma-separated user IDs)
|
||||
ADMIN_USER_IDS=
|
||||
31
services/mana-api-gateway/.gitignore
vendored
31
services/mana-api-gateway/.gitignore
vendored
|
|
@ -1,31 +0,0 @@
|
|||
# Dependencies
|
||||
node_modules/
|
||||
|
||||
# Build output
|
||||
dist/
|
||||
|
||||
# Environment files
|
||||
.env
|
||||
.env.local
|
||||
.env.*.local
|
||||
|
||||
# IDE
|
||||
.idea/
|
||||
.vscode/
|
||||
*.swp
|
||||
*.swo
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
npm-debug.log*
|
||||
pnpm-debug.log*
|
||||
|
||||
# Test coverage
|
||||
coverage/
|
||||
|
||||
# Drizzle
|
||||
drizzle/meta/
|
||||
|
|
@ -1,287 +0,0 @@
|
|||
# Mana API Gateway
|
||||
|
||||
Custom NestJS API Gateway for monetizing ManaCore services (mana-search, mana-stt, mana-tts).
|
||||
|
||||
## Overview
|
||||
|
||||
- **Port**: 3030
|
||||
- **Technology**: NestJS 10 + Drizzle ORM + Redis
|
||||
- **Purpose**: API Key Management, Usage Tracking, Rate Limiting, Credit-based Billing
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────┐
|
||||
│ API Gateway │
|
||||
│ (Port 3030) │
|
||||
│ │
|
||||
Clients ───────────>│ • API Key Validation │
|
||||
(X-API-Key Header) │ • Rate Limiting │
|
||||
│ • Usage Tracking │
|
||||
│ • Credit Deduction │
|
||||
└───────────┬─────────────┘
|
||||
│
|
||||
┌─────────────────────┼─────────────────────┐
|
||||
│ │ │
|
||||
▼ ▼ ▼
|
||||
┌───────────────┐ ┌───────────────┐ ┌───────────────┐
|
||||
│ mana-search │ │ mana-stt │ │ mana-tts │
|
||||
│ (Port 3021) │ │ (Port 3020) │ │ (Port 3022) │
|
||||
└───────────────┘ └───────────────┘ └───────────────┘
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Development
|
||||
|
||||
```bash
|
||||
# Install dependencies
|
||||
pnpm install
|
||||
|
||||
# Push database schema
|
||||
pnpm db:push
|
||||
|
||||
# Start in development mode
|
||||
pnpm dev
|
||||
```
|
||||
|
||||
### Production
|
||||
|
||||
```bash
|
||||
# Build
|
||||
pnpm build
|
||||
|
||||
# Start
|
||||
pnpm start
|
||||
```
|
||||
|
||||
## API Endpoints
|
||||
|
||||
### Public API (with API Key)
|
||||
|
||||
| Method | Endpoint | Description | Credits |
|
||||
|--------|----------|-------------|---------|
|
||||
| POST | `/v1/search` | Web search | 1 |
|
||||
| POST | `/v1/extract` | Content extraction | 1 |
|
||||
| POST | `/v1/extract/bulk` | Bulk extraction | 1 per URL |
|
||||
| GET | `/v1/search/engines` | Available search engines | 0 |
|
||||
| POST | `/v1/stt/transcribe` | Audio → Text | 10/min |
|
||||
| GET | `/v1/stt/models` | Available STT models | 0 |
|
||||
| GET | `/v1/stt/languages` | Supported languages | 0 |
|
||||
| POST | `/v1/tts/synthesize` | Text → Audio | 1/1000 chars |
|
||||
| GET | `/v1/tts/voices` | Available voices | 0 |
|
||||
| GET | `/v1/tts/languages` | Supported languages | 0 |
|
||||
|
||||
### Management API (with JWT)
|
||||
|
||||
| Method | Endpoint | Description |
|
||||
|--------|----------|-------------|
|
||||
| POST | `/api-keys` | Create new API key |
|
||||
| GET | `/api-keys` | List all API keys |
|
||||
| GET | `/api-keys/:id` | Get API key details |
|
||||
| PATCH | `/api-keys/:id` | Update API key |
|
||||
| DELETE | `/api-keys/:id` | Delete API key |
|
||||
| POST | `/api-keys/:id/regenerate` | Regenerate API key |
|
||||
| GET | `/api-keys/:id/usage` | Get usage statistics |
|
||||
| GET | `/api-keys/:id/usage/summary` | Get usage summary |
|
||||
|
||||
### Admin API (with JWT + Admin Role)
|
||||
|
||||
| Method | Endpoint | Description |
|
||||
|--------|----------|-------------|
|
||||
| GET | `/admin/api-keys` | List all API keys (paginated) |
|
||||
| GET | `/admin/api-keys/:id` | Get any API key details |
|
||||
| PATCH | `/admin/api-keys/:id` | Update any key (tier, credits, limits) |
|
||||
| DELETE | `/admin/api-keys/:id` | Delete any API key |
|
||||
| GET | `/admin/usage/summary` | System-wide usage stats |
|
||||
| GET | `/admin/usage/top-users` | Top users by usage |
|
||||
|
||||
### System
|
||||
|
||||
| Method | Endpoint | Description |
|
||||
|--------|----------|-------------|
|
||||
| GET | `/health` | Health check |
|
||||
| GET | `/metrics` | Prometheus metrics |
|
||||
| GET | `/docs` | Swagger/OpenAPI documentation |
|
||||
|
||||
## Pricing Tiers
|
||||
|
||||
| Tier | Rate Limit | Monthly Credits | Endpoints | Price |
|
||||
|------|------------|-----------------|-----------|-------|
|
||||
| Free | 10 req/min | 100 | Search only | Free |
|
||||
| Pro | 100 req/min | 5,000 | All | €19/month |
|
||||
| Enterprise | 1,000 req/min | 50,000 | All | €99/month |
|
||||
|
||||
## Credit Costs
|
||||
|
||||
| Operation | Cost |
|
||||
|-----------|------|
|
||||
| Search | 1 credit |
|
||||
| Extract | 1 credit |
|
||||
| STT | 10 credits/minute |
|
||||
| TTS | 1 credit/1000 chars |
|
||||
|
||||
## Usage Examples
|
||||
|
||||
### Create an API Key
|
||||
|
||||
```bash
|
||||
# First, get a JWT token from mana-core-auth
|
||||
TOKEN=$(curl -s -X POST http://localhost:3001/api/v1/auth/login \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"email": "user@example.com", "password": "password"}' | jq -r '.accessToken')
|
||||
|
||||
# Create an API key
|
||||
curl -X POST http://localhost:3030/api-keys \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"name": "My API Key", "tier": "free"}'
|
||||
```
|
||||
|
||||
### Use the API
|
||||
|
||||
```bash
|
||||
# Search
|
||||
curl -X POST http://localhost:3030/v1/search \
|
||||
-H "X-API-Key: sk_live_xxx" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"query": "quantum computing"}'
|
||||
|
||||
# Extract content
|
||||
curl -X POST http://localhost:3030/v1/extract \
|
||||
-H "X-API-Key: sk_live_xxx" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"url": "https://example.com/article"}'
|
||||
|
||||
# Text-to-Speech
|
||||
curl -X POST http://localhost:3030/v1/tts/synthesize \
|
||||
-H "X-API-Key: sk_live_xxx" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"text": "Hello, world!", "voice": "en-US-1"}' \
|
||||
--output audio.mp3
|
||||
|
||||
# Speech-to-Text
|
||||
curl -X POST http://localhost:3030/v1/stt/transcribe \
|
||||
-H "X-API-Key: sk_live_xxx" \
|
||||
-F "file=@audio.wav"
|
||||
```
|
||||
|
||||
### Check Usage
|
||||
|
||||
```bash
|
||||
curl http://localhost:3030/api-keys/{id}/usage \
|
||||
-H "Authorization: Bearer $TOKEN"
|
||||
```
|
||||
|
||||
## Environment Variables
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `PORT` | 3030 | API port |
|
||||
| `DATABASE_URL` | - | PostgreSQL connection URL |
|
||||
| `REDIS_HOST` | localhost | Redis host |
|
||||
| `REDIS_PORT` | 6379 | Redis port |
|
||||
| `SEARCH_SERVICE_URL` | http://localhost:3021 | mana-search URL |
|
||||
| `STT_SERVICE_URL` | http://localhost:3020 | mana-stt URL |
|
||||
| `TTS_SERVICE_URL` | http://localhost:3022 | mana-tts URL |
|
||||
| `MANA_CORE_AUTH_URL` | http://localhost:3001 | Auth service URL |
|
||||
| `ADMIN_USER_IDS` | - | Comma-separated admin user IDs |
|
||||
|
||||
## Development Commands
|
||||
|
||||
```bash
|
||||
# Install dependencies
|
||||
pnpm install
|
||||
|
||||
# Start development server
|
||||
pnpm dev
|
||||
|
||||
# Build for production
|
||||
pnpm build
|
||||
|
||||
# Start production server
|
||||
pnpm start
|
||||
|
||||
# Type checking
|
||||
pnpm type-check
|
||||
|
||||
# Linting
|
||||
pnpm lint
|
||||
|
||||
# Database commands
|
||||
pnpm db:push # Push schema to database
|
||||
pnpm db:generate # Generate migrations
|
||||
pnpm db:migrate # Run migrations
|
||||
pnpm db:studio # Open Drizzle Studio
|
||||
```
|
||||
|
||||
## Database Schema
|
||||
|
||||
The gateway uses its own schema (`api_gateway`) in the shared ManaCore database:
|
||||
|
||||
- `api_gateway.api_keys` - API key storage and configuration
|
||||
- `api_gateway.api_usage` - Detailed usage logs
|
||||
- `api_gateway.api_usage_daily` - Aggregated daily usage for billing
|
||||
|
||||
## Rate Limiting
|
||||
|
||||
Rate limiting uses Redis with a sliding window algorithm:
|
||||
- Each API key has a configurable rate limit (requests per minute)
|
||||
- Rate limit headers are included in responses:
|
||||
- `X-RateLimit-Limit` - Maximum requests per minute
|
||||
- `X-RateLimit-Remaining` - Remaining requests
|
||||
- `X-RateLimit-Reset` - Unix timestamp when limit resets
|
||||
|
||||
## Authentication
|
||||
|
||||
Two types of authentication:
|
||||
1. **X-API-Key header** - For public API endpoints (`/v1/*`)
|
||||
2. **Bearer JWT token** - For management endpoints (`/api-keys/*`)
|
||||
|
||||
The JWT token is validated against mana-core-auth service.
|
||||
|
||||
## Project Structure
|
||||
|
||||
```
|
||||
services/mana-api-gateway/
|
||||
├── src/
|
||||
│ ├── main.ts # Application entry point
|
||||
│ ├── app.module.ts # Root module
|
||||
│ ├── config/
|
||||
│ │ ├── configuration.ts # App configuration
|
||||
│ │ └── pricing.ts # Pricing tiers and credit costs
|
||||
│ ├── db/
|
||||
│ │ ├── schema/ # Drizzle schemas
|
||||
│ │ ├── database.module.ts # Database provider
|
||||
│ │ ├── connection.ts # DB connection
|
||||
│ │ └── migrate.ts # Migration script
|
||||
│ ├── api-keys/ # API key management
|
||||
│ ├── usage/ # Usage tracking
|
||||
│ ├── proxy/ # Proxy services to backends
|
||||
│ ├── guards/ # Auth, rate limit, credits guards
|
||||
│ ├── common/ # Decorators, filters, interceptors
|
||||
│ ├── credits/ # Credits service (mana-core-auth client)
|
||||
│ ├── metrics/ # Prometheus metrics
|
||||
│ └── health/ # Health check endpoint
|
||||
├── drizzle.config.ts # Drizzle Kit configuration
|
||||
├── package.json
|
||||
├── tsconfig.json
|
||||
└── Dockerfile
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### API Key not working
|
||||
|
||||
1. Check the key is valid: `curl -H "X-API-Key: $KEY" http://localhost:3030/health`
|
||||
2. Check the key is active in the database
|
||||
3. Check the key hasn't expired
|
||||
4. Check the endpoint is allowed for the key's tier
|
||||
|
||||
### Rate limit exceeded
|
||||
|
||||
Wait for the `X-RateLimit-Reset` timestamp, or upgrade to a higher tier.
|
||||
|
||||
### Credits exhausted
|
||||
|
||||
Check usage with the management API, or wait for monthly reset.
|
||||
|
|
@ -1,49 +0,0 @@
|
|||
# syntax=docker/dockerfile:1
|
||||
# ================================
|
||||
# Build Stage (Monorepo-aware)
|
||||
# ================================
|
||||
FROM node:20-alpine AS base
|
||||
RUN corepack enable && corepack prepare pnpm@9.15.0 --activate
|
||||
WORKDIR /app
|
||||
|
||||
# Install dependencies
|
||||
FROM base AS deps
|
||||
COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
|
||||
COPY services/mana-api-gateway/package.json ./services/mana-api-gateway/
|
||||
COPY packages/shared-nestjs-auth/package.json ./packages/shared-nestjs-auth/
|
||||
COPY packages/shared-drizzle-config/package.json ./packages/shared-drizzle-config/
|
||||
RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --frozen-lockfile --filter @manacore/api-gateway...
|
||||
|
||||
# Build the application
|
||||
FROM base AS builder
|
||||
COPY --from=deps /app/node_modules ./node_modules
|
||||
COPY --from=deps /app/services/mana-api-gateway/node_modules ./services/mana-api-gateway/node_modules
|
||||
COPY --from=deps /app/packages/shared-nestjs-auth/node_modules ./packages/shared-nestjs-auth/node_modules 2>/dev/null || true
|
||||
COPY packages/shared-nestjs-auth ./packages/shared-nestjs-auth
|
||||
COPY packages/shared-drizzle-config ./packages/shared-drizzle-config
|
||||
COPY services/mana-api-gateway ./services/mana-api-gateway
|
||||
WORKDIR /app/services/mana-api-gateway
|
||||
RUN pnpm build
|
||||
|
||||
# ================================
|
||||
# Production Stage
|
||||
# ================================
|
||||
FROM node:20-alpine AS runner
|
||||
RUN corepack enable && corepack prepare pnpm@9.15.0 --activate
|
||||
ENV NODE_ENV=production
|
||||
|
||||
# Create non-root user
|
||||
RUN addgroup --system --gid 1001 nodejs
|
||||
RUN adduser --system --uid 1001 nestjs
|
||||
USER nestjs
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy built application
|
||||
COPY --from=builder --chown=nestjs:nodejs /app/services/mana-api-gateway/dist ./dist
|
||||
COPY --from=builder --chown=nestjs:nodejs /app/services/mana-api-gateway/node_modules ./node_modules
|
||||
COPY --from=builder --chown=nestjs:nodejs /app/services/mana-api-gateway/package.json ./
|
||||
|
||||
EXPOSE 3030
|
||||
|
||||
CMD ["node", "dist/main.js"]
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
import { createDrizzleConfig } from '@manacore/shared-drizzle-config';
|
||||
|
||||
export default createDrizzleConfig({
|
||||
dbName: 'manacore',
|
||||
schemaFilter: ['api_gateway'],
|
||||
});
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
{
|
||||
"$schema": "https://json.schemastore.org/nest-cli",
|
||||
"collection": "@nestjs/schematics",
|
||||
"sourceRoot": "src",
|
||||
"compilerOptions": {
|
||||
"deleteOutDir": true
|
||||
}
|
||||
}
|
||||
|
|
@ -1,61 +0,0 @@
|
|||
{
|
||||
"name": "@manacore/api-gateway",
|
||||
"version": "1.0.0",
|
||||
"description": "ManaCore API Gateway for monetizing core services",
|
||||
"private": true,
|
||||
"license": "UNLICENSED",
|
||||
"scripts": {
|
||||
"build": "nest build",
|
||||
"dev": "nest start --watch",
|
||||
"start": "node dist/main",
|
||||
"start:dev": "nest start --watch",
|
||||
"start:debug": "nest start --debug --watch",
|
||||
"start:prod": "node dist/main",
|
||||
"lint": "eslint \"{src,test}/**/*.ts\" --fix",
|
||||
"type-check": "tsc --noEmit",
|
||||
"test": "jest",
|
||||
"test:watch": "jest --watch",
|
||||
"test:cov": "jest --coverage",
|
||||
"db:push": "drizzle-kit push",
|
||||
"db:generate": "drizzle-kit generate",
|
||||
"db:migrate": "tsx src/db/migrate.ts",
|
||||
"db:studio": "drizzle-kit studio"
|
||||
},
|
||||
"dependencies": {
|
||||
"@manacore/shared-nestjs-auth": "workspace:*",
|
||||
"@nestjs/common": "^10.4.17",
|
||||
"@nestjs/config": "^3.3.0",
|
||||
"@nestjs/core": "^10.4.17",
|
||||
"@nestjs/platform-express": "^10.4.17",
|
||||
"@nestjs/schedule": "^4.1.2",
|
||||
"@nestjs/swagger": "^11.2.5",
|
||||
"class-transformer": "^0.5.1",
|
||||
"class-validator": "^0.14.1",
|
||||
"drizzle-orm": "^0.38.4",
|
||||
"ioredis": "^5.4.2",
|
||||
"multer": "^1.4.5-lts.1",
|
||||
"postgres": "^3.4.5",
|
||||
"prom-client": "^15.1.3",
|
||||
"reflect-metadata": "^0.2.2",
|
||||
"rxjs": "^7.8.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@manacore/shared-drizzle-config": "workspace:*",
|
||||
"@nestjs/cli": "^10.4.9",
|
||||
"@nestjs/schematics": "^10.2.3",
|
||||
"@nestjs/testing": "^10.4.17",
|
||||
"@types/express": "^5.0.0",
|
||||
"@types/multer": "^1.4.12",
|
||||
"@types/node": "^22.10.7",
|
||||
"drizzle-kit": "^0.30.4",
|
||||
"jest": "^29.7.0",
|
||||
"ts-jest": "^29.2.5",
|
||||
"ts-node": "^10.9.2",
|
||||
"tsx": "^4.19.2",
|
||||
"typescript": "^5.7.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0",
|
||||
"pnpm": ">=9.0.0"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,118 +0,0 @@
|
|||
import { Controller, Get, Patch, Delete, Param, Query, Body, UseGuards } from '@nestjs/common';
|
||||
import {
|
||||
ApiTags,
|
||||
ApiBearerAuth,
|
||||
ApiOperation,
|
||||
ApiResponse,
|
||||
ApiParam,
|
||||
ApiQuery,
|
||||
} from '@nestjs/swagger';
|
||||
import { JwtAuthGuard, CurrentUser, CurrentUserData } from '@manacore/shared-nestjs-auth';
|
||||
import { AdminGuard } from '../guards/admin.guard';
|
||||
import { AdminService } from './admin.service';
|
||||
import { AdminUpdateKeyDto } from './dto/admin-update-key.dto';
|
||||
|
||||
@ApiTags('Admin')
|
||||
@ApiBearerAuth('jwt')
|
||||
@Controller('admin')
|
||||
@UseGuards(JwtAuthGuard, AdminGuard)
|
||||
export class AdminController {
|
||||
constructor(private readonly adminService: AdminService) {}
|
||||
|
||||
@Get('api-keys')
|
||||
@ApiOperation({
|
||||
summary: 'List all API keys',
|
||||
description: 'Returns all API keys in the system. Requires admin role.',
|
||||
})
|
||||
@ApiQuery({ name: 'page', required: false, type: Number })
|
||||
@ApiQuery({ name: 'limit', required: false, type: Number })
|
||||
@ApiQuery({ name: 'userId', required: false, type: String })
|
||||
@ApiQuery({ name: 'tier', required: false, enum: ['free', 'pro', 'enterprise'] })
|
||||
@ApiQuery({ name: 'active', required: false, type: Boolean })
|
||||
@ApiResponse({ status: 200, description: 'List of all API keys' })
|
||||
@ApiResponse({ status: 403, description: 'Forbidden - admin role required' })
|
||||
async listAllKeys(
|
||||
@Query('page') page?: string,
|
||||
@Query('limit') limit?: string,
|
||||
@Query('userId') userId?: string,
|
||||
@Query('tier') tier?: string,
|
||||
@Query('active') active?: string
|
||||
) {
|
||||
const pageNum = parseInt(page || '1', 10);
|
||||
const limitNum = parseInt(limit || '50', 10);
|
||||
const isActive = active === undefined ? undefined : active === 'true';
|
||||
|
||||
const result = await this.adminService.listAllKeys({
|
||||
page: pageNum,
|
||||
limit: limitNum,
|
||||
userId,
|
||||
tier,
|
||||
active: isActive,
|
||||
});
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
@Get('api-keys/:id')
|
||||
@ApiOperation({
|
||||
summary: 'Get API key details (admin)',
|
||||
description: 'Returns full details of any API key including usage stats.',
|
||||
})
|
||||
@ApiParam({ name: 'id', description: 'API key ID (UUID)' })
|
||||
@ApiResponse({ status: 200, description: 'API key details with usage' })
|
||||
@ApiResponse({ status: 404, description: 'API key not found' })
|
||||
async getKey(@Param('id') id: string) {
|
||||
return this.adminService.getKeyDetails(id);
|
||||
}
|
||||
|
||||
@Patch('api-keys/:id')
|
||||
@ApiOperation({
|
||||
summary: 'Update API key (admin)',
|
||||
description: 'Update any API key including tier, credits, rate limits.',
|
||||
})
|
||||
@ApiParam({ name: 'id', description: 'API key ID (UUID)' })
|
||||
@ApiResponse({ status: 200, description: 'API key updated' })
|
||||
@ApiResponse({ status: 404, description: 'API key not found' })
|
||||
async updateKey(@Param('id') id: string, @Body() dto: AdminUpdateKeyDto) {
|
||||
return this.adminService.updateKey(id, dto);
|
||||
}
|
||||
|
||||
@Delete('api-keys/:id')
|
||||
@ApiOperation({
|
||||
summary: 'Delete API key (admin)',
|
||||
description: 'Permanently delete any API key.',
|
||||
})
|
||||
@ApiParam({ name: 'id', description: 'API key ID (UUID)' })
|
||||
@ApiResponse({ status: 200, description: 'API key deleted' })
|
||||
@ApiResponse({ status: 404, description: 'API key not found' })
|
||||
async deleteKey(@Param('id') id: string) {
|
||||
await this.adminService.deleteKey(id);
|
||||
return { message: 'API key deleted successfully' };
|
||||
}
|
||||
|
||||
@Get('usage/summary')
|
||||
@ApiOperation({
|
||||
summary: 'Get system-wide usage summary',
|
||||
description: 'Returns aggregated usage stats for all API keys.',
|
||||
})
|
||||
@ApiQuery({ name: 'days', required: false, type: Number })
|
||||
@ApiResponse({ status: 200, description: 'System usage summary' })
|
||||
async getSystemUsage(@Query('days') days?: string) {
|
||||
const daysNum = parseInt(days || '30', 10);
|
||||
return this.adminService.getSystemUsage(daysNum);
|
||||
}
|
||||
|
||||
@Get('usage/top-users')
|
||||
@ApiOperation({
|
||||
summary: 'Get top users by usage',
|
||||
description: 'Returns users with highest API usage.',
|
||||
})
|
||||
@ApiQuery({ name: 'limit', required: false, type: Number })
|
||||
@ApiQuery({ name: 'days', required: false, type: Number })
|
||||
@ApiResponse({ status: 200, description: 'Top users by usage' })
|
||||
async getTopUsers(@Query('limit') limit?: string, @Query('days') days?: string) {
|
||||
const limitNum = parseInt(limit || '10', 10);
|
||||
const daysNum = parseInt(days || '30', 10);
|
||||
return this.adminService.getTopUsers(limitNum, daysNum);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
import { Module } from '@nestjs/common';
|
||||
import { AdminController } from './admin.controller';
|
||||
import { AdminService } from './admin.service';
|
||||
|
||||
@Module({
|
||||
controllers: [AdminController],
|
||||
providers: [AdminService],
|
||||
exports: [AdminService],
|
||||
})
|
||||
export class AdminModule {}
|
||||
|
|
@ -1,264 +0,0 @@
|
|||
import { Injectable, Inject, NotFoundException } from '@nestjs/common';
|
||||
import { eq, sql, desc, and, gte } from 'drizzle-orm';
|
||||
import { DATABASE_CONNECTION } from '../db/database.module';
|
||||
import { apiKeys, apiUsage, apiUsageDaily } from '../db/schema';
|
||||
import { PRICING_TIERS, PricingTier } from '../config/pricing';
|
||||
import { AdminUpdateKeyDto } from './dto/admin-update-key.dto';
|
||||
|
||||
interface ListKeysOptions {
|
||||
page: number;
|
||||
limit: number;
|
||||
userId?: string;
|
||||
tier?: string;
|
||||
active?: boolean;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class AdminService {
|
||||
constructor(
|
||||
@Inject(DATABASE_CONNECTION)
|
||||
private readonly db: ReturnType<typeof import('../db/connection').getDb>
|
||||
) {}
|
||||
|
||||
async listAllKeys(options: ListKeysOptions) {
|
||||
const { page, limit, userId, tier, active } = options;
|
||||
const offset = (page - 1) * limit;
|
||||
|
||||
// Build conditions
|
||||
const conditions = [];
|
||||
if (userId) {
|
||||
conditions.push(eq(apiKeys.userId, userId));
|
||||
}
|
||||
if (tier) {
|
||||
conditions.push(eq(apiKeys.tier, tier));
|
||||
}
|
||||
if (active !== undefined) {
|
||||
conditions.push(eq(apiKeys.active, active));
|
||||
}
|
||||
|
||||
const whereClause = conditions.length > 0 ? and(...conditions) : undefined;
|
||||
|
||||
const [keys, countResult] = await Promise.all([
|
||||
this.db
|
||||
.select({
|
||||
id: apiKeys.id,
|
||||
name: apiKeys.name,
|
||||
keyPrefix: apiKeys.keyPrefix,
|
||||
userId: apiKeys.userId,
|
||||
organizationId: apiKeys.organizationId,
|
||||
tier: apiKeys.tier,
|
||||
rateLimit: apiKeys.rateLimit,
|
||||
monthlyCredits: apiKeys.monthlyCredits,
|
||||
creditsUsed: apiKeys.creditsUsed,
|
||||
active: apiKeys.active,
|
||||
lastUsedAt: apiKeys.lastUsedAt,
|
||||
createdAt: apiKeys.createdAt,
|
||||
})
|
||||
.from(apiKeys)
|
||||
.where(whereClause)
|
||||
.orderBy(desc(apiKeys.createdAt))
|
||||
.limit(limit)
|
||||
.offset(offset),
|
||||
this.db
|
||||
.select({ count: sql<number>`count(*)` })
|
||||
.from(apiKeys)
|
||||
.where(whereClause),
|
||||
]);
|
||||
|
||||
const total = Number(countResult[0]?.count || 0);
|
||||
|
||||
return {
|
||||
apiKeys: keys,
|
||||
pagination: {
|
||||
page,
|
||||
limit,
|
||||
total,
|
||||
totalPages: Math.ceil(total / limit),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async getKeyDetails(id: string) {
|
||||
const key = await this.db.select().from(apiKeys).where(eq(apiKeys.id, id)).limit(1);
|
||||
|
||||
if (!key.length) {
|
||||
throw new NotFoundException('API key not found');
|
||||
}
|
||||
|
||||
// Get recent usage
|
||||
const thirtyDaysAgo = new Date();
|
||||
thirtyDaysAgo.setDate(thirtyDaysAgo.getDate() - 30);
|
||||
|
||||
const recentUsage = await this.db
|
||||
.select({
|
||||
endpoint: apiUsageDaily.endpoint,
|
||||
requestCount: sql<number>`sum(${apiUsageDaily.requestCount})`,
|
||||
creditsUsed: sql<number>`sum(${apiUsageDaily.creditsUsed})`,
|
||||
})
|
||||
.from(apiUsageDaily)
|
||||
.where(
|
||||
and(
|
||||
eq(apiUsageDaily.apiKeyId, id),
|
||||
gte(apiUsageDaily.date, thirtyDaysAgo.toISOString().split('T')[0])
|
||||
)
|
||||
)
|
||||
.groupBy(apiUsageDaily.endpoint);
|
||||
|
||||
return {
|
||||
apiKey: {
|
||||
...key[0],
|
||||
keyHash: undefined, // Don't expose hash
|
||||
},
|
||||
usage: {
|
||||
last30Days: recentUsage,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async updateKey(id: string, dto: AdminUpdateKeyDto) {
|
||||
// Verify key exists
|
||||
const existing = await this.db.select().from(apiKeys).where(eq(apiKeys.id, id)).limit(1);
|
||||
|
||||
if (!existing.length) {
|
||||
throw new NotFoundException('API key not found');
|
||||
}
|
||||
|
||||
// Build update object
|
||||
const updates: Record<string, unknown> = {
|
||||
updatedAt: new Date(),
|
||||
};
|
||||
|
||||
if (dto.name !== undefined) updates.name = dto.name;
|
||||
if (dto.description !== undefined) updates.description = dto.description;
|
||||
if (dto.active !== undefined) updates.active = dto.active;
|
||||
if (dto.expiresAt !== undefined) updates.expiresAt = new Date(dto.expiresAt);
|
||||
if (dto.rateLimit !== undefined) updates.rateLimit = dto.rateLimit;
|
||||
if (dto.monthlyCredits !== undefined) updates.monthlyCredits = dto.monthlyCredits;
|
||||
|
||||
if (dto.allowedEndpoints !== undefined) {
|
||||
updates.allowedEndpoints = JSON.stringify(dto.allowedEndpoints);
|
||||
}
|
||||
|
||||
if (dto.allowedIps !== undefined) {
|
||||
updates.allowedIps = JSON.stringify(dto.allowedIps);
|
||||
}
|
||||
|
||||
if (dto.resetCredits) {
|
||||
updates.creditsUsed = 0;
|
||||
}
|
||||
|
||||
// If tier is changed, apply tier defaults
|
||||
if (dto.tier !== undefined) {
|
||||
const tierConfig = PRICING_TIERS[dto.tier as PricingTier];
|
||||
updates.tier = dto.tier;
|
||||
// Only apply tier defaults if not explicitly set
|
||||
if (dto.rateLimit === undefined) updates.rateLimit = tierConfig.rateLimit;
|
||||
if (dto.monthlyCredits === undefined) updates.monthlyCredits = tierConfig.monthlyCredits;
|
||||
}
|
||||
|
||||
const [updated] = await this.db
|
||||
.update(apiKeys)
|
||||
.set(updates)
|
||||
.where(eq(apiKeys.id, id))
|
||||
.returning();
|
||||
|
||||
return {
|
||||
apiKey: {
|
||||
...updated,
|
||||
keyHash: undefined,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async deleteKey(id: string) {
|
||||
const existing = await this.db.select().from(apiKeys).where(eq(apiKeys.id, id)).limit(1);
|
||||
|
||||
if (!existing.length) {
|
||||
throw new NotFoundException('API key not found');
|
||||
}
|
||||
|
||||
// Delete usage data first (foreign key constraint)
|
||||
await this.db.delete(apiUsageDaily).where(eq(apiUsageDaily.apiKeyId, id));
|
||||
await this.db.delete(apiUsage).where(eq(apiUsage.apiKeyId, id));
|
||||
|
||||
// Delete the key
|
||||
await this.db.delete(apiKeys).where(eq(apiKeys.id, id));
|
||||
}
|
||||
|
||||
async getSystemUsage(days: number) {
|
||||
const startDate = new Date();
|
||||
startDate.setDate(startDate.getDate() - days);
|
||||
|
||||
const dailyStats = await this.db
|
||||
.select({
|
||||
date: apiUsageDaily.date,
|
||||
requestCount: sql<number>`sum(${apiUsageDaily.requestCount})`,
|
||||
creditsUsed: sql<number>`sum(${apiUsageDaily.creditsUsed})`,
|
||||
errorCount: sql<number>`sum(${apiUsageDaily.errorCount})`,
|
||||
})
|
||||
.from(apiUsageDaily)
|
||||
.where(gte(apiUsageDaily.date, startDate.toISOString().split('T')[0]))
|
||||
.groupBy(apiUsageDaily.date)
|
||||
.orderBy(apiUsageDaily.date);
|
||||
|
||||
const endpointStats = await this.db
|
||||
.select({
|
||||
endpoint: apiUsageDaily.endpoint,
|
||||
requestCount: sql<number>`sum(${apiUsageDaily.requestCount})`,
|
||||
creditsUsed: sql<number>`sum(${apiUsageDaily.creditsUsed})`,
|
||||
})
|
||||
.from(apiUsageDaily)
|
||||
.where(gte(apiUsageDaily.date, startDate.toISOString().split('T')[0]))
|
||||
.groupBy(apiUsageDaily.endpoint);
|
||||
|
||||
const tierStats = await this.db
|
||||
.select({
|
||||
tier: apiKeys.tier,
|
||||
keyCount: sql<number>`count(*)`,
|
||||
activeCount: sql<number>`sum(case when ${apiKeys.active} then 1 else 0 end)`,
|
||||
})
|
||||
.from(apiKeys)
|
||||
.groupBy(apiKeys.tier);
|
||||
|
||||
return {
|
||||
period: {
|
||||
start: startDate.toISOString().split('T')[0],
|
||||
end: new Date().toISOString().split('T')[0],
|
||||
days,
|
||||
},
|
||||
daily: dailyStats,
|
||||
byEndpoint: endpointStats,
|
||||
byTier: tierStats,
|
||||
};
|
||||
}
|
||||
|
||||
async getTopUsers(limit: number, days: number) {
|
||||
const startDate = new Date();
|
||||
startDate.setDate(startDate.getDate() - days);
|
||||
|
||||
const topUsers = await this.db
|
||||
.select({
|
||||
apiKeyId: apiUsageDaily.apiKeyId,
|
||||
keyName: apiKeys.name,
|
||||
userId: apiKeys.userId,
|
||||
tier: apiKeys.tier,
|
||||
requestCount: sql<number>`sum(${apiUsageDaily.requestCount})`,
|
||||
creditsUsed: sql<number>`sum(${apiUsageDaily.creditsUsed})`,
|
||||
})
|
||||
.from(apiUsageDaily)
|
||||
.innerJoin(apiKeys, eq(apiUsageDaily.apiKeyId, apiKeys.id))
|
||||
.where(gte(apiUsageDaily.date, startDate.toISOString().split('T')[0]))
|
||||
.groupBy(apiUsageDaily.apiKeyId, apiKeys.name, apiKeys.userId, apiKeys.tier)
|
||||
.orderBy(desc(sql`sum(${apiUsageDaily.requestCount})`))
|
||||
.limit(limit);
|
||||
|
||||
return {
|
||||
period: {
|
||||
start: startDate.toISOString().split('T')[0],
|
||||
end: new Date().toISOString().split('T')[0],
|
||||
days,
|
||||
},
|
||||
topUsers,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
@ -1,100 +0,0 @@
|
|||
import {
|
||||
IsString,
|
||||
IsOptional,
|
||||
IsBoolean,
|
||||
IsArray,
|
||||
IsDateString,
|
||||
IsInt,
|
||||
IsEnum,
|
||||
Min,
|
||||
} from 'class-validator';
|
||||
import { ApiPropertyOptional } from '@nestjs/swagger';
|
||||
|
||||
export class AdminUpdateKeyDto {
|
||||
@ApiPropertyOptional({
|
||||
description: 'Update the display name',
|
||||
example: 'Updated API Key Name',
|
||||
})
|
||||
@IsString()
|
||||
@IsOptional()
|
||||
name?: string;
|
||||
|
||||
@ApiPropertyOptional({
|
||||
description: 'Update the description',
|
||||
example: 'Updated description',
|
||||
})
|
||||
@IsString()
|
||||
@IsOptional()
|
||||
description?: string;
|
||||
|
||||
@ApiPropertyOptional({
|
||||
description: 'Change the pricing tier',
|
||||
enum: ['free', 'pro', 'enterprise'],
|
||||
})
|
||||
@IsString()
|
||||
@IsOptional()
|
||||
@IsEnum(['free', 'pro', 'enterprise'])
|
||||
tier?: 'free' | 'pro' | 'enterprise';
|
||||
|
||||
@ApiPropertyOptional({
|
||||
description: 'Custom rate limit (requests per minute)',
|
||||
example: 100,
|
||||
})
|
||||
@IsInt()
|
||||
@IsOptional()
|
||||
@Min(1)
|
||||
rateLimit?: number;
|
||||
|
||||
@ApiPropertyOptional({
|
||||
description: 'Custom monthly credits limit',
|
||||
example: 5000,
|
||||
})
|
||||
@IsInt()
|
||||
@IsOptional()
|
||||
@Min(0)
|
||||
monthlyCredits?: number;
|
||||
|
||||
@ApiPropertyOptional({
|
||||
description: 'Reset credits used to 0',
|
||||
example: true,
|
||||
})
|
||||
@IsBoolean()
|
||||
@IsOptional()
|
||||
resetCredits?: boolean;
|
||||
|
||||
@ApiPropertyOptional({
|
||||
description: 'Update allowed endpoints',
|
||||
example: ['search', 'stt', 'tts'],
|
||||
type: [String],
|
||||
})
|
||||
@IsArray()
|
||||
@IsString({ each: true })
|
||||
@IsOptional()
|
||||
allowedEndpoints?: string[];
|
||||
|
||||
@ApiPropertyOptional({
|
||||
description: 'Update IP whitelist',
|
||||
example: ['192.168.1.0/24'],
|
||||
type: [String],
|
||||
})
|
||||
@IsArray()
|
||||
@IsString({ each: true })
|
||||
@IsOptional()
|
||||
allowedIps?: string[];
|
||||
|
||||
@ApiPropertyOptional({
|
||||
description: 'Enable or disable the API key',
|
||||
example: true,
|
||||
})
|
||||
@IsBoolean()
|
||||
@IsOptional()
|
||||
active?: boolean;
|
||||
|
||||
@ApiPropertyOptional({
|
||||
description: 'Update expiration date (ISO 8601)',
|
||||
example: '2025-12-31T23:59:59Z',
|
||||
})
|
||||
@IsDateString()
|
||||
@IsOptional()
|
||||
expiresAt?: string;
|
||||
}
|
||||
|
|
@ -1,160 +0,0 @@
|
|||
import {
|
||||
Controller,
|
||||
Get,
|
||||
Post,
|
||||
Patch,
|
||||
Delete,
|
||||
Body,
|
||||
Param,
|
||||
Query,
|
||||
UseGuards,
|
||||
} from '@nestjs/common';
|
||||
import {
|
||||
ApiTags,
|
||||
ApiBearerAuth,
|
||||
ApiOperation,
|
||||
ApiResponse,
|
||||
ApiParam,
|
||||
ApiQuery,
|
||||
} from '@nestjs/swagger';
|
||||
import { JwtAuthGuard, CurrentUser, CurrentUserData } from '@manacore/shared-nestjs-auth';
|
||||
import { ApiKeysService } from './api-keys.service';
|
||||
import { CreateApiKeyDto, UpdateApiKeyDto } from './dto';
|
||||
import { UsageService } from '../usage/usage.service';
|
||||
|
||||
@ApiTags('API Keys')
|
||||
@ApiBearerAuth('jwt')
|
||||
@Controller('api-keys')
|
||||
@UseGuards(JwtAuthGuard)
|
||||
export class ApiKeysController {
|
||||
constructor(
|
||||
private readonly apiKeyService: ApiKeysService,
|
||||
private readonly usageService: UsageService
|
||||
) {}
|
||||
|
||||
@Post()
|
||||
@ApiOperation({
|
||||
summary: 'Create API key',
|
||||
description: 'Creates a new API key. The full key is only returned once - save it securely.',
|
||||
})
|
||||
@ApiResponse({ status: 201, description: 'API key created successfully' })
|
||||
@ApiResponse({ status: 401, description: 'Unauthorized - invalid or missing JWT' })
|
||||
async create(@CurrentUser() user: CurrentUserData, @Body() dto: CreateApiKeyDto) {
|
||||
const result = await this.apiKeyService.create(user.userId, dto);
|
||||
return {
|
||||
message: 'API key created successfully. Save your key - it will not be shown again.',
|
||||
key: result.key,
|
||||
apiKey: result.apiKey,
|
||||
};
|
||||
}
|
||||
|
||||
@Get()
|
||||
@ApiOperation({
|
||||
summary: 'List API keys',
|
||||
description: 'Returns all API keys for the authenticated user (without full key values)',
|
||||
})
|
||||
@ApiResponse({ status: 200, description: 'List of API keys' })
|
||||
async list(@CurrentUser() user: CurrentUserData) {
|
||||
const keys = await this.apiKeyService.listByUser(user.userId);
|
||||
return { apiKeys: keys };
|
||||
}
|
||||
|
||||
@Get(':id')
|
||||
@ApiOperation({
|
||||
summary: 'Get API key details',
|
||||
description: 'Returns details for a specific API key',
|
||||
})
|
||||
@ApiParam({ name: 'id', description: 'API key ID (UUID)' })
|
||||
@ApiResponse({ status: 200, description: 'API key details' })
|
||||
@ApiResponse({ status: 404, description: 'API key not found' })
|
||||
async get(@CurrentUser() user: CurrentUserData, @Param('id') id: string) {
|
||||
const key = await this.apiKeyService.getByIdAndUser(id, user.userId);
|
||||
return { apiKey: key };
|
||||
}
|
||||
|
||||
@Patch(':id')
|
||||
@ApiOperation({
|
||||
summary: 'Update API key',
|
||||
description: 'Updates name, description, allowed endpoints, IP whitelist, or active status',
|
||||
})
|
||||
@ApiParam({ name: 'id', description: 'API key ID (UUID)' })
|
||||
@ApiResponse({ status: 200, description: 'API key updated successfully' })
|
||||
@ApiResponse({ status: 404, description: 'API key not found' })
|
||||
async update(
|
||||
@CurrentUser() user: CurrentUserData,
|
||||
@Param('id') id: string,
|
||||
@Body() dto: UpdateApiKeyDto
|
||||
) {
|
||||
const key = await this.apiKeyService.update(id, user.userId, dto);
|
||||
return { apiKey: key };
|
||||
}
|
||||
|
||||
@Delete(':id')
|
||||
@ApiOperation({
|
||||
summary: 'Delete API key',
|
||||
description: 'Permanently deletes an API key. This action cannot be undone.',
|
||||
})
|
||||
@ApiParam({ name: 'id', description: 'API key ID (UUID)' })
|
||||
@ApiResponse({ status: 200, description: 'API key deleted successfully' })
|
||||
@ApiResponse({ status: 404, description: 'API key not found' })
|
||||
async delete(@CurrentUser() user: CurrentUserData, @Param('id') id: string) {
|
||||
await this.apiKeyService.delete(id, user.userId);
|
||||
return { message: 'API key deleted successfully' };
|
||||
}
|
||||
|
||||
@Post(':id/regenerate')
|
||||
@ApiOperation({
|
||||
summary: 'Regenerate API key',
|
||||
description:
|
||||
'Generates a new key value for an existing API key. The old key immediately stops working.',
|
||||
})
|
||||
@ApiParam({ name: 'id', description: 'API key ID (UUID)' })
|
||||
@ApiResponse({ status: 200, description: 'New key generated successfully' })
|
||||
@ApiResponse({ status: 404, description: 'API key not found' })
|
||||
async regenerate(@CurrentUser() user: CurrentUserData, @Param('id') id: string) {
|
||||
const result = await this.apiKeyService.regenerate(id, user.userId);
|
||||
return {
|
||||
message: 'API key regenerated successfully. Save your new key - it will not be shown again.',
|
||||
key: result.key,
|
||||
apiKey: result.apiKey,
|
||||
};
|
||||
}
|
||||
|
||||
@Get(':id/usage')
|
||||
@ApiOperation({
|
||||
summary: 'Get daily usage',
|
||||
description: 'Returns daily usage statistics for an API key',
|
||||
})
|
||||
@ApiParam({ name: 'id', description: 'API key ID (UUID)' })
|
||||
@ApiQuery({ name: 'days', required: false, description: 'Number of days (default: 30)' })
|
||||
@ApiResponse({ status: 200, description: 'Daily usage statistics' })
|
||||
async getUsage(
|
||||
@CurrentUser() user: CurrentUserData,
|
||||
@Param('id') id: string,
|
||||
@Query('days') days?: string
|
||||
) {
|
||||
// Verify ownership
|
||||
await this.apiKeyService.getByIdAndUser(id, user.userId);
|
||||
|
||||
const daysNum = parseInt(days || '30', 10);
|
||||
const usage = await this.usageService.getDailyUsage(id, daysNum);
|
||||
|
||||
return { usage };
|
||||
}
|
||||
|
||||
@Get(':id/usage/summary')
|
||||
@ApiOperation({
|
||||
summary: 'Get usage summary',
|
||||
description: 'Returns aggregated usage summary for an API key',
|
||||
})
|
||||
@ApiParam({ name: 'id', description: 'API key ID (UUID)' })
|
||||
@ApiResponse({ status: 200, description: 'Usage summary' })
|
||||
async getUsageSummary(@CurrentUser() user: CurrentUserData, @Param('id') id: string) {
|
||||
// Verify ownership
|
||||
await this.apiKeyService.getByIdAndUser(id, user.userId);
|
||||
|
||||
const summary = await this.usageService.getUsageSummary(id);
|
||||
|
||||
return { summary };
|
||||
}
|
||||
}
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
import { Module, forwardRef } from '@nestjs/common';
|
||||
import { ApiKeysController } from './api-keys.controller';
|
||||
import { ApiKeysService } from './api-keys.service';
|
||||
import { UsageModule } from '../usage/usage.module';
|
||||
|
||||
@Module({
|
||||
imports: [forwardRef(() => UsageModule)],
|
||||
controllers: [ApiKeysController],
|
||||
providers: [ApiKeysService],
|
||||
exports: [ApiKeysService],
|
||||
})
|
||||
export class ApiKeysModule {}
|
||||
|
|
@ -1,277 +0,0 @@
|
|||
import { Injectable, Inject, NotFoundException, ForbiddenException } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { eq, and } from 'drizzle-orm';
|
||||
import * as crypto from 'crypto';
|
||||
import { DATABASE_CONNECTION } from '../db/database.module';
|
||||
import { apiKeys, ApiKey, NewApiKey } from '../db/schema';
|
||||
import { CreateApiKeyDto, UpdateApiKeyDto } from './dto';
|
||||
import { PRICING_TIERS, PricingTier } from '../config/pricing';
|
||||
|
||||
export interface ApiKeyData {
|
||||
id: string;
|
||||
userId: string | null;
|
||||
organizationId: string | null;
|
||||
name: string;
|
||||
tier: string;
|
||||
rateLimit: number;
|
||||
monthlyCredits: number;
|
||||
creditsUsed: number;
|
||||
allowedEndpoints: string | null;
|
||||
allowedIps: string | null;
|
||||
active: boolean;
|
||||
expiresAt: Date | null;
|
||||
lastUsedAt: Date | null;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class ApiKeysService {
|
||||
private readonly keyPrefixLive: string;
|
||||
private readonly keyPrefixTest: string;
|
||||
|
||||
constructor(
|
||||
@Inject(DATABASE_CONNECTION)
|
||||
private readonly db: ReturnType<typeof import('../db/connection').getDb>,
|
||||
private readonly configService: ConfigService
|
||||
) {
|
||||
this.keyPrefixLive = this.configService.get('apiKey.prefixLive') || 'sk_live_';
|
||||
this.keyPrefixTest = this.configService.get('apiKey.prefixTest') || 'sk_test_';
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a new API key
|
||||
*/
|
||||
private generateKey(isTest: boolean = false): { key: string; hash: string; prefix: string } {
|
||||
const prefix = isTest ? this.keyPrefixTest : this.keyPrefixLive;
|
||||
const randomPart = crypto.randomBytes(24).toString('base64url');
|
||||
const key = `${prefix}${randomPart}`;
|
||||
const hash = crypto.createHash('sha256').update(key).digest('hex');
|
||||
return { key, hash, prefix };
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new API key for a user
|
||||
*/
|
||||
async create(userId: string, dto: CreateApiKeyDto): Promise<{ key: string; apiKey: ApiKey }> {
|
||||
const { key, hash, prefix } = this.generateKey(dto.isTest);
|
||||
const tier = (dto.tier || 'free') as PricingTier;
|
||||
const tierConfig = PRICING_TIERS[tier];
|
||||
|
||||
const newKey: NewApiKey = {
|
||||
key: key,
|
||||
keyHash: hash,
|
||||
keyPrefix: prefix,
|
||||
userId,
|
||||
name: dto.name,
|
||||
description: dto.description,
|
||||
tier,
|
||||
rateLimit: tierConfig.rateLimit,
|
||||
monthlyCredits: tierConfig.monthlyCredits,
|
||||
creditsUsed: 0,
|
||||
creditsResetAt: this.getNextMonthReset(),
|
||||
allowedEndpoints: dto.allowedEndpoints
|
||||
? JSON.stringify(dto.allowedEndpoints)
|
||||
: JSON.stringify(tierConfig.endpoints),
|
||||
allowedIps: dto.allowedIps ? JSON.stringify(dto.allowedIps) : null,
|
||||
active: true,
|
||||
expiresAt: dto.expiresAt ? new Date(dto.expiresAt) : null,
|
||||
};
|
||||
|
||||
const [created] = await this.db.insert(apiKeys).values(newKey).returning();
|
||||
|
||||
// Return the full key only on creation (it's not stored)
|
||||
return {
|
||||
key,
|
||||
apiKey: { ...created, key: this.maskKey(key) },
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* List all API keys for a user (keys are masked)
|
||||
*/
|
||||
async listByUser(userId: string): Promise<ApiKey[]> {
|
||||
const keys = await this.db.select().from(apiKeys).where(eq(apiKeys.userId, userId));
|
||||
|
||||
return keys.map((k) => ({
|
||||
...k,
|
||||
key: this.maskKey(k.key),
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a single API key by ID (verified for user ownership)
|
||||
*/
|
||||
async getByIdAndUser(id: string, userId: string): Promise<ApiKey> {
|
||||
const [key] = await this.db
|
||||
.select()
|
||||
.from(apiKeys)
|
||||
.where(and(eq(apiKeys.id, id), eq(apiKeys.userId, userId)));
|
||||
|
||||
if (!key) {
|
||||
throw new NotFoundException('API key not found');
|
||||
}
|
||||
|
||||
return { ...key, key: this.maskKey(key.key) };
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate an API key and return its data
|
||||
*/
|
||||
async validateKey(rawKey: string): Promise<ApiKeyData | null> {
|
||||
const hash = crypto.createHash('sha256').update(rawKey).digest('hex');
|
||||
|
||||
const [key] = await this.db.select().from(apiKeys).where(eq(apiKeys.keyHash, hash));
|
||||
|
||||
if (!key) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Update last used timestamp
|
||||
await this.db.update(apiKeys).set({ lastUsedAt: new Date() }).where(eq(apiKeys.id, key.id));
|
||||
|
||||
return {
|
||||
id: key.id,
|
||||
userId: key.userId,
|
||||
organizationId: key.organizationId,
|
||||
name: key.name,
|
||||
tier: key.tier,
|
||||
rateLimit: key.rateLimit,
|
||||
monthlyCredits: key.monthlyCredits,
|
||||
creditsUsed: key.creditsUsed,
|
||||
allowedEndpoints: key.allowedEndpoints,
|
||||
allowedIps: key.allowedIps,
|
||||
active: key.active,
|
||||
expiresAt: key.expiresAt,
|
||||
lastUsedAt: key.lastUsedAt,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Update an API key
|
||||
*/
|
||||
async update(id: string, userId: string, dto: UpdateApiKeyDto): Promise<ApiKey> {
|
||||
// Verify ownership
|
||||
await this.getByIdAndUser(id, userId);
|
||||
|
||||
const updates: Partial<NewApiKey> = {
|
||||
updatedAt: new Date(),
|
||||
};
|
||||
|
||||
if (dto.name !== undefined) updates.name = dto.name;
|
||||
if (dto.description !== undefined) updates.description = dto.description;
|
||||
if (dto.allowedEndpoints !== undefined) {
|
||||
updates.allowedEndpoints = JSON.stringify(dto.allowedEndpoints);
|
||||
}
|
||||
if (dto.allowedIps !== undefined) {
|
||||
updates.allowedIps = JSON.stringify(dto.allowedIps);
|
||||
}
|
||||
if (dto.active !== undefined) updates.active = dto.active;
|
||||
if (dto.expiresAt !== undefined) {
|
||||
updates.expiresAt = dto.expiresAt ? new Date(dto.expiresAt) : null;
|
||||
}
|
||||
|
||||
const [updated] = await this.db
|
||||
.update(apiKeys)
|
||||
.set(updates)
|
||||
.where(eq(apiKeys.id, id))
|
||||
.returning();
|
||||
|
||||
return { ...updated, key: this.maskKey(updated.key) };
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete an API key
|
||||
*/
|
||||
async delete(id: string, userId: string): Promise<void> {
|
||||
// Verify ownership
|
||||
await this.getByIdAndUser(id, userId);
|
||||
|
||||
await this.db.delete(apiKeys).where(eq(apiKeys.id, id));
|
||||
}
|
||||
|
||||
/**
|
||||
* Regenerate an API key
|
||||
*/
|
||||
async regenerate(id: string, userId: string): Promise<{ key: string; apiKey: ApiKey }> {
|
||||
// Verify ownership
|
||||
const existing = await this.getByIdAndUser(id, userId);
|
||||
const isTest = existing.keyPrefix === this.keyPrefixTest;
|
||||
|
||||
const { key, hash, prefix } = this.generateKey(isTest);
|
||||
|
||||
const [updated] = await this.db
|
||||
.update(apiKeys)
|
||||
.set({
|
||||
key,
|
||||
keyHash: hash,
|
||||
keyPrefix: prefix,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(apiKeys.id, id))
|
||||
.returning();
|
||||
|
||||
return {
|
||||
key,
|
||||
apiKey: { ...updated, key: this.maskKey(key) },
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Increment credits used for an API key
|
||||
*/
|
||||
async incrementCreditsUsed(id: string, amount: number): Promise<void> {
|
||||
const [key] = await this.db.select().from(apiKeys).where(eq(apiKeys.id, id));
|
||||
|
||||
if (key) {
|
||||
// Check if we need to reset credits
|
||||
if (key.creditsResetAt && new Date() > key.creditsResetAt) {
|
||||
await this.db
|
||||
.update(apiKeys)
|
||||
.set({
|
||||
creditsUsed: amount,
|
||||
creditsResetAt: this.getNextMonthReset(),
|
||||
})
|
||||
.where(eq(apiKeys.id, id));
|
||||
} else {
|
||||
await this.db
|
||||
.update(apiKeys)
|
||||
.set({
|
||||
creditsUsed: key.creditsUsed + amount,
|
||||
})
|
||||
.where(eq(apiKeys.id, id));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if API key has enough credits
|
||||
*/
|
||||
async hasEnoughCredits(id: string, requiredCredits: number): Promise<boolean> {
|
||||
const [key] = await this.db.select().from(apiKeys).where(eq(apiKeys.id, id));
|
||||
|
||||
if (!key) return false;
|
||||
|
||||
// Check if we need to reset credits
|
||||
if (key.creditsResetAt && new Date() > key.creditsResetAt) {
|
||||
return true; // Credits will be reset
|
||||
}
|
||||
|
||||
return key.creditsUsed + requiredCredits <= key.monthlyCredits;
|
||||
}
|
||||
|
||||
/**
|
||||
* Mask an API key for display (show only prefix and last 4 chars)
|
||||
*/
|
||||
private maskKey(key: string): string {
|
||||
if (key.length <= 12) return key;
|
||||
const prefix = key.startsWith(this.keyPrefixTest) ? this.keyPrefixTest : this.keyPrefixLive;
|
||||
return `${prefix}...${key.slice(-4)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the next month reset date
|
||||
*/
|
||||
private getNextMonthReset(): Date {
|
||||
const now = new Date();
|
||||
return new Date(now.getFullYear(), now.getMonth() + 1, 1);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,66 +0,0 @@
|
|||
import { IsString, IsOptional, IsEnum, IsArray, IsDateString, IsBoolean } from 'class-validator';
|
||||
import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
|
||||
import { PricingTier } from '../../config/pricing';
|
||||
|
||||
export class CreateApiKeyDto {
|
||||
@ApiProperty({
|
||||
description: 'Display name for the API key',
|
||||
example: 'Production API Key',
|
||||
})
|
||||
@IsString()
|
||||
name: string;
|
||||
|
||||
@ApiPropertyOptional({
|
||||
description: 'Optional description for the API key',
|
||||
example: 'Used for production web application',
|
||||
})
|
||||
@IsString()
|
||||
@IsOptional()
|
||||
description?: string;
|
||||
|
||||
@ApiPropertyOptional({
|
||||
description: 'Pricing tier (determines rate limits and credits)',
|
||||
enum: ['free', 'pro', 'enterprise'],
|
||||
default: 'free',
|
||||
})
|
||||
@IsString()
|
||||
@IsOptional()
|
||||
@IsEnum(['free', 'pro', 'enterprise'])
|
||||
tier?: PricingTier;
|
||||
|
||||
@ApiPropertyOptional({
|
||||
description: 'List of allowed endpoints (null = all endpoints allowed)',
|
||||
example: ['search', 'tts'],
|
||||
type: [String],
|
||||
})
|
||||
@IsArray()
|
||||
@IsString({ each: true })
|
||||
@IsOptional()
|
||||
allowedEndpoints?: string[];
|
||||
|
||||
@ApiPropertyOptional({
|
||||
description: 'IP whitelist for this key (null = all IPs allowed)',
|
||||
example: ['192.168.1.0/24', '10.0.0.1'],
|
||||
type: [String],
|
||||
})
|
||||
@IsArray()
|
||||
@IsString({ each: true })
|
||||
@IsOptional()
|
||||
allowedIps?: string[];
|
||||
|
||||
@ApiPropertyOptional({
|
||||
description: 'Expiration date for the API key (ISO 8601)',
|
||||
example: '2025-12-31T23:59:59Z',
|
||||
})
|
||||
@IsDateString()
|
||||
@IsOptional()
|
||||
expiresAt?: string;
|
||||
|
||||
@ApiPropertyOptional({
|
||||
description: 'Create a test key (sk_test_ prefix) instead of live key',
|
||||
default: false,
|
||||
})
|
||||
@IsBoolean()
|
||||
@IsOptional()
|
||||
isTest?: boolean;
|
||||
}
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
export * from './create-api-key.dto';
|
||||
export * from './update-api-key.dto';
|
||||
|
|
@ -1,56 +0,0 @@
|
|||
import { IsString, IsOptional, IsBoolean, IsArray, IsDateString } from 'class-validator';
|
||||
import { ApiPropertyOptional } from '@nestjs/swagger';
|
||||
|
||||
export class UpdateApiKeyDto {
|
||||
@ApiPropertyOptional({
|
||||
description: 'Update the display name',
|
||||
example: 'Updated API Key Name',
|
||||
})
|
||||
@IsString()
|
||||
@IsOptional()
|
||||
name?: string;
|
||||
|
||||
@ApiPropertyOptional({
|
||||
description: 'Update the description',
|
||||
example: 'Updated description for this key',
|
||||
})
|
||||
@IsString()
|
||||
@IsOptional()
|
||||
description?: string;
|
||||
|
||||
@ApiPropertyOptional({
|
||||
description: 'Update allowed endpoints',
|
||||
example: ['search', 'stt', 'tts'],
|
||||
type: [String],
|
||||
})
|
||||
@IsArray()
|
||||
@IsString({ each: true })
|
||||
@IsOptional()
|
||||
allowedEndpoints?: string[];
|
||||
|
||||
@ApiPropertyOptional({
|
||||
description: 'Update IP whitelist',
|
||||
example: ['192.168.1.0/24'],
|
||||
type: [String],
|
||||
})
|
||||
@IsArray()
|
||||
@IsString({ each: true })
|
||||
@IsOptional()
|
||||
allowedIps?: string[];
|
||||
|
||||
@ApiPropertyOptional({
|
||||
description: 'Enable or disable the API key',
|
||||
example: true,
|
||||
})
|
||||
@IsBoolean()
|
||||
@IsOptional()
|
||||
active?: boolean;
|
||||
|
||||
@ApiPropertyOptional({
|
||||
description: 'Update expiration date (ISO 8601)',
|
||||
example: '2025-12-31T23:59:59Z',
|
||||
})
|
||||
@IsDateString()
|
||||
@IsOptional()
|
||||
expiresAt?: string;
|
||||
}
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
import { Module } from '@nestjs/common';
|
||||
import { ConfigModule } from '@nestjs/config';
|
||||
import configuration from './config/configuration';
|
||||
import { DatabaseModule } from './db/database.module';
|
||||
import { HealthModule } from './health/health.module';
|
||||
import { ApiKeysModule } from './api-keys/api-keys.module';
|
||||
import { UsageModule } from './usage/usage.module';
|
||||
import { ProxyModule } from './proxy/proxy.module';
|
||||
import { CreditsModule } from './credits/credits.module';
|
||||
import { MetricsModule } from './metrics/metrics.module';
|
||||
import { SchedulerModule } from './scheduler/scheduler.module';
|
||||
import { AdminModule } from './admin/admin.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
ConfigModule.forRoot({
|
||||
isGlobal: true,
|
||||
load: [configuration],
|
||||
}),
|
||||
DatabaseModule,
|
||||
HealthModule,
|
||||
ApiKeysModule,
|
||||
UsageModule,
|
||||
ProxyModule,
|
||||
CreditsModule,
|
||||
MetricsModule,
|
||||
SchedulerModule,
|
||||
AdminModule,
|
||||
],
|
||||
})
|
||||
export class AppModule {}
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
import { createParamDecorator, ExecutionContext } from '@nestjs/common';
|
||||
import { ApiKeyData } from '../../api-keys/api-keys.service';
|
||||
|
||||
/**
|
||||
* Parameter decorator to extract the validated API key data from the request.
|
||||
* Must be used with ApiKeyGuard.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* @Post('search')
|
||||
* @UseGuards(ApiKeyGuard)
|
||||
* search(@ApiKeyData() apiKey: ApiKeyData) {
|
||||
* return { keyId: apiKey.id };
|
||||
* }
|
||||
* ```
|
||||
*/
|
||||
export const ApiKeyParam = createParamDecorator(
|
||||
(data: unknown, ctx: ExecutionContext): ApiKeyData => {
|
||||
const request = ctx.switchToHttp().getRequest();
|
||||
return request.apiKey;
|
||||
}
|
||||
);
|
||||
|
|
@ -1,37 +0,0 @@
|
|||
import { ExceptionFilter, Catch, ArgumentsHost, HttpException, HttpStatus } from '@nestjs/common';
|
||||
import { Response } from 'express';
|
||||
|
||||
@Catch()
|
||||
export class HttpExceptionFilter implements ExceptionFilter {
|
||||
catch(exception: unknown, host: ArgumentsHost) {
|
||||
const ctx = host.switchToHttp();
|
||||
const response = ctx.getResponse<Response>();
|
||||
|
||||
let status = HttpStatus.INTERNAL_SERVER_ERROR;
|
||||
let message = 'Internal server error';
|
||||
let details: any = undefined;
|
||||
|
||||
if (exception instanceof HttpException) {
|
||||
status = exception.getStatus();
|
||||
const exceptionResponse = exception.getResponse();
|
||||
|
||||
if (typeof exceptionResponse === 'string') {
|
||||
message = exceptionResponse;
|
||||
} else if (typeof exceptionResponse === 'object') {
|
||||
const resp = exceptionResponse as any;
|
||||
message = resp.message || message;
|
||||
details = resp;
|
||||
}
|
||||
} else if (exception instanceof Error) {
|
||||
message = exception.message;
|
||||
console.error('Unhandled exception:', exception);
|
||||
}
|
||||
|
||||
response.status(status).json({
|
||||
statusCode: status,
|
||||
message,
|
||||
timestamp: new Date().toISOString(),
|
||||
...(details && status !== HttpStatus.INTERNAL_SERVER_ERROR && { details }),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -1,113 +0,0 @@
|
|||
import { Injectable, NestInterceptor, ExecutionContext, CallHandler } from '@nestjs/common';
|
||||
import { Observable } from 'rxjs';
|
||||
import { tap, catchError } from 'rxjs/operators';
|
||||
import { ApiKeyData } from '../../api-keys/api-keys.service';
|
||||
import { ApiKeysService } from '../../api-keys/api-keys.service';
|
||||
import { UsageService } from '../../usage/usage.service';
|
||||
import { CreditsService } from '../../credits/credits.service';
|
||||
import { CREDIT_COSTS } from '../../config/pricing';
|
||||
|
||||
@Injectable()
|
||||
export class UsageTrackingInterceptor implements NestInterceptor {
|
||||
constructor(
|
||||
private readonly usageService: UsageService,
|
||||
private readonly creditsService: CreditsService,
|
||||
private readonly apiKeysService: ApiKeysService
|
||||
) {}
|
||||
|
||||
intercept(context: ExecutionContext, next: CallHandler): Observable<any> {
|
||||
const request = context.switchToHttp().getRequest();
|
||||
const response = context.switchToHttp().getResponse();
|
||||
const apiKey = request.apiKey as ApiKeyData;
|
||||
const startTime = Date.now();
|
||||
|
||||
if (!apiKey) {
|
||||
return next.handle();
|
||||
}
|
||||
|
||||
return next.handle().pipe(
|
||||
tap(async (responseBody) => {
|
||||
const latencyMs = Date.now() - startTime;
|
||||
const endpoint = this.extractEndpoint(request.path);
|
||||
|
||||
// Calculate credits
|
||||
const creditsUsed = this.calculateCredits(endpoint, request, responseBody);
|
||||
|
||||
// Track usage
|
||||
await this.usageService.track({
|
||||
apiKeyId: apiKey.id,
|
||||
endpoint,
|
||||
method: request.method,
|
||||
path: request.path,
|
||||
latencyMs,
|
||||
statusCode: response.statusCode || 200,
|
||||
creditsUsed,
|
||||
metadata: {
|
||||
userAgent: request.headers['user-agent'],
|
||||
},
|
||||
});
|
||||
|
||||
// Increment credits used on the API key
|
||||
if (creditsUsed > 0) {
|
||||
await this.apiKeysService.incrementCreditsUsed(apiKey.id, creditsUsed);
|
||||
}
|
||||
|
||||
// Deduct credits from user account if applicable
|
||||
if (apiKey.userId && creditsUsed > 0) {
|
||||
try {
|
||||
await this.creditsService.deduct(apiKey.userId, creditsUsed, {
|
||||
appId: 'api-gateway',
|
||||
description: `API: ${endpoint}`,
|
||||
apiKeyId: apiKey.id,
|
||||
});
|
||||
} catch (error) {
|
||||
// Log but don't fail the request
|
||||
console.error('Failed to deduct credits from user account:', error);
|
||||
}
|
||||
}
|
||||
}),
|
||||
catchError(async (error) => {
|
||||
const latencyMs = Date.now() - startTime;
|
||||
const endpoint = this.extractEndpoint(request.path);
|
||||
|
||||
// Track failed requests (no credits deducted)
|
||||
await this.usageService.track({
|
||||
apiKeyId: apiKey.id,
|
||||
endpoint,
|
||||
method: request.method,
|
||||
path: request.path,
|
||||
latencyMs,
|
||||
statusCode: error.status || 500,
|
||||
creditsUsed: 0,
|
||||
metadata: {
|
||||
userAgent: request.headers['user-agent'],
|
||||
error: error.message,
|
||||
},
|
||||
});
|
||||
|
||||
throw error;
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
private extractEndpoint(path: string): string {
|
||||
const match = path.match(/\/v1\/(\w+)/);
|
||||
return match ? match[1] : 'unknown';
|
||||
}
|
||||
|
||||
private calculateCredits(endpoint: string, request: any, response: any): number {
|
||||
switch (endpoint) {
|
||||
case 'search':
|
||||
return CREDIT_COSTS.search;
|
||||
case 'tts':
|
||||
const text = request.body?.text || '';
|
||||
return Math.max(1, Math.ceil(text.length / 1000) * CREDIT_COSTS.tts.per1000Chars);
|
||||
case 'stt':
|
||||
// Calculate from actual audio duration if available in response
|
||||
const minutes = response?.duration ? response.duration / 60 : 1;
|
||||
return Math.max(1, Math.ceil(minutes) * CREDIT_COSTS.stt.perMinute);
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,47 +0,0 @@
|
|||
export default () => ({
|
||||
port: parseInt(process.env.PORT || '3030', 10),
|
||||
nodeEnv: process.env.NODE_ENV || 'development',
|
||||
|
||||
database: {
|
||||
url: process.env.DATABASE_URL || 'postgresql://manacore:devpassword@localhost:5432/manacore',
|
||||
},
|
||||
|
||||
cors: {
|
||||
origins: process.env.CORS_ORIGINS?.split(',') || [
|
||||
'http://localhost:3000',
|
||||
'http://localhost:5173',
|
||||
],
|
||||
},
|
||||
|
||||
redis: {
|
||||
host: process.env.REDIS_HOST || 'localhost',
|
||||
port: parseInt(process.env.REDIS_PORT || '6379', 10),
|
||||
password: process.env.REDIS_PASSWORD,
|
||||
keyPrefix: 'api-gateway:',
|
||||
},
|
||||
|
||||
services: {
|
||||
search: process.env.SEARCH_SERVICE_URL || 'http://localhost:3021',
|
||||
stt: process.env.STT_SERVICE_URL || 'http://localhost:3020',
|
||||
tts: process.env.TTS_SERVICE_URL || 'http://localhost:3022',
|
||||
},
|
||||
|
||||
auth: {
|
||||
url: process.env.MANA_CORE_AUTH_URL || 'http://localhost:3001',
|
||||
},
|
||||
|
||||
apiKey: {
|
||||
prefixLive: process.env.API_KEY_PREFIX_LIVE || 'sk_live_',
|
||||
prefixTest: process.env.API_KEY_PREFIX_TEST || 'sk_test_',
|
||||
},
|
||||
|
||||
defaults: {
|
||||
rateLimit: parseInt(process.env.DEFAULT_RATE_LIMIT || '10', 10),
|
||||
monthlyCredits: parseInt(process.env.DEFAULT_MONTHLY_CREDITS || '100', 10),
|
||||
},
|
||||
|
||||
admin: {
|
||||
// Comma-separated list of user IDs that have admin access
|
||||
userIds: process.env.ADMIN_USER_IDS || '',
|
||||
},
|
||||
});
|
||||
|
|
@ -1,41 +0,0 @@
|
|||
export const PRICING_TIERS = {
|
||||
free: {
|
||||
name: 'Free',
|
||||
rateLimit: 10, // 10 requests/minute
|
||||
monthlyCredits: 100,
|
||||
endpoints: ['search'] as const, // Only search
|
||||
features: [] as const,
|
||||
price: 0,
|
||||
},
|
||||
pro: {
|
||||
name: 'Pro',
|
||||
rateLimit: 100, // 100 requests/minute
|
||||
monthlyCredits: 5000,
|
||||
endpoints: ['search', 'stt', 'tts'] as const,
|
||||
features: ['priority_support'] as const,
|
||||
price: 1900, // 19 EUR in cents
|
||||
},
|
||||
enterprise: {
|
||||
name: 'Enterprise',
|
||||
rateLimit: 1000, // 1000 requests/minute
|
||||
monthlyCredits: 50000,
|
||||
endpoints: ['search', 'stt', 'tts'] as const,
|
||||
features: ['priority_support', 'sla', 'dedicated_support'] as const,
|
||||
price: 9900, // 99 EUR in cents
|
||||
},
|
||||
} as const;
|
||||
|
||||
export type PricingTier = keyof typeof PRICING_TIERS;
|
||||
|
||||
// Credit costs per operation
|
||||
export const CREDIT_COSTS = {
|
||||
search: 1, // 1 credit per search
|
||||
stt: {
|
||||
perMinute: 10, // 10 credits per minute of audio
|
||||
},
|
||||
tts: {
|
||||
per1000Chars: 1, // 1 credit per 1000 characters
|
||||
},
|
||||
} as const;
|
||||
|
||||
export type Endpoint = 'search' | 'stt' | 'tts';
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
import { Module } from '@nestjs/common';
|
||||
import { CreditsService } from './credits.service';
|
||||
|
||||
@Module({
|
||||
providers: [CreditsService],
|
||||
exports: [CreditsService],
|
||||
})
|
||||
export class CreditsModule {}
|
||||
|
|
@ -1,92 +0,0 @@
|
|||
import { Injectable, HttpException, HttpStatus } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
|
||||
export interface DeductCreditsOptions {
|
||||
appId: string;
|
||||
description: string;
|
||||
apiKeyId?: string;
|
||||
metadata?: Record<string, any>;
|
||||
}
|
||||
|
||||
export interface CreditBalance {
|
||||
balance: number;
|
||||
freeCreditsRemaining: number;
|
||||
dailyFreeCredits: number;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class CreditsService {
|
||||
private readonly authUrl: string;
|
||||
|
||||
constructor(private readonly configService: ConfigService) {
|
||||
this.authUrl = this.configService.get('auth.url') || 'http://localhost:3001';
|
||||
}
|
||||
|
||||
/**
|
||||
* Deduct credits from a user's account
|
||||
*/
|
||||
async deduct(userId: string, amount: number, options: DeductCreditsOptions): Promise<void> {
|
||||
const response = await fetch(`${this.authUrl}/api/v1/credits/consume`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
userId,
|
||||
amount,
|
||||
appId: options.appId,
|
||||
description: options.description,
|
||||
metadata: {
|
||||
...options.metadata,
|
||||
apiKeyId: options.apiKeyId,
|
||||
source: 'api-gateway',
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await response.text();
|
||||
console.error(`Failed to deduct credits for user ${userId}:`, error);
|
||||
// Don't throw - credit deduction failure shouldn't fail the request
|
||||
// The API key credits are already tracked
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a user's credit balance
|
||||
*/
|
||||
async getBalance(userId: string): Promise<CreditBalance | null> {
|
||||
try {
|
||||
const response = await fetch(`${this.authUrl}/api/v1/credits/balance/${userId}`);
|
||||
|
||||
if (!response.ok) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return response.json();
|
||||
} catch (error) {
|
||||
console.error(`Failed to get credit balance for user ${userId}:`, error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Add credits to a user's account (for testing/admin)
|
||||
*/
|
||||
async addCredits(userId: string, amount: number, reason: string): Promise<void> {
|
||||
const response = await fetch(`${this.authUrl}/api/v1/credits/add`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
userId,
|
||||
amount,
|
||||
appId: 'api-gateway',
|
||||
description: reason,
|
||||
type: 'bonus',
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await response.text();
|
||||
throw new HttpException(`Failed to add credits: ${error}`, HttpStatus.BAD_GATEWAY);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
import { drizzle } from 'drizzle-orm/postgres-js';
|
||||
import postgres from 'postgres';
|
||||
import * as schema from './schema';
|
||||
|
||||
let connection: ReturnType<typeof postgres> | null = null;
|
||||
let db: ReturnType<typeof drizzle> | null = null;
|
||||
|
||||
export function getConnection(databaseUrl: string) {
|
||||
if (!connection) {
|
||||
connection = postgres(databaseUrl, {
|
||||
max: 10,
|
||||
idle_timeout: 20,
|
||||
connect_timeout: 10,
|
||||
});
|
||||
}
|
||||
return connection;
|
||||
}
|
||||
|
||||
export function getDb(databaseUrl: string) {
|
||||
if (!db) {
|
||||
const conn = getConnection(databaseUrl);
|
||||
db = drizzle(conn, { schema });
|
||||
}
|
||||
return db;
|
||||
}
|
||||
|
||||
export async function closeConnection() {
|
||||
if (connection) {
|
||||
await connection.end();
|
||||
connection = null;
|
||||
db = null;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
import { Global, Module } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { getDb } from './connection';
|
||||
|
||||
export const DATABASE_CONNECTION = 'DATABASE_CONNECTION';
|
||||
|
||||
@Global()
|
||||
@Module({
|
||||
providers: [
|
||||
{
|
||||
provide: DATABASE_CONNECTION,
|
||||
useFactory: (configService: ConfigService) => {
|
||||
const databaseUrl = configService.get<string>('database.url');
|
||||
if (!databaseUrl) {
|
||||
throw new Error('DATABASE_URL is not configured');
|
||||
}
|
||||
return getDb(databaseUrl);
|
||||
},
|
||||
inject: [ConfigService],
|
||||
},
|
||||
],
|
||||
exports: [DATABASE_CONNECTION],
|
||||
})
|
||||
export class DatabaseModule {}
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
import 'dotenv/config';
|
||||
import { drizzle } from 'drizzle-orm/postgres-js';
|
||||
import { migrate } from 'drizzle-orm/postgres-js/migrator';
|
||||
import postgres from 'postgres';
|
||||
|
||||
async function main() {
|
||||
const databaseUrl =
|
||||
process.env.DATABASE_URL || 'postgresql://manacore:devpassword@localhost:5432/manacore';
|
||||
|
||||
console.log('Connecting to database...');
|
||||
const connection = postgres(databaseUrl, { max: 1 });
|
||||
const db = drizzle(connection);
|
||||
|
||||
console.log('Running migrations...');
|
||||
await migrate(db, { migrationsFolder: './src/db/migrations' });
|
||||
|
||||
console.log('Migrations complete!');
|
||||
await connection.end();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error('Migration failed:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
@ -1,51 +0,0 @@
|
|||
import { pgSchema, uuid, text, integer, boolean, timestamp, index } from 'drizzle-orm/pg-core';
|
||||
|
||||
export const apiGatewaySchema = pgSchema('api_gateway');
|
||||
|
||||
export const apiKeys = apiGatewaySchema.table(
|
||||
'api_keys',
|
||||
{
|
||||
id: uuid('id').defaultRandom().primaryKey(),
|
||||
|
||||
// Key identifiers
|
||||
key: text('key').notNull().unique(), // sk_live_xxx or sk_test_xxx
|
||||
keyHash: text('key_hash').notNull(), // SHA256 hash for lookup
|
||||
keyPrefix: text('key_prefix').notNull(), // sk_live_ or sk_test_
|
||||
|
||||
// Owner (can be user or organization)
|
||||
userId: text('user_id'), // B2C owner
|
||||
organizationId: text('organization_id'), // B2B owner
|
||||
|
||||
// Metadata
|
||||
name: text('name').notNull(), // "Production API Key"
|
||||
description: text('description'),
|
||||
|
||||
// Tier & Limits
|
||||
tier: text('tier').notNull().default('free'), // free, pro, enterprise
|
||||
rateLimit: integer('rate_limit').notNull().default(10), // requests/minute
|
||||
monthlyCredits: integer('monthly_credits').notNull().default(100),
|
||||
creditsUsed: integer('credits_used').notNull().default(0),
|
||||
creditsResetAt: timestamp('credits_reset_at', { withTimezone: true }),
|
||||
|
||||
// Permissions
|
||||
allowedEndpoints: text('allowed_endpoints'), // JSON array: ["search", "tts"]
|
||||
allowedIps: text('allowed_ips'), // JSON array or null for any
|
||||
|
||||
// Status
|
||||
active: boolean('active').notNull().default(true),
|
||||
expiresAt: timestamp('expires_at', { withTimezone: true }),
|
||||
lastUsedAt: timestamp('last_used_at', { withTimezone: true }),
|
||||
|
||||
// Timestamps
|
||||
createdAt: timestamp('created_at', { withTimezone: true }).defaultNow().notNull(),
|
||||
updatedAt: timestamp('updated_at', { withTimezone: true }).defaultNow().notNull(),
|
||||
},
|
||||
(table) => ({
|
||||
keyHashIdx: index('api_keys_key_hash_idx').on(table.keyHash),
|
||||
userIdIdx: index('api_keys_user_id_idx').on(table.userId),
|
||||
organizationIdIdx: index('api_keys_organization_id_idx').on(table.organizationId),
|
||||
})
|
||||
);
|
||||
|
||||
export type ApiKey = typeof apiKeys.$inferSelect;
|
||||
export type NewApiKey = typeof apiKeys.$inferInsert;
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
export * from './api-keys.schema';
|
||||
export * from './usage.schema';
|
||||
|
|
@ -1,70 +0,0 @@
|
|||
import { uuid, text, integer, timestamp, jsonb, index, unique, date } from 'drizzle-orm/pg-core';
|
||||
import { apiGatewaySchema, apiKeys } from './api-keys.schema';
|
||||
|
||||
// Detailed usage log
|
||||
export const apiUsage = apiGatewaySchema.table(
|
||||
'api_usage',
|
||||
{
|
||||
id: uuid('id').defaultRandom().primaryKey(),
|
||||
|
||||
// Key reference
|
||||
apiKeyId: uuid('api_key_id')
|
||||
.references(() => apiKeys.id, { onDelete: 'cascade' })
|
||||
.notNull(),
|
||||
|
||||
// Request details
|
||||
endpoint: text('endpoint').notNull(), // search, stt, tts
|
||||
method: text('method').notNull(), // POST, GET
|
||||
path: text('path').notNull(), // /v1/search
|
||||
|
||||
// Metrics
|
||||
requestSize: integer('request_size'), // bytes
|
||||
responseSize: integer('response_size'), // bytes
|
||||
latencyMs: integer('latency_ms'), // milliseconds
|
||||
statusCode: integer('status_code'), // 200, 400, 500...
|
||||
|
||||
// Credit calculation
|
||||
creditsUsed: integer('credits_used').notNull().default(0),
|
||||
creditReason: text('credit_reason'), // "1000 characters TTS"
|
||||
|
||||
// Metadata
|
||||
metadata: jsonb('metadata'), // additional context (user agent, etc.)
|
||||
|
||||
// Timestamp
|
||||
createdAt: timestamp('created_at', { withTimezone: true }).defaultNow().notNull(),
|
||||
},
|
||||
(table) => ({
|
||||
apiKeyIdIdx: index('api_usage_api_key_id_idx').on(table.apiKeyId),
|
||||
createdAtIdx: index('api_usage_created_at_idx').on(table.createdAt),
|
||||
endpointIdx: index('api_usage_endpoint_idx').on(table.endpoint),
|
||||
})
|
||||
);
|
||||
|
||||
// Aggregated daily usage (for dashboard/billing)
|
||||
export const apiUsageDaily = apiGatewaySchema.table(
|
||||
'api_usage_daily',
|
||||
{
|
||||
id: uuid('id').defaultRandom().primaryKey(),
|
||||
apiKeyId: uuid('api_key_id')
|
||||
.references(() => apiKeys.id, { onDelete: 'cascade' })
|
||||
.notNull(),
|
||||
date: date('date').notNull(),
|
||||
endpoint: text('endpoint').notNull(),
|
||||
|
||||
// Aggregates
|
||||
requestCount: integer('request_count').notNull().default(0),
|
||||
creditsUsed: integer('credits_used').notNull().default(0),
|
||||
totalLatencyMs: integer('total_latency_ms').notNull().default(0),
|
||||
errorCount: integer('error_count').notNull().default(0),
|
||||
},
|
||||
(table) => ({
|
||||
// Unique constraint for upsert
|
||||
uniqueDaily: unique('api_usage_daily_unique').on(table.apiKeyId, table.date, table.endpoint),
|
||||
dateIdx: index('api_usage_daily_date_idx').on(table.date),
|
||||
})
|
||||
);
|
||||
|
||||
export type ApiUsage = typeof apiUsage.$inferSelect;
|
||||
export type NewApiUsage = typeof apiUsage.$inferInsert;
|
||||
export type ApiUsageDaily = typeof apiUsageDaily.$inferSelect;
|
||||
export type NewApiUsageDaily = typeof apiUsageDaily.$inferInsert;
|
||||
|
|
@ -1,37 +0,0 @@
|
|||
import { Injectable, CanActivate, ExecutionContext, ForbiddenException } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
|
||||
@Injectable()
|
||||
export class AdminGuard implements CanActivate {
|
||||
private readonly adminUserIds: string[];
|
||||
|
||||
constructor(private readonly configService: ConfigService) {
|
||||
// Admin user IDs from environment variable (comma-separated)
|
||||
const adminIds = this.configService.get<string>('admin.userIds') || '';
|
||||
this.adminUserIds = adminIds
|
||||
.split(',')
|
||||
.map((id) => id.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
canActivate(context: ExecutionContext): boolean {
|
||||
const request = context.switchToHttp().getRequest();
|
||||
const user = request.user;
|
||||
|
||||
if (!user || !user.userId) {
|
||||
throw new ForbiddenException('User not authenticated');
|
||||
}
|
||||
|
||||
// Check if user has admin role
|
||||
if (user.role === 'admin') {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Check if user ID is in the admin list
|
||||
if (this.adminUserIds.includes(user.userId)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
throw new ForbiddenException('Admin access required');
|
||||
}
|
||||
}
|
||||
|
|
@ -1,91 +0,0 @@
|
|||
import {
|
||||
Injectable,
|
||||
CanActivate,
|
||||
ExecutionContext,
|
||||
UnauthorizedException,
|
||||
ForbiddenException,
|
||||
} from '@nestjs/common';
|
||||
import { ApiKeysService, ApiKeyData } from '../api-keys/api-keys.service';
|
||||
|
||||
@Injectable()
|
||||
export class ApiKeyGuard implements CanActivate {
|
||||
constructor(private readonly apiKeyService: ApiKeysService) {}
|
||||
|
||||
async canActivate(context: ExecutionContext): Promise<boolean> {
|
||||
const request = context.switchToHttp().getRequest();
|
||||
const apiKey = this.extractApiKey(request);
|
||||
|
||||
if (!apiKey) {
|
||||
throw new UnauthorizedException('API key required. Use X-API-Key header.');
|
||||
}
|
||||
|
||||
// Validate key
|
||||
const keyData = await this.apiKeyService.validateKey(apiKey);
|
||||
|
||||
if (!keyData) {
|
||||
throw new UnauthorizedException('Invalid API key');
|
||||
}
|
||||
|
||||
if (!keyData.active) {
|
||||
throw new UnauthorizedException('API key is disabled');
|
||||
}
|
||||
|
||||
if (keyData.expiresAt && new Date(keyData.expiresAt) < new Date()) {
|
||||
throw new UnauthorizedException('API key has expired');
|
||||
}
|
||||
|
||||
// Check endpoint permission
|
||||
const endpoint = this.extractEndpoint(request.path);
|
||||
if (!this.hasEndpointPermission(keyData, endpoint)) {
|
||||
throw new ForbiddenException(
|
||||
`Endpoint '${endpoint}' not allowed for this API key. Upgrade your plan to access this endpoint.`
|
||||
);
|
||||
}
|
||||
|
||||
// Check IP restriction
|
||||
if (!this.hasIpPermission(keyData, request)) {
|
||||
throw new ForbiddenException('Request from this IP address is not allowed');
|
||||
}
|
||||
|
||||
// Attach key data to request for later use
|
||||
request.apiKey = keyData;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private extractApiKey(request: any): string | undefined {
|
||||
return request.headers['x-api-key'];
|
||||
}
|
||||
|
||||
private extractEndpoint(path: string): string {
|
||||
// /v1/search -> search, /v1/stt/transcribe -> stt
|
||||
const match = path.match(/\/v1\/(\w+)/);
|
||||
return match ? match[1] : 'unknown';
|
||||
}
|
||||
|
||||
private hasEndpointPermission(keyData: ApiKeyData, endpoint: string): boolean {
|
||||
if (!keyData.allowedEndpoints) return true; // No restrictions
|
||||
try {
|
||||
const allowed = JSON.parse(keyData.allowedEndpoints) as string[];
|
||||
return allowed.includes(endpoint);
|
||||
} catch {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
private hasIpPermission(keyData: ApiKeyData, request: any): boolean {
|
||||
if (!keyData.allowedIps) return true; // No restrictions
|
||||
|
||||
try {
|
||||
const allowedIps = JSON.parse(keyData.allowedIps) as string[];
|
||||
const clientIp =
|
||||
request.headers['x-forwarded-for']?.split(',')[0]?.trim() ||
|
||||
request.connection?.remoteAddress ||
|
||||
request.ip;
|
||||
|
||||
return allowedIps.includes(clientIp);
|
||||
} catch {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,63 +0,0 @@
|
|||
import {
|
||||
Injectable,
|
||||
CanActivate,
|
||||
ExecutionContext,
|
||||
HttpException,
|
||||
HttpStatus,
|
||||
} from '@nestjs/common';
|
||||
import { ApiKeysService, ApiKeyData } from '../api-keys/api-keys.service';
|
||||
import { CREDIT_COSTS } from '../config/pricing';
|
||||
|
||||
@Injectable()
|
||||
export class CreditsGuard implements CanActivate {
|
||||
constructor(private readonly apiKeyService: ApiKeysService) {}
|
||||
|
||||
async canActivate(context: ExecutionContext): Promise<boolean> {
|
||||
const request = context.switchToHttp().getRequest();
|
||||
const apiKey = request.apiKey as ApiKeyData;
|
||||
|
||||
if (!apiKey) {
|
||||
return true; // Let ApiKeyGuard handle missing key
|
||||
}
|
||||
|
||||
const endpoint = this.extractEndpoint(request.path);
|
||||
const estimatedCredits = this.estimateCredits(endpoint, request);
|
||||
|
||||
const hasCredits = await this.apiKeyService.hasEnoughCredits(apiKey.id, estimatedCredits);
|
||||
|
||||
if (!hasCredits) {
|
||||
throw new HttpException(
|
||||
{
|
||||
statusCode: HttpStatus.PAYMENT_REQUIRED,
|
||||
message: 'Insufficient credits. Please upgrade your plan or wait for monthly reset.',
|
||||
creditsRequired: estimatedCredits,
|
||||
creditsUsed: apiKey.creditsUsed,
|
||||
monthlyLimit: apiKey.monthlyCredits,
|
||||
},
|
||||
HttpStatus.PAYMENT_REQUIRED
|
||||
);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private extractEndpoint(path: string): string {
|
||||
const match = path.match(/\/v1\/(\w+)/);
|
||||
return match ? match[1] : 'unknown';
|
||||
}
|
||||
|
||||
private estimateCredits(endpoint: string, request: any): number {
|
||||
switch (endpoint) {
|
||||
case 'search':
|
||||
return CREDIT_COSTS.search;
|
||||
case 'tts':
|
||||
const text = request.body?.text || '';
|
||||
return Math.max(1, Math.ceil(text.length / 1000) * CREDIT_COSTS.tts.per1000Chars);
|
||||
case 'stt':
|
||||
// Estimate based on file size or default to 1 minute
|
||||
return CREDIT_COSTS.stt.perMinute;
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
export * from './api-key.guard';
|
||||
export * from './rate-limit.guard';
|
||||
export * from './credits.guard';
|
||||
|
|
@ -1,70 +0,0 @@
|
|||
import {
|
||||
Injectable,
|
||||
CanActivate,
|
||||
ExecutionContext,
|
||||
HttpException,
|
||||
HttpStatus,
|
||||
Inject,
|
||||
} from '@nestjs/common';
|
||||
import Redis from 'ioredis';
|
||||
import { ApiKeyData } from '../api-keys/api-keys.service';
|
||||
|
||||
export const REDIS_CLIENT = 'REDIS_CLIENT';
|
||||
|
||||
@Injectable()
|
||||
export class RateLimitGuard implements CanActivate {
|
||||
constructor(@Inject(REDIS_CLIENT) private readonly redis: Redis) {}
|
||||
|
||||
async canActivate(context: ExecutionContext): Promise<boolean> {
|
||||
const request = context.switchToHttp().getRequest();
|
||||
const apiKey = request.apiKey as ApiKeyData;
|
||||
|
||||
if (!apiKey) {
|
||||
return true; // Let ApiKeyGuard handle missing key
|
||||
}
|
||||
|
||||
const key = `ratelimit:${apiKey.id}`;
|
||||
const limit = apiKey.rateLimit;
|
||||
const window = 60; // 60 seconds
|
||||
|
||||
// Sliding window rate limiting using sorted set
|
||||
const now = Date.now();
|
||||
const windowStart = now - window * 1000;
|
||||
|
||||
// Remove old entries
|
||||
await this.redis.zremrangebyscore(key, 0, windowStart);
|
||||
|
||||
// Count current requests
|
||||
const count = await this.redis.zcard(key);
|
||||
|
||||
if (count >= limit) {
|
||||
// Get the oldest entry to calculate retry-after
|
||||
const oldestEntries = await this.redis.zrange(key, 0, 0, 'WITHSCORES');
|
||||
const oldestTimestamp = oldestEntries.length > 1 ? parseInt(oldestEntries[1], 10) : now;
|
||||
const retryAfter = Math.ceil((oldestTimestamp + window * 1000 - now) / 1000);
|
||||
|
||||
throw new HttpException(
|
||||
{
|
||||
statusCode: HttpStatus.TOO_MANY_REQUESTS,
|
||||
message: 'Rate limit exceeded',
|
||||
retryAfter,
|
||||
limit,
|
||||
remaining: 0,
|
||||
},
|
||||
HttpStatus.TOO_MANY_REQUESTS
|
||||
);
|
||||
}
|
||||
|
||||
// Add current request
|
||||
await this.redis.zadd(key, now, `${now}`);
|
||||
await this.redis.expire(key, window);
|
||||
|
||||
// Add rate limit headers to response
|
||||
const response = context.switchToHttp().getResponse();
|
||||
response.setHeader('X-RateLimit-Limit', limit);
|
||||
response.setHeader('X-RateLimit-Remaining', Math.max(0, limit - count - 1));
|
||||
response.setHeader('X-RateLimit-Reset', Math.ceil(now / 1000) + window);
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
import { Controller, Get } from '@nestjs/common';
|
||||
import { ApiTags, ApiOperation, ApiResponse } from '@nestjs/swagger';
|
||||
|
||||
@ApiTags('System')
|
||||
@Controller('health')
|
||||
export class HealthController {
|
||||
@Get()
|
||||
@ApiOperation({
|
||||
summary: 'Health check',
|
||||
description: 'Returns service health status. No authentication required.',
|
||||
})
|
||||
@ApiResponse({
|
||||
status: 200,
|
||||
description: 'Service is healthy',
|
||||
schema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
status: { type: 'string', example: 'ok' },
|
||||
service: { type: 'string', example: 'api-gateway' },
|
||||
timestamp: { type: 'string', example: '2025-01-29T10:30:00.000Z' },
|
||||
},
|
||||
},
|
||||
})
|
||||
check() {
|
||||
return {
|
||||
status: 'ok',
|
||||
service: 'api-gateway',
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
import { Module } from '@nestjs/common';
|
||||
import { HealthController } from './health.controller';
|
||||
|
||||
@Module({
|
||||
controllers: [HealthController],
|
||||
})
|
||||
export class HealthModule {}
|
||||
|
|
@ -1,78 +0,0 @@
|
|||
import { NestFactory } from '@nestjs/core';
|
||||
import { ValidationPipe } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger';
|
||||
import { AppModule } from './app.module';
|
||||
import { HttpExceptionFilter } from './common/filters/http-exception.filter';
|
||||
|
||||
async function bootstrap() {
|
||||
const app = await NestFactory.create(AppModule);
|
||||
|
||||
const configService = app.get(ConfigService);
|
||||
const port = configService.get<number>('port') || 3030;
|
||||
const corsOrigins = configService.get<string[]>('cors.origins') || [];
|
||||
|
||||
// Enable CORS
|
||||
app.enableCors({
|
||||
origin: corsOrigins,
|
||||
credentials: true,
|
||||
});
|
||||
|
||||
// Global validation pipe
|
||||
app.useGlobalPipes(
|
||||
new ValidationPipe({
|
||||
whitelist: true,
|
||||
transform: true,
|
||||
forbidNonWhitelisted: true,
|
||||
})
|
||||
);
|
||||
|
||||
// Global exception filter
|
||||
app.useGlobalFilters(new HttpExceptionFilter());
|
||||
|
||||
// Swagger/OpenAPI documentation
|
||||
const config = new DocumentBuilder()
|
||||
.setTitle('ManaCore API Gateway')
|
||||
.setDescription(
|
||||
'API Gateway for ManaCore services (Search, STT, TTS). ' +
|
||||
'Use X-API-Key header for public endpoints (/v1/*) and Bearer JWT for management endpoints (/api-keys/*).'
|
||||
)
|
||||
.setVersion('1.0')
|
||||
.addApiKey(
|
||||
{
|
||||
type: 'apiKey',
|
||||
name: 'X-API-Key',
|
||||
in: 'header',
|
||||
description: 'API Key for accessing public endpoints',
|
||||
},
|
||||
'api-key'
|
||||
)
|
||||
.addBearerAuth(
|
||||
{
|
||||
type: 'http',
|
||||
scheme: 'bearer',
|
||||
bearerFormat: 'JWT',
|
||||
description: 'JWT token from mana-core-auth for management endpoints',
|
||||
},
|
||||
'jwt'
|
||||
)
|
||||
.addTag('Search', 'Web search and content extraction')
|
||||
.addTag('STT', 'Speech-to-Text transcription')
|
||||
.addTag('TTS', 'Text-to-Speech synthesis')
|
||||
.addTag('API Keys', 'API key management (requires JWT authentication)')
|
||||
.addTag('System', 'Health checks and metrics')
|
||||
.build();
|
||||
|
||||
const document = SwaggerModule.createDocument(app, config);
|
||||
SwaggerModule.setup('docs', app, document, {
|
||||
swaggerOptions: {
|
||||
persistAuthorization: true,
|
||||
},
|
||||
});
|
||||
|
||||
await app.listen(port);
|
||||
console.log(`API Gateway running on port ${port}`);
|
||||
console.log(`Swagger docs available at http://localhost:${port}/docs`);
|
||||
}
|
||||
|
||||
bootstrap();
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
import { Controller, Get, Res } from '@nestjs/common';
|
||||
import { ApiTags, ApiOperation, ApiResponse, ApiProduces } from '@nestjs/swagger';
|
||||
import { Response } from 'express';
|
||||
import { MetricsService } from './metrics.service';
|
||||
|
||||
@ApiTags('System')
|
||||
@Controller('metrics')
|
||||
export class MetricsController {
|
||||
constructor(private readonly metricsService: MetricsService) {}
|
||||
|
||||
@Get()
|
||||
@ApiOperation({
|
||||
summary: 'Prometheus metrics',
|
||||
description: 'Returns Prometheus-formatted metrics for monitoring. No authentication required.',
|
||||
})
|
||||
@ApiProduces('text/plain')
|
||||
@ApiResponse({
|
||||
status: 200,
|
||||
description: 'Prometheus metrics in text format',
|
||||
})
|
||||
async getMetrics(@Res() res: Response) {
|
||||
const metrics = await this.metricsService.getMetrics();
|
||||
res.setHeader('Content-Type', this.metricsService.getContentType());
|
||||
res.send(metrics);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
import { Global, Module } from '@nestjs/common';
|
||||
import { MetricsController } from './metrics.controller';
|
||||
import { MetricsService } from './metrics.service';
|
||||
|
||||
@Global()
|
||||
@Module({
|
||||
controllers: [MetricsController],
|
||||
providers: [MetricsService],
|
||||
exports: [MetricsService],
|
||||
})
|
||||
export class MetricsModule {}
|
||||
|
|
@ -1,82 +0,0 @@
|
|||
import { Injectable, OnModuleInit } from '@nestjs/common';
|
||||
import * as client from 'prom-client';
|
||||
|
||||
@Injectable()
|
||||
export class MetricsService implements OnModuleInit {
|
||||
private readonly register: client.Registry;
|
||||
|
||||
// Counters
|
||||
public readonly requestsTotal: client.Counter<string>;
|
||||
public readonly creditsUsedTotal: client.Counter<string>;
|
||||
public readonly errorsTotal: client.Counter<string>;
|
||||
|
||||
// Histograms
|
||||
public readonly requestDuration: client.Histogram<string>;
|
||||
|
||||
// Gauges
|
||||
public readonly activeApiKeys: client.Gauge<string>;
|
||||
public readonly rateLimitExceeded: client.Counter<string>;
|
||||
|
||||
constructor() {
|
||||
this.register = new client.Registry();
|
||||
|
||||
// Add default metrics
|
||||
client.collectDefaultMetrics({ register: this.register });
|
||||
|
||||
// Custom metrics
|
||||
this.requestsTotal = new client.Counter({
|
||||
name: 'api_gateway_requests_total',
|
||||
help: 'Total number of API requests',
|
||||
labelNames: ['endpoint', 'method', 'status', 'tier'],
|
||||
registers: [this.register],
|
||||
});
|
||||
|
||||
this.creditsUsedTotal = new client.Counter({
|
||||
name: 'api_gateway_credits_used_total',
|
||||
help: 'Total credits consumed',
|
||||
labelNames: ['endpoint', 'tier'],
|
||||
registers: [this.register],
|
||||
});
|
||||
|
||||
this.errorsTotal = new client.Counter({
|
||||
name: 'api_gateway_errors_total',
|
||||
help: 'Total number of errors',
|
||||
labelNames: ['endpoint', 'error_type'],
|
||||
registers: [this.register],
|
||||
});
|
||||
|
||||
this.requestDuration = new client.Histogram({
|
||||
name: 'api_gateway_request_duration_seconds',
|
||||
help: 'Request duration in seconds',
|
||||
labelNames: ['endpoint', 'method'],
|
||||
buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10],
|
||||
registers: [this.register],
|
||||
});
|
||||
|
||||
this.activeApiKeys = new client.Gauge({
|
||||
name: 'api_gateway_active_api_keys',
|
||||
help: 'Number of active API keys',
|
||||
labelNames: ['tier'],
|
||||
registers: [this.register],
|
||||
});
|
||||
|
||||
this.rateLimitExceeded = new client.Counter({
|
||||
name: 'api_gateway_rate_limit_exceeded_total',
|
||||
help: 'Total number of rate limit exceeded events',
|
||||
labelNames: ['tier'],
|
||||
registers: [this.register],
|
||||
});
|
||||
}
|
||||
|
||||
onModuleInit() {
|
||||
// Initial setup if needed
|
||||
}
|
||||
|
||||
async getMetrics(): Promise<string> {
|
||||
return this.register.metrics();
|
||||
}
|
||||
|
||||
getContentType(): string {
|
||||
return this.register.contentType;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,275 +0,0 @@
|
|||
import {
|
||||
Controller,
|
||||
Post,
|
||||
Get,
|
||||
Body,
|
||||
UseGuards,
|
||||
UseInterceptors,
|
||||
UploadedFile,
|
||||
Res,
|
||||
Query,
|
||||
} from '@nestjs/common';
|
||||
import {
|
||||
ApiTags,
|
||||
ApiSecurity,
|
||||
ApiOperation,
|
||||
ApiResponse,
|
||||
ApiConsumes,
|
||||
ApiBody,
|
||||
ApiHeader,
|
||||
} from '@nestjs/swagger';
|
||||
import { FileInterceptor } from '@nestjs/platform-express';
|
||||
import { Response } from 'express';
|
||||
import { ApiKeyGuard } from '../guards/api-key.guard';
|
||||
import { RateLimitGuard } from '../guards/rate-limit.guard';
|
||||
import { CreditsGuard } from '../guards/credits.guard';
|
||||
import { UsageTrackingInterceptor } from '../common/interceptors/usage-tracking.interceptor';
|
||||
import { ApiKeyParam } from '../common/decorators/api-key.decorator';
|
||||
import { ApiKeyData } from '../api-keys/api-keys.service';
|
||||
import {
|
||||
SearchProxyService,
|
||||
SearchRequestDto,
|
||||
ExtractRequestDto,
|
||||
BulkExtractRequestDto,
|
||||
} from './services/search-proxy.service';
|
||||
import { SttProxyService, TranscribeRequestDto } from './services/stt-proxy.service';
|
||||
import { TtsProxyService, SynthesizeRequestDto } from './services/tts-proxy.service';
|
||||
|
||||
@ApiSecurity('api-key')
|
||||
@ApiHeader({
|
||||
name: 'X-API-Key',
|
||||
description: 'Your API key (sk_live_xxx or sk_test_xxx)',
|
||||
required: true,
|
||||
})
|
||||
@Controller('v1')
|
||||
@UseGuards(ApiKeyGuard, RateLimitGuard, CreditsGuard)
|
||||
@UseInterceptors(UsageTrackingInterceptor)
|
||||
export class ProxyController {
|
||||
constructor(
|
||||
private readonly searchProxy: SearchProxyService,
|
||||
private readonly sttProxy: SttProxyService,
|
||||
private readonly ttsProxy: TtsProxyService
|
||||
) {}
|
||||
|
||||
// === SEARCH ===
|
||||
|
||||
@Post('search')
|
||||
@ApiTags('Search')
|
||||
@ApiOperation({
|
||||
summary: 'Web search',
|
||||
description: 'Search the web using multiple search engines. Costs 1 credit per request.',
|
||||
})
|
||||
@ApiBody({
|
||||
schema: {
|
||||
type: 'object',
|
||||
required: ['query'],
|
||||
properties: {
|
||||
query: { type: 'string', example: 'quantum computing' },
|
||||
options: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
categories: {
|
||||
type: 'array',
|
||||
items: { type: 'string' },
|
||||
example: ['general', 'science'],
|
||||
},
|
||||
engines: { type: 'array', items: { type: 'string' }, example: ['google', 'bing'] },
|
||||
language: { type: 'string', example: 'en' },
|
||||
limit: { type: 'number', example: 10 },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
@ApiResponse({ status: 200, description: 'Search results' })
|
||||
@ApiResponse({ status: 429, description: 'Rate limit exceeded' })
|
||||
@ApiResponse({ status: 402, description: 'Insufficient credits' })
|
||||
async search(@Body() body: SearchRequestDto, @ApiKeyParam() apiKey: ApiKeyData) {
|
||||
return this.searchProxy.search(body);
|
||||
}
|
||||
|
||||
@Get('search/engines')
|
||||
@ApiTags('Search')
|
||||
@ApiOperation({
|
||||
summary: 'Get available search engines',
|
||||
description: 'Returns a list of available search engines and categories. Free endpoint.',
|
||||
})
|
||||
@ApiResponse({ status: 200, description: 'List of search engines' })
|
||||
async getEngines() {
|
||||
return this.searchProxy.getEngines();
|
||||
}
|
||||
|
||||
@Post('extract')
|
||||
@ApiTags('Search')
|
||||
@ApiOperation({
|
||||
summary: 'Extract content from URL',
|
||||
description: 'Extracts main content from a webpage. Costs 1 credit per request.',
|
||||
})
|
||||
@ApiBody({
|
||||
schema: {
|
||||
type: 'object',
|
||||
required: ['url'],
|
||||
properties: {
|
||||
url: { type: 'string', example: 'https://example.com/article' },
|
||||
options: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
includeMarkdown: { type: 'boolean', example: true },
|
||||
maxLength: { type: 'number', example: 5000 },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
@ApiResponse({ status: 200, description: 'Extracted content' })
|
||||
async extract(@Body() body: ExtractRequestDto) {
|
||||
return this.searchProxy.extract(body);
|
||||
}
|
||||
|
||||
@Post('extract/bulk')
|
||||
@ApiTags('Search')
|
||||
@ApiOperation({
|
||||
summary: 'Bulk extract content',
|
||||
description: 'Extracts content from multiple URLs (max 20). Costs 1 credit per URL.',
|
||||
})
|
||||
@ApiBody({
|
||||
schema: {
|
||||
type: 'object',
|
||||
required: ['urls'],
|
||||
properties: {
|
||||
urls: {
|
||||
type: 'array',
|
||||
items: { type: 'string' },
|
||||
example: ['https://example.com/article1', 'https://example.com/article2'],
|
||||
},
|
||||
options: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
includeMarkdown: { type: 'boolean', example: true },
|
||||
maxLength: { type: 'number', example: 5000 },
|
||||
},
|
||||
},
|
||||
concurrency: { type: 'number', example: 5 },
|
||||
},
|
||||
},
|
||||
})
|
||||
@ApiResponse({ status: 200, description: 'Extracted content from all URLs' })
|
||||
async bulkExtract(@Body() body: BulkExtractRequestDto) {
|
||||
return this.searchProxy.bulkExtract(body);
|
||||
}
|
||||
|
||||
// === STT ===
|
||||
|
||||
@Post('stt/transcribe')
|
||||
@ApiTags('STT')
|
||||
@ApiOperation({
|
||||
summary: 'Transcribe audio to text',
|
||||
description:
|
||||
'Converts audio file to text. Costs 10 credits per minute of audio. Supports WAV, MP3, OGG, FLAC.',
|
||||
})
|
||||
@ApiConsumes('multipart/form-data')
|
||||
@ApiBody({
|
||||
schema: {
|
||||
type: 'object',
|
||||
required: ['file'],
|
||||
properties: {
|
||||
file: { type: 'string', format: 'binary', description: 'Audio file' },
|
||||
language: { type: 'string', example: 'en', description: 'Language code (optional)' },
|
||||
model: { type: 'string', example: 'base', description: 'Model name (optional)' },
|
||||
},
|
||||
},
|
||||
})
|
||||
@ApiResponse({ status: 200, description: 'Transcription result' })
|
||||
@ApiResponse({ status: 400, description: 'Invalid audio file' })
|
||||
@UseInterceptors(FileInterceptor('file'))
|
||||
async transcribe(@UploadedFile() file: Express.Multer.File, @Body() body: TranscribeRequestDto) {
|
||||
return this.sttProxy.transcribe(file, body);
|
||||
}
|
||||
|
||||
@Get('stt/models')
|
||||
@ApiTags('STT')
|
||||
@ApiOperation({
|
||||
summary: 'Get available STT models',
|
||||
description: 'Returns a list of available speech-to-text models. Free endpoint.',
|
||||
})
|
||||
@ApiResponse({ status: 200, description: 'List of STT models' })
|
||||
async getSttModels() {
|
||||
return this.sttProxy.getModels();
|
||||
}
|
||||
|
||||
@Get('stt/languages')
|
||||
@ApiTags('STT')
|
||||
@ApiOperation({
|
||||
summary: 'Get supported languages',
|
||||
description: 'Returns a list of languages supported by STT. Free endpoint.',
|
||||
})
|
||||
@ApiResponse({ status: 200, description: 'List of supported languages' })
|
||||
async getSttLanguages() {
|
||||
return this.sttProxy.getLanguages();
|
||||
}
|
||||
|
||||
// === TTS ===
|
||||
|
||||
@Post('tts/synthesize')
|
||||
@ApiTags('TTS')
|
||||
@ApiOperation({
|
||||
summary: 'Synthesize text to speech',
|
||||
description:
|
||||
'Converts text to audio. Costs 1 credit per 1000 characters. Returns audio file (MP3, WAV, or OGG).',
|
||||
})
|
||||
@ApiBody({
|
||||
schema: {
|
||||
type: 'object',
|
||||
required: ['text'],
|
||||
properties: {
|
||||
text: { type: 'string', example: 'Hello, world! This is a test.' },
|
||||
voice: { type: 'string', example: 'en-US-1', description: 'Voice ID' },
|
||||
language: { type: 'string', example: 'en-US', description: 'Language code' },
|
||||
speed: { type: 'number', example: 1.0, minimum: 0.5, maximum: 2.0 },
|
||||
format: { type: 'string', enum: ['mp3', 'wav', 'ogg'], default: 'mp3' },
|
||||
},
|
||||
},
|
||||
})
|
||||
@ApiResponse({
|
||||
status: 200,
|
||||
description: 'Audio file',
|
||||
content: {
|
||||
'audio/mpeg': { schema: { type: 'string', format: 'binary' } },
|
||||
'audio/wav': { schema: { type: 'string', format: 'binary' } },
|
||||
'audio/ogg': { schema: { type: 'string', format: 'binary' } },
|
||||
},
|
||||
})
|
||||
async synthesize(@Body() body: SynthesizeRequestDto, @Res() res: Response) {
|
||||
const audio = await this.ttsProxy.synthesize(body);
|
||||
|
||||
const format = body.format || 'mp3';
|
||||
const contentType =
|
||||
format === 'wav' ? 'audio/wav' : format === 'ogg' ? 'audio/ogg' : 'audio/mpeg';
|
||||
|
||||
res.setHeader('Content-Type', contentType);
|
||||
res.setHeader('Content-Length', audio.length);
|
||||
res.send(audio);
|
||||
}
|
||||
|
||||
@Get('tts/voices')
|
||||
@ApiTags('TTS')
|
||||
@ApiOperation({
|
||||
summary: 'Get available voices',
|
||||
description: 'Returns a list of available TTS voices. Free endpoint.',
|
||||
})
|
||||
@ApiResponse({ status: 200, description: 'List of available voices' })
|
||||
async getTtsVoices() {
|
||||
return this.ttsProxy.getVoices();
|
||||
}
|
||||
|
||||
@Get('tts/languages')
|
||||
@ApiTags('TTS')
|
||||
@ApiOperation({
|
||||
summary: 'Get supported TTS languages',
|
||||
description: 'Returns a list of languages supported by TTS. Free endpoint.',
|
||||
})
|
||||
@ApiResponse({ status: 200, description: 'List of supported languages' })
|
||||
async getTtsLanguages() {
|
||||
return this.ttsProxy.getLanguages();
|
||||
}
|
||||
}
|
||||
|
|
@ -1,56 +0,0 @@
|
|||
import { Module } from '@nestjs/common';
|
||||
import { MulterModule } from '@nestjs/platform-express';
|
||||
import { memoryStorage } from 'multer';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import Redis from 'ioredis';
|
||||
import { ProxyController } from './proxy.controller';
|
||||
import { SearchProxyService, SttProxyService, TtsProxyService } from './services';
|
||||
import { ApiKeysModule } from '../api-keys/api-keys.module';
|
||||
import { UsageModule } from '../usage/usage.module';
|
||||
import { CreditsModule } from '../credits/credits.module';
|
||||
import { ApiKeyGuard } from '../guards/api-key.guard';
|
||||
import { RateLimitGuard, REDIS_CLIENT } from '../guards/rate-limit.guard';
|
||||
import { CreditsGuard } from '../guards/credits.guard';
|
||||
import { UsageTrackingInterceptor } from '../common/interceptors/usage-tracking.interceptor';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
MulterModule.register({
|
||||
storage: memoryStorage(),
|
||||
limits: {
|
||||
fileSize: 100 * 1024 * 1024, // 100MB max file size
|
||||
},
|
||||
}),
|
||||
ApiKeysModule,
|
||||
UsageModule,
|
||||
CreditsModule,
|
||||
],
|
||||
controllers: [ProxyController],
|
||||
providers: [
|
||||
SearchProxyService,
|
||||
SttProxyService,
|
||||
TtsProxyService,
|
||||
ApiKeyGuard,
|
||||
RateLimitGuard,
|
||||
CreditsGuard,
|
||||
UsageTrackingInterceptor,
|
||||
{
|
||||
provide: REDIS_CLIENT,
|
||||
useFactory: (configService: ConfigService) => {
|
||||
const host = configService.get<string>('redis.host') || 'localhost';
|
||||
const port = configService.get<number>('redis.port') || 6379;
|
||||
const password = configService.get<string>('redis.password');
|
||||
|
||||
return new Redis({
|
||||
host,
|
||||
port,
|
||||
password: password || undefined,
|
||||
keyPrefix: configService.get<string>('redis.keyPrefix') || 'api-gateway:',
|
||||
});
|
||||
},
|
||||
inject: [ConfigService],
|
||||
},
|
||||
],
|
||||
exports: [REDIS_CLIENT],
|
||||
})
|
||||
export class ProxyModule {}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
export * from './search-proxy.service';
|
||||
export * from './stt-proxy.service';
|
||||
export * from './tts-proxy.service';
|
||||
|
|
@ -1,102 +0,0 @@
|
|||
import { Injectable, HttpException, HttpStatus } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
|
||||
export interface SearchRequestDto {
|
||||
query: string;
|
||||
options?: {
|
||||
categories?: string[];
|
||||
engines?: string[];
|
||||
language?: string;
|
||||
limit?: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface ExtractRequestDto {
|
||||
url: string;
|
||||
options?: {
|
||||
includeMarkdown?: boolean;
|
||||
maxLength?: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface BulkExtractRequestDto {
|
||||
urls: string[];
|
||||
options?: {
|
||||
includeMarkdown?: boolean;
|
||||
maxLength?: number;
|
||||
};
|
||||
concurrency?: number;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class SearchProxyService {
|
||||
private readonly searchUrl: string;
|
||||
|
||||
constructor(private readonly configService: ConfigService) {
|
||||
this.searchUrl = this.configService.get('services.search') || 'http://localhost:3021';
|
||||
}
|
||||
|
||||
async search(body: SearchRequestDto): Promise<any> {
|
||||
const response = await fetch(`${this.searchUrl}/api/v1/search`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await response.text();
|
||||
throw new HttpException(
|
||||
`Search service error: ${error}`,
|
||||
response.status || HttpStatus.BAD_GATEWAY
|
||||
);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async extract(body: ExtractRequestDto): Promise<any> {
|
||||
const response = await fetch(`${this.searchUrl}/api/v1/extract`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await response.text();
|
||||
throw new HttpException(
|
||||
`Extract service error: ${error}`,
|
||||
response.status || HttpStatus.BAD_GATEWAY
|
||||
);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async bulkExtract(body: BulkExtractRequestDto): Promise<any> {
|
||||
const response = await fetch(`${this.searchUrl}/api/v1/extract/bulk`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await response.text();
|
||||
throw new HttpException(
|
||||
`Bulk extract service error: ${error}`,
|
||||
response.status || HttpStatus.BAD_GATEWAY
|
||||
);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async getEngines(): Promise<any> {
|
||||
const response = await fetch(`${this.searchUrl}/api/v1/search/engines`);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new HttpException('Failed to get search engines', HttpStatus.BAD_GATEWAY);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
}
|
||||
|
|
@ -1,64 +0,0 @@
|
|||
import { Injectable, HttpException, HttpStatus } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
|
||||
export interface TranscribeRequestDto {
|
||||
language?: string;
|
||||
model?: string;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class SttProxyService {
|
||||
private readonly sttUrl: string;
|
||||
|
||||
constructor(private readonly configService: ConfigService) {
|
||||
this.sttUrl = this.configService.get('services.stt') || 'http://localhost:3020';
|
||||
}
|
||||
|
||||
async transcribe(file: Express.Multer.File, options: TranscribeRequestDto): Promise<any> {
|
||||
const formData = new FormData();
|
||||
const uint8Array = new Uint8Array(file.buffer);
|
||||
formData.append('file', new Blob([uint8Array], { type: file.mimetype }), file.originalname);
|
||||
|
||||
if (options.language) {
|
||||
formData.append('language', options.language);
|
||||
}
|
||||
if (options.model) {
|
||||
formData.append('model', options.model);
|
||||
}
|
||||
|
||||
const response = await fetch(`${this.sttUrl}/api/v1/transcribe`, {
|
||||
method: 'POST',
|
||||
body: formData,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await response.text();
|
||||
throw new HttpException(
|
||||
`STT service error: ${error}`,
|
||||
response.status || HttpStatus.BAD_GATEWAY
|
||||
);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async getModels(): Promise<any> {
|
||||
const response = await fetch(`${this.sttUrl}/api/v1/models`);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new HttpException('Failed to get STT models', HttpStatus.BAD_GATEWAY);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async getLanguages(): Promise<any> {
|
||||
const response = await fetch(`${this.sttUrl}/api/v1/languages`);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new HttpException('Failed to get STT languages', HttpStatus.BAD_GATEWAY);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
}
|
||||
|
|
@ -1,58 +0,0 @@
|
|||
import { Injectable, HttpException, HttpStatus } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
|
||||
export interface SynthesizeRequestDto {
|
||||
text: string;
|
||||
voice?: string;
|
||||
language?: string;
|
||||
speed?: number;
|
||||
format?: 'mp3' | 'wav' | 'ogg';
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class TtsProxyService {
|
||||
private readonly ttsUrl: string;
|
||||
|
||||
constructor(private readonly configService: ConfigService) {
|
||||
this.ttsUrl = this.configService.get('services.tts') || 'http://localhost:3022';
|
||||
}
|
||||
|
||||
async synthesize(body: SynthesizeRequestDto): Promise<Buffer> {
|
||||
const response = await fetch(`${this.ttsUrl}/api/v1/synthesize`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await response.text();
|
||||
throw new HttpException(
|
||||
`TTS service error: ${error}`,
|
||||
response.status || HttpStatus.BAD_GATEWAY
|
||||
);
|
||||
}
|
||||
|
||||
const arrayBuffer = await response.arrayBuffer();
|
||||
return Buffer.from(arrayBuffer);
|
||||
}
|
||||
|
||||
async getVoices(): Promise<any> {
|
||||
const response = await fetch(`${this.ttsUrl}/api/v1/voices`);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new HttpException('Failed to get TTS voices', HttpStatus.BAD_GATEWAY);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async getLanguages(): Promise<any> {
|
||||
const response = await fetch(`${this.ttsUrl}/api/v1/languages`);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new HttpException('Failed to get TTS languages', HttpStatus.BAD_GATEWAY);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
}
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
import { Module } from '@nestjs/common';
|
||||
import { ScheduleModule } from '@nestjs/schedule';
|
||||
import { SchedulerService } from './scheduler.service';
|
||||
|
||||
@Module({
|
||||
imports: [ScheduleModule.forRoot()],
|
||||
providers: [SchedulerService],
|
||||
})
|
||||
export class SchedulerModule {}
|
||||
|
|
@ -1,72 +0,0 @@
|
|||
import { Injectable, Inject } from '@nestjs/common';
|
||||
import { Cron, CronExpression } from '@nestjs/schedule';
|
||||
import { sql } from 'drizzle-orm';
|
||||
import { DATABASE_CONNECTION } from '../db/database.module';
|
||||
import { apiKeys } from '../db/schema';
|
||||
|
||||
@Injectable()
|
||||
export class SchedulerService {
|
||||
constructor(
|
||||
@Inject(DATABASE_CONNECTION)
|
||||
private readonly db: ReturnType<typeof import('../db/connection').getDb>
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Reset monthly credits on the 1st of each month at 00:00 UTC
|
||||
*/
|
||||
@Cron('0 0 1 * *')
|
||||
async resetMonthlyCredits() {
|
||||
console.log('[Scheduler] Running monthly credit reset...');
|
||||
|
||||
try {
|
||||
const result = await this.db
|
||||
.update(apiKeys)
|
||||
.set({
|
||||
creditsUsed: 0,
|
||||
creditsResetAt: this.getNextMonthReset(),
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.returning({ id: apiKeys.id });
|
||||
|
||||
console.log(`[Scheduler] Reset credits for ${result.length} API keys`);
|
||||
} catch (error) {
|
||||
console.error('[Scheduler] Failed to reset monthly credits:', error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up old usage logs (older than 90 days) - runs weekly
|
||||
*/
|
||||
@Cron(CronExpression.EVERY_WEEK)
|
||||
async cleanupOldUsageLogs() {
|
||||
console.log('[Scheduler] Cleaning up old usage logs...');
|
||||
|
||||
try {
|
||||
const cutoffDate = new Date();
|
||||
cutoffDate.setDate(cutoffDate.getDate() - 90);
|
||||
|
||||
await this.db.execute(
|
||||
sql`DELETE FROM api_gateway.api_usage WHERE created_at < ${cutoffDate.toISOString()}`
|
||||
);
|
||||
|
||||
console.log('[Scheduler] Cleaned up usage logs older than 90 days');
|
||||
} catch (error) {
|
||||
console.error('[Scheduler] Failed to cleanup usage logs:', error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Aggregate daily usage stats - runs at 1:00 AM UTC
|
||||
*/
|
||||
@Cron('0 1 * * *')
|
||||
async aggregateDailyUsage() {
|
||||
console.log('[Scheduler] Daily usage aggregation completed (handled by interceptor)');
|
||||
// Note: Daily aggregation is already handled in real-time by UsageTrackingInterceptor
|
||||
// This cron is a placeholder for any additional daily processing
|
||||
}
|
||||
|
||||
private getNextMonthReset(): Date {
|
||||
const now = new Date();
|
||||
return new Date(now.getFullYear(), now.getMonth() + 1, 1);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,42 +0,0 @@
|
|||
import { IsOptional, IsString, IsDateString, IsInt, Min, Max } from 'class-validator';
|
||||
import { Transform } from 'class-transformer';
|
||||
import { ApiPropertyOptional } from '@nestjs/swagger';
|
||||
|
||||
export class UsageQueryDto {
|
||||
@ApiPropertyOptional({
|
||||
description: 'Number of days to query (1-365)',
|
||||
minimum: 1,
|
||||
maximum: 365,
|
||||
default: 30,
|
||||
})
|
||||
@IsOptional()
|
||||
@Transform(({ value }) => parseInt(value, 10))
|
||||
@IsInt()
|
||||
@Min(1)
|
||||
@Max(365)
|
||||
days?: number = 30;
|
||||
|
||||
@ApiPropertyOptional({
|
||||
description: 'Start date for custom range (ISO 8601)',
|
||||
example: '2025-01-01',
|
||||
})
|
||||
@IsOptional()
|
||||
@IsDateString()
|
||||
startDate?: string;
|
||||
|
||||
@ApiPropertyOptional({
|
||||
description: 'End date for custom range (ISO 8601)',
|
||||
example: '2025-01-31',
|
||||
})
|
||||
@IsOptional()
|
||||
@IsDateString()
|
||||
endDate?: string;
|
||||
|
||||
@ApiPropertyOptional({
|
||||
description: 'Filter by endpoint',
|
||||
example: 'search',
|
||||
})
|
||||
@IsOptional()
|
||||
@IsString()
|
||||
endpoint?: string;
|
||||
}
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
import { Module, forwardRef } from '@nestjs/common';
|
||||
import { UsageService } from './usage.service';
|
||||
import { ApiKeysModule } from '../api-keys/api-keys.module';
|
||||
|
||||
@Module({
|
||||
imports: [forwardRef(() => ApiKeysModule)],
|
||||
providers: [UsageService],
|
||||
exports: [UsageService],
|
||||
})
|
||||
export class UsageModule {}
|
||||
|
|
@ -1,187 +0,0 @@
|
|||
import { Injectable, Inject } from '@nestjs/common';
|
||||
import { eq, sql, gte, and, desc } from 'drizzle-orm';
|
||||
import { DATABASE_CONNECTION } from '../db/database.module';
|
||||
import { apiUsage, apiUsageDaily, NewApiUsage } from '../db/schema';
|
||||
|
||||
export interface TrackUsageParams {
|
||||
apiKeyId: string;
|
||||
endpoint: string;
|
||||
method: string;
|
||||
path: string;
|
||||
latencyMs: number;
|
||||
statusCode: number;
|
||||
creditsUsed: number;
|
||||
requestSize?: number;
|
||||
responseSize?: number;
|
||||
creditReason?: string;
|
||||
metadata?: Record<string, any>;
|
||||
}
|
||||
|
||||
export interface UsageSummary {
|
||||
totalRequests: number;
|
||||
totalCreditsUsed: number;
|
||||
avgLatencyMs: number;
|
||||
errorCount: number;
|
||||
byEndpoint: Record<
|
||||
string,
|
||||
{
|
||||
requests: number;
|
||||
credits: number;
|
||||
avgLatencyMs: number;
|
||||
errors: number;
|
||||
}
|
||||
>;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class UsageService {
|
||||
constructor(
|
||||
@Inject(DATABASE_CONNECTION)
|
||||
private readonly db: ReturnType<typeof import('../db/connection').getDb>
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Track a single API usage event
|
||||
*/
|
||||
async track(params: TrackUsageParams): Promise<void> {
|
||||
const usage: NewApiUsage = {
|
||||
apiKeyId: params.apiKeyId,
|
||||
endpoint: params.endpoint,
|
||||
method: params.method,
|
||||
path: params.path,
|
||||
latencyMs: params.latencyMs,
|
||||
statusCode: params.statusCode,
|
||||
creditsUsed: params.creditsUsed,
|
||||
requestSize: params.requestSize,
|
||||
responseSize: params.responseSize,
|
||||
creditReason: params.creditReason,
|
||||
metadata: params.metadata,
|
||||
};
|
||||
|
||||
await this.db.insert(apiUsage).values(usage);
|
||||
|
||||
// Also update daily aggregates
|
||||
await this.updateDailyAggregate(params);
|
||||
}
|
||||
|
||||
/**
|
||||
* Update daily usage aggregate
|
||||
*/
|
||||
private async updateDailyAggregate(params: TrackUsageParams): Promise<void> {
|
||||
const today = new Date().toISOString().split('T')[0];
|
||||
const isError = params.statusCode >= 400;
|
||||
|
||||
// Upsert daily aggregate
|
||||
await this.db
|
||||
.insert(apiUsageDaily)
|
||||
.values({
|
||||
apiKeyId: params.apiKeyId,
|
||||
date: today,
|
||||
endpoint: params.endpoint,
|
||||
requestCount: 1,
|
||||
creditsUsed: params.creditsUsed,
|
||||
totalLatencyMs: params.latencyMs,
|
||||
errorCount: isError ? 1 : 0,
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: [apiUsageDaily.apiKeyId, apiUsageDaily.date, apiUsageDaily.endpoint],
|
||||
set: {
|
||||
requestCount: sql`${apiUsageDaily.requestCount} + 1`,
|
||||
creditsUsed: sql`${apiUsageDaily.creditsUsed} + ${params.creditsUsed}`,
|
||||
totalLatencyMs: sql`${apiUsageDaily.totalLatencyMs} + ${params.latencyMs}`,
|
||||
errorCount: isError ? sql`${apiUsageDaily.errorCount} + 1` : apiUsageDaily.errorCount,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get daily usage for an API key
|
||||
*/
|
||||
async getDailyUsage(apiKeyId: string, days: number = 30) {
|
||||
const startDate = new Date();
|
||||
startDate.setDate(startDate.getDate() - days);
|
||||
|
||||
const usage = await this.db
|
||||
.select()
|
||||
.from(apiUsageDaily)
|
||||
.where(
|
||||
and(
|
||||
eq(apiUsageDaily.apiKeyId, apiKeyId),
|
||||
gte(apiUsageDaily.date, startDate.toISOString().split('T')[0])
|
||||
)
|
||||
)
|
||||
.orderBy(desc(apiUsageDaily.date));
|
||||
|
||||
return usage;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get usage summary for an API key
|
||||
*/
|
||||
async getUsageSummary(apiKeyId: string, days: number = 30): Promise<UsageSummary> {
|
||||
const startDate = new Date();
|
||||
startDate.setDate(startDate.getDate() - days);
|
||||
|
||||
const dailyUsage = await this.getDailyUsage(apiKeyId, days);
|
||||
|
||||
const summary: UsageSummary = {
|
||||
totalRequests: 0,
|
||||
totalCreditsUsed: 0,
|
||||
avgLatencyMs: 0,
|
||||
errorCount: 0,
|
||||
byEndpoint: {},
|
||||
};
|
||||
|
||||
let totalLatency = 0;
|
||||
|
||||
for (const day of dailyUsage) {
|
||||
summary.totalRequests += day.requestCount;
|
||||
summary.totalCreditsUsed += day.creditsUsed;
|
||||
totalLatency += day.totalLatencyMs;
|
||||
summary.errorCount += day.errorCount;
|
||||
|
||||
if (!summary.byEndpoint[day.endpoint]) {
|
||||
summary.byEndpoint[day.endpoint] = {
|
||||
requests: 0,
|
||||
credits: 0,
|
||||
avgLatencyMs: 0,
|
||||
errors: 0,
|
||||
};
|
||||
}
|
||||
|
||||
const ep = summary.byEndpoint[day.endpoint];
|
||||
ep.requests += day.requestCount;
|
||||
ep.credits += day.creditsUsed;
|
||||
ep.avgLatencyMs += day.totalLatencyMs;
|
||||
ep.errors += day.errorCount;
|
||||
}
|
||||
|
||||
if (summary.totalRequests > 0) {
|
||||
summary.avgLatencyMs = Math.round(totalLatency / summary.totalRequests);
|
||||
}
|
||||
|
||||
// Calculate average latency per endpoint
|
||||
for (const endpoint of Object.keys(summary.byEndpoint)) {
|
||||
const ep = summary.byEndpoint[endpoint];
|
||||
if (ep.requests > 0) {
|
||||
ep.avgLatencyMs = Math.round(ep.avgLatencyMs / ep.requests);
|
||||
}
|
||||
}
|
||||
|
||||
return summary;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get recent usage logs for an API key
|
||||
*/
|
||||
async getRecentLogs(apiKeyId: string, limit: number = 100) {
|
||||
const logs = await this.db
|
||||
.select()
|
||||
.from(apiUsage)
|
||||
.where(eq(apiUsage.apiKeyId, apiKeyId))
|
||||
.orderBy(desc(apiUsage.createdAt))
|
||||
.limit(limit);
|
||||
|
||||
return logs;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
{
|
||||
"extends": "./tsconfig.json",
|
||||
"exclude": ["node_modules", "dist", "test", "**/*spec.ts"]
|
||||
}
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"module": "commonjs",
|
||||
"declaration": true,
|
||||
"removeComments": true,
|
||||
"emitDecoratorMetadata": true,
|
||||
"experimentalDecorators": true,
|
||||
"allowSyntheticDefaultImports": true,
|
||||
"target": "ES2022",
|
||||
"sourceMap": true,
|
||||
"outDir": "./dist",
|
||||
"baseUrl": "./",
|
||||
"incremental": true,
|
||||
"skipLibCheck": true,
|
||||
"strictNullChecks": true,
|
||||
"noImplicitAny": true,
|
||||
"strictBindCallApply": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"noFallthroughCasesInSwitch": true,
|
||||
"esModuleInterop": true,
|
||||
"resolveJsonModule": true
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
1
services/mana-search-go/.gitignore
vendored
Normal file
1
services/mana-search-go/.gitignore
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
bin/
|
||||
73
services/mana-search-go/CLAUDE.md
Normal file
73
services/mana-search-go/CLAUDE.md
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
# mana-search (Go)
|
||||
|
||||
Go replacement for the NestJS mana-search service. Unified web search and content extraction microservice using SearXNG + Redis.
|
||||
|
||||
## Architecture
|
||||
|
||||
- **Language:** Go 1.25
|
||||
- **Search Engine:** SearXNG (meta-search)
|
||||
- **Cache:** Redis (graceful degradation if unavailable)
|
||||
- **Metrics:** Prometheus
|
||||
- **Port:** 3021
|
||||
|
||||
## Endpoints
|
||||
|
||||
### Search
|
||||
- `POST /api/v1/search` — Web search via SearXNG
|
||||
- `GET /api/v1/search/engines` — List available engines
|
||||
- `GET /api/v1/search/health` — Search service health + cache stats
|
||||
- `DELETE /api/v1/search/cache` — Clear all cached results
|
||||
|
||||
### Extract
|
||||
- `POST /api/v1/extract` — Extract content from URL (readability + optional markdown)
|
||||
- `POST /api/v1/extract/bulk` — Bulk extract (max 20 URLs, configurable concurrency)
|
||||
|
||||
### System
|
||||
- `GET /health` — Health check (SearXNG + Redis)
|
||||
- `GET /metrics` — Prometheus metrics
|
||||
|
||||
## Commands
|
||||
|
||||
```bash
|
||||
go run ./cmd/server # Dev
|
||||
go build -o bin/mana-search ./cmd/server # Build
|
||||
go test ./... # Test
|
||||
```
|
||||
|
||||
## Environment Variables
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `PORT` | 3021 | Server port |
|
||||
| `SEARXNG_URL` | http://localhost:8080 | SearXNG base URL |
|
||||
| `SEARXNG_TIMEOUT` | 15000 | SearXNG timeout (ms) |
|
||||
| `SEARXNG_DEFAULT_LANGUAGE` | de-DE | Default search language |
|
||||
| `REDIS_HOST` | localhost | Redis host |
|
||||
| `REDIS_PORT` | 6379 | Redis port |
|
||||
| `REDIS_PASSWORD` | | Redis password |
|
||||
| `CACHE_SEARCH_TTL` | 3600 | Search cache TTL (seconds) |
|
||||
| `CACHE_EXTRACT_TTL` | 86400 | Extract cache TTL (seconds) |
|
||||
| `EXTRACT_TIMEOUT` | 10000 | Content extraction timeout (ms) |
|
||||
| `EXTRACT_MAX_LENGTH` | 50000 | Max extracted text length (chars) |
|
||||
| `CORS_ORIGINS` | localhost:3000,5173,8081 | Allowed CORS origins |
|
||||
|
||||
## Search Categories
|
||||
|
||||
| Category | Engines |
|
||||
|----------|---------|
|
||||
| `general` | Google, Bing, DuckDuckGo, Brave, Wikipedia |
|
||||
| `news` | Google News, Bing News |
|
||||
| `science` | arXiv, Google Scholar, PubMed |
|
||||
| `it` | GitHub, StackOverflow, NPM, MDN |
|
||||
|
||||
## Docker
|
||||
|
||||
Uses the same `docker-compose.dev.yml` from `services/mana-search/` for SearXNG + Redis.
|
||||
|
||||
```bash
|
||||
# Start SearXNG + Redis
|
||||
cd services/mana-search && docker-compose -f docker-compose.dev.yml up -d
|
||||
|
||||
# Run Go service
|
||||
cd services/mana-search-go && go run ./cmd/server
|
||||
```
|
||||
21
services/mana-search-go/Dockerfile
Normal file
21
services/mana-search-go/Dockerfile
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
FROM golang:1.25-alpine AS builder
|
||||
|
||||
WORKDIR /app
|
||||
COPY services/mana-search-go/go.mod services/mana-search-go/go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
COPY services/mana-search-go/ .
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /mana-search ./cmd/server
|
||||
|
||||
FROM alpine:3.21
|
||||
|
||||
RUN apk --no-cache add ca-certificates tzdata
|
||||
|
||||
COPY --from=builder /mana-search /usr/local/bin/mana-search
|
||||
|
||||
EXPOSE 3021
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=5s --retries=3 \
|
||||
CMD wget -q --spider http://localhost:3021/health || exit 1
|
||||
|
||||
ENTRYPOINT ["mana-search"]
|
||||
93
services/mana-search-go/cmd/server/main.go
Normal file
93
services/mana-search-go/cmd/server/main.go
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"github.com/rs/cors"
|
||||
|
||||
"github.com/manacore/mana-search/internal/cache"
|
||||
"github.com/manacore/mana-search/internal/config"
|
||||
"github.com/manacore/mana-search/internal/extract"
|
||||
"github.com/manacore/mana-search/internal/handler"
|
||||
"github.com/manacore/mana-search/internal/metrics"
|
||||
"github.com/manacore/mana-search/internal/search"
|
||||
)
|
||||
|
||||
func main() {
|
||||
slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
|
||||
Level: slog.LevelInfo,
|
||||
})))
|
||||
|
||||
cfg := config.Load()
|
||||
m := metrics.New()
|
||||
c := cache.New(cfg, m)
|
||||
defer c.Close()
|
||||
|
||||
provider := search.NewSearxngProvider(cfg)
|
||||
extractor := extract.New(cfg)
|
||||
|
||||
searchHandler := handler.NewSearchHandler(provider, c, m, cfg)
|
||||
extractHandler := handler.NewExtractHandler(extractor, c, m, cfg)
|
||||
healthHandler := handler.NewHealthHandler(provider, c)
|
||||
|
||||
mux := http.NewServeMux()
|
||||
|
||||
// Health & metrics
|
||||
mux.HandleFunc("GET /health", healthHandler.Health)
|
||||
mux.HandleFunc("GET /api/v1/health", healthHandler.Health)
|
||||
mux.Handle("GET /metrics", promhttp.Handler())
|
||||
|
||||
// Search endpoints
|
||||
mux.HandleFunc("POST /api/v1/search", searchHandler.Search)
|
||||
mux.HandleFunc("GET /api/v1/search/engines", searchHandler.Engines)
|
||||
mux.HandleFunc("GET /api/v1/search/health", searchHandler.Health)
|
||||
mux.HandleFunc("DELETE /api/v1/search/cache", searchHandler.ClearCache)
|
||||
|
||||
// Extract endpoints
|
||||
mux.HandleFunc("POST /api/v1/extract", extractHandler.Extract)
|
||||
mux.HandleFunc("POST /api/v1/extract/bulk", extractHandler.BulkExtract)
|
||||
|
||||
c_handler := cors.New(cors.Options{
|
||||
AllowedOrigins: cfg.CORSOrigins,
|
||||
AllowedMethods: []string{"GET", "POST", "DELETE", "OPTIONS"},
|
||||
AllowedHeaders: []string{"Content-Type", "Authorization"},
|
||||
AllowCredentials: true,
|
||||
}).Handler(mux)
|
||||
|
||||
server := &http.Server{
|
||||
Addr: fmt.Sprintf(":%d", cfg.Port),
|
||||
Handler: c_handler,
|
||||
ReadTimeout: 30 * time.Second,
|
||||
WriteTimeout: 60 * time.Second,
|
||||
IdleTimeout: 120 * time.Second,
|
||||
}
|
||||
|
||||
// Graceful shutdown
|
||||
sigCh := make(chan os.Signal, 1)
|
||||
signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
|
||||
|
||||
go func() {
|
||||
slog.Info("mana-search started", "port", cfg.Port, "searxng", cfg.SearxngURL)
|
||||
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||
slog.Error("server error", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}()
|
||||
|
||||
<-sigCh
|
||||
slog.Info("shutting down...")
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
server.Shutdown(ctx)
|
||||
|
||||
slog.Info("server stopped")
|
||||
}
|
||||
31
services/mana-search-go/go.mod
Normal file
31
services/mana-search-go/go.mod
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
module github.com/manacore/mana-search
|
||||
|
||||
go 1.25.0
|
||||
|
||||
require (
|
||||
github.com/JohannesKaufmann/html-to-markdown/v2 v2.3.3
|
||||
github.com/go-shiori/go-readability v0.0.0-20251205110129-5db1dc9836f0
|
||||
github.com/prometheus/client_golang v1.22.0
|
||||
github.com/redis/go-redis/v9 v9.18.0
|
||||
github.com/rs/cors v1.11.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/JohannesKaufmann/dom v0.2.0 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c // indirect
|
||||
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/prometheus/client_model v0.6.1 // indirect
|
||||
github.com/prometheus/common v0.62.0 // indirect
|
||||
github.com/prometheus/procfs v0.15.1 // indirect
|
||||
go.uber.org/atomic v1.11.0 // indirect
|
||||
golang.org/x/net v0.39.0 // indirect
|
||||
golang.org/x/sys v0.32.0 // indirect
|
||||
golang.org/x/text v0.24.0 // indirect
|
||||
google.golang.org/protobuf v1.36.5 // indirect
|
||||
)
|
||||
145
services/mana-search-go/go.sum
Normal file
145
services/mana-search-go/go.sum
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
github.com/JohannesKaufmann/dom v0.2.0 h1:1bragmEb19K8lHAqgFgqCpiPCFEZMTXzOIEjuxkUfLQ=
|
||||
github.com/JohannesKaufmann/dom v0.2.0/go.mod h1:57iSUl5RKric4bUkgos4zu6Xt5LMHUnw3TF1l5CbGZo=
|
||||
github.com/JohannesKaufmann/html-to-markdown/v2 v2.3.3 h1:r3fokGFRDk/8pHmwLwJ8zsX4qiqfS1/1TZm2BH8ueY8=
|
||||
github.com/JohannesKaufmann/html-to-markdown/v2 v2.3.3/go.mod h1:HtsP+1Fchp4dVvaiIsLHAl/yqL3H1YLwqLC9kNwqQEg=
|
||||
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
|
||||
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
|
||||
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA=
|
||||
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c h1:wpkoddUomPfHiOziHZixGO5ZBS73cKqVzZipfrLmO1w=
|
||||
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c/go.mod h1:oVDCh3qjJMLVUSILBRwrm+Bc6RNXGZYtoh9xdvf1ffM=
|
||||
github.com/go-shiori/go-readability v0.0.0-20251205110129-5db1dc9836f0 h1:A3B75Yp163FAIf9nLlFMl4pwIj+T3uKxfI7mbvvY2Ls=
|
||||
github.com/go-shiori/go-readability v0.0.0-20251205110129-5db1dc9836f0/go.mod h1:suxK0Wpz4BM3/2+z1mnOVTIWHDiMCIOGoKDCRumSsk0=
|
||||
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs=
|
||||
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
||||
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q=
|
||||
github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0=
|
||||
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
|
||||
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
|
||||
github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io=
|
||||
github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I=
|
||||
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
|
||||
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
|
||||
github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs=
|
||||
github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0=
|
||||
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA=
|
||||
github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
|
||||
github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg=
|
||||
github.com/sebdah/goldie/v2 v2.5.5 h1:rx1mwF95RxZ3/83sdS4Yp7t2C5TCokvWP4TBRbAyEWY=
|
||||
github.com/sebdah/goldie/v2 v2.5.5/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI=
|
||||
github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
|
||||
github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/yuin/goldmark v1.7.11 h1:ZCxLyDMtz0nT2HFfsYG8WZ47Trip2+JyLysKcMYE5bo=
|
||||
github.com/yuin/goldmark v1.7.11/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
|
||||
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
|
||||
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
|
||||
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
|
||||
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY=
|
||||
golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
|
||||
golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
|
||||
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
|
||||
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
|
||||
golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM=
|
||||
google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
159
services/mana-search-go/internal/cache/cache.go
vendored
Normal file
159
services/mana-search-go/internal/cache/cache.go
vendored
Normal file
|
|
@ -0,0 +1,159 @@
|
|||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"github.com/manacore/mana-search/internal/config"
|
||||
"github.com/manacore/mana-search/internal/metrics"
|
||||
)
|
||||
|
||||
type Cache struct {
|
||||
client *redis.Client
|
||||
prefix string
|
||||
metrics *metrics.Metrics
|
||||
hits atomic.Int64
|
||||
misses atomic.Int64
|
||||
}
|
||||
|
||||
func New(cfg *config.Config, m *metrics.Metrics) *Cache {
|
||||
c := &Cache{
|
||||
prefix: cfg.RedisPrefix,
|
||||
metrics: m,
|
||||
}
|
||||
|
||||
client := redis.NewClient(&redis.Options{
|
||||
Addr: fmt.Sprintf("%s:%d", cfg.RedisHost, cfg.RedisPort),
|
||||
Password: cfg.RedisPassword,
|
||||
MaxRetries: 3,
|
||||
MinRetryBackoff: 200 * time.Millisecond,
|
||||
MaxRetryBackoff: 2 * time.Second,
|
||||
})
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if err := client.Ping(ctx).Err(); err != nil {
|
||||
slog.Warn("redis unavailable, running without cache", "error", err)
|
||||
return c
|
||||
}
|
||||
|
||||
slog.Info("redis connected", "addr", fmt.Sprintf("%s:%d", cfg.RedisHost, cfg.RedisPort))
|
||||
c.client = client
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *Cache) Get(ctx context.Context, key string) ([]byte, bool) {
|
||||
if c.client == nil {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
val, err := c.client.Get(ctx, c.prefix+key).Bytes()
|
||||
if err != nil {
|
||||
c.misses.Add(1)
|
||||
if c.metrics != nil {
|
||||
c.metrics.CacheMisses.Inc()
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
c.hits.Add(1)
|
||||
if c.metrics != nil {
|
||||
c.metrics.CacheHits.Inc()
|
||||
}
|
||||
return val, true
|
||||
}
|
||||
|
||||
func (c *Cache) Set(ctx context.Context, key string, value any, ttl time.Duration) {
|
||||
if c.client == nil {
|
||||
return
|
||||
}
|
||||
|
||||
data, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
slog.Error("cache marshal error", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
if err := c.client.Set(ctx, c.prefix+key, data, ttl).Err(); err != nil {
|
||||
slog.Error("cache set error", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cache) Delete(ctx context.Context, key string) {
|
||||
if c.client == nil {
|
||||
return
|
||||
}
|
||||
c.client.Del(ctx, c.prefix+key)
|
||||
}
|
||||
|
||||
func (c *Cache) Clear(ctx context.Context) (int64, error) {
|
||||
if c.client == nil {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
keys, err := c.client.Keys(ctx, c.prefix+"*").Result()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if len(keys) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
deleted, err := c.client.Del(ctx, keys...).Result()
|
||||
return deleted, err
|
||||
}
|
||||
|
||||
type HealthStatus struct {
|
||||
Status string `json:"status"`
|
||||
Latency int64 `json:"latency"`
|
||||
}
|
||||
|
||||
func (c *Cache) HealthCheck(ctx context.Context) HealthStatus {
|
||||
if c.client == nil {
|
||||
return HealthStatus{Status: "disabled", Latency: 0}
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
err := c.client.Ping(ctx).Err()
|
||||
latency := time.Since(start).Milliseconds()
|
||||
|
||||
if err != nil {
|
||||
return HealthStatus{Status: "error", Latency: latency}
|
||||
}
|
||||
return HealthStatus{Status: "ok", Latency: latency}
|
||||
}
|
||||
|
||||
func (c *Cache) IsConnected() bool {
|
||||
return c.client != nil
|
||||
}
|
||||
|
||||
type Stats struct {
|
||||
Hits int64 `json:"hits"`
|
||||
Misses int64 `json:"misses"`
|
||||
HitRate float64 `json:"hitRate"`
|
||||
}
|
||||
|
||||
func (c *Cache) Stats() Stats {
|
||||
hits := c.hits.Load()
|
||||
misses := c.misses.Load()
|
||||
total := hits + misses
|
||||
var rate float64
|
||||
if total > 0 {
|
||||
rate = float64(hits) / float64(total)
|
||||
}
|
||||
return Stats{Hits: hits, Misses: misses, HitRate: rate}
|
||||
}
|
||||
|
||||
func (c *Cache) Close() error {
|
||||
if c.client != nil {
|
||||
return c.client.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
81
services/mana-search-go/internal/config/config.go
Normal file
81
services/mana-search-go/internal/config/config.go
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Port int
|
||||
|
||||
// SearXNG
|
||||
SearxngURL string
|
||||
SearxngTimeout int // ms
|
||||
SearxngDefaultLanguage string
|
||||
|
||||
// Redis
|
||||
RedisHost string
|
||||
RedisPort int
|
||||
RedisPassword string
|
||||
RedisPrefix string
|
||||
|
||||
// Cache TTLs (seconds)
|
||||
CacheSearchTTL int
|
||||
CacheExtractTTL int
|
||||
|
||||
// Extract
|
||||
ExtractTimeout int // ms
|
||||
ExtractMaxLength int
|
||||
ExtractUserAgent string
|
||||
|
||||
// CORS
|
||||
CORSOrigins []string
|
||||
}
|
||||
|
||||
func Load() *Config {
|
||||
return &Config{
|
||||
Port: getEnvInt("PORT", 3021),
|
||||
|
||||
SearxngURL: getEnv("SEARXNG_URL", "http://localhost:8080"),
|
||||
SearxngTimeout: getEnvInt("SEARXNG_TIMEOUT", 15000),
|
||||
SearxngDefaultLanguage: getEnv("SEARXNG_DEFAULT_LANGUAGE", "de-DE"),
|
||||
|
||||
RedisHost: getEnv("REDIS_HOST", "localhost"),
|
||||
RedisPort: getEnvInt("REDIS_PORT", 6379),
|
||||
RedisPassword: getEnv("REDIS_PASSWORD", ""),
|
||||
RedisPrefix: "mana-search:",
|
||||
|
||||
CacheSearchTTL: getEnvInt("CACHE_SEARCH_TTL", 3600),
|
||||
CacheExtractTTL: getEnvInt("CACHE_EXTRACT_TTL", 86400),
|
||||
|
||||
ExtractTimeout: getEnvInt("EXTRACT_TIMEOUT", 10000),
|
||||
ExtractMaxLength: getEnvInt("EXTRACT_MAX_LENGTH", 50000),
|
||||
ExtractUserAgent: getEnv("EXTRACT_USER_AGENT", "Mozilla/5.0 (compatible; ManaSearchBot/1.0; +https://mana.how)"),
|
||||
|
||||
CORSOrigins: getEnvSlice("CORS_ORIGINS", []string{"http://localhost:3000", "http://localhost:5173", "http://localhost:8081"}),
|
||||
}
|
||||
}
|
||||
|
||||
func getEnv(key, fallback string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func getEnvInt(key string, fallback int) int {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
if i, err := strconv.Atoi(v); err == nil {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func getEnvSlice(key string, fallback []string) []string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return strings.Split(v, ",")
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
278
services/mana-search-go/internal/extract/extractor.go
Normal file
278
services/mana-search-go/internal/extract/extractor.go
Normal file
|
|
@ -0,0 +1,278 @@
|
|||
package extract
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown/v2"
|
||||
readability "github.com/go-shiori/go-readability"
|
||||
|
||||
"github.com/manacore/mana-search/internal/config"
|
||||
)
|
||||
|
||||
type Extractor struct {
|
||||
timeout time.Duration
|
||||
maxLength int
|
||||
userAgent string
|
||||
}
|
||||
|
||||
func New(cfg *config.Config) *Extractor {
|
||||
return &Extractor{
|
||||
timeout: time.Duration(cfg.ExtractTimeout) * time.Millisecond,
|
||||
maxLength: cfg.ExtractMaxLength,
|
||||
userAgent: cfg.ExtractUserAgent,
|
||||
}
|
||||
}
|
||||
|
||||
// ExtractRequest from the client.
|
||||
type ExtractRequest struct {
|
||||
URL string `json:"url"`
|
||||
Options *ExtractOptions `json:"options,omitempty"`
|
||||
}
|
||||
|
||||
type ExtractOptions struct {
|
||||
IncludeHTML bool `json:"includeHtml,omitempty"`
|
||||
IncludeMarkdown bool `json:"includeMarkdown,omitempty"`
|
||||
MaxLength int `json:"maxLength,omitempty"`
|
||||
Timeout int `json:"timeout,omitempty"`
|
||||
}
|
||||
|
||||
// BulkExtractRequest for multiple URLs.
|
||||
type BulkExtractRequest struct {
|
||||
URLs []string `json:"urls"`
|
||||
Options *ExtractOptions `json:"options,omitempty"`
|
||||
Concurrency int `json:"concurrency,omitempty"`
|
||||
}
|
||||
|
||||
// ExtractResponse returned to the client.
|
||||
type ExtractResponse struct {
|
||||
Success bool `json:"success"`
|
||||
Content *ExtractedContent `json:"content,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Meta ExtractMeta `json:"meta"`
|
||||
}
|
||||
|
||||
type ExtractedContent struct {
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Author string `json:"author,omitempty"`
|
||||
PublishedDate string `json:"publishedDate,omitempty"`
|
||||
SiteName string `json:"siteName,omitempty"`
|
||||
Text string `json:"text"`
|
||||
Markdown string `json:"markdown,omitempty"`
|
||||
HTML string `json:"html,omitempty"`
|
||||
WordCount int `json:"wordCount"`
|
||||
ReadingTime int `json:"readingTime"`
|
||||
OgImage string `json:"ogImage,omitempty"`
|
||||
}
|
||||
|
||||
type ExtractMeta struct {
|
||||
URL string `json:"url"`
|
||||
Duration int64 `json:"duration"`
|
||||
Cached bool `json:"cached"`
|
||||
ContentType string `json:"contentType"`
|
||||
}
|
||||
|
||||
type BulkExtractResponse struct {
|
||||
Results []BulkExtractResult `json:"results"`
|
||||
Meta BulkMeta `json:"meta"`
|
||||
}
|
||||
|
||||
type BulkExtractResult struct {
|
||||
URL string `json:"url"`
|
||||
Success bool `json:"success"`
|
||||
Content *ExtractedContent `json:"content,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type BulkMeta struct {
|
||||
Total int `json:"total"`
|
||||
Successful int `json:"successful"`
|
||||
Failed int `json:"failed"`
|
||||
Duration int64 `json:"duration"`
|
||||
}
|
||||
|
||||
// Extract fetches a URL and extracts its content using readability.
|
||||
func (e *Extractor) Extract(ctx context.Context, req *ExtractRequest) *ExtractResponse {
|
||||
start := time.Now()
|
||||
|
||||
timeout := e.timeout
|
||||
if req.Options != nil && req.Options.Timeout > 0 {
|
||||
timeout = time.Duration(req.Options.Timeout) * time.Millisecond
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
parsedURL, err := url.Parse(req.URL)
|
||||
if err != nil {
|
||||
return errorResponse(req.URL, "invalid URL", start)
|
||||
}
|
||||
|
||||
article, err := readability.FromURL(parsedURL.String(), timeout, func(req *http.Request) {
|
||||
req.Header.Set("User-Agent", e.userAgent)
|
||||
})
|
||||
if err != nil {
|
||||
slog.Warn("extraction failed", "url", req.URL, "error", err)
|
||||
return errorResponse(req.URL, fmt.Sprintf("extraction failed: %v", err), start)
|
||||
}
|
||||
|
||||
text := cleanText(article.TextContent)
|
||||
maxLen := e.maxLength
|
||||
if req.Options != nil && req.Options.MaxLength > 0 {
|
||||
maxLen = req.Options.MaxLength
|
||||
}
|
||||
if len(text) > maxLen {
|
||||
text = text[:maxLen]
|
||||
}
|
||||
|
||||
wordCount := countWords(text)
|
||||
readingTime := int(math.Ceil(float64(wordCount) / 200.0))
|
||||
|
||||
content := &ExtractedContent{
|
||||
Title: article.Title,
|
||||
Description: article.Excerpt,
|
||||
Author: article.Byline,
|
||||
PublishedDate: formatTime(article.PublishedTime),
|
||||
SiteName: article.SiteName,
|
||||
Text: text,
|
||||
WordCount: wordCount,
|
||||
ReadingTime: readingTime,
|
||||
OgImage: article.Image,
|
||||
}
|
||||
|
||||
if req.Options != nil && req.Options.IncludeMarkdown && article.Content != "" {
|
||||
md, err := htmltomarkdown.ConvertString(article.Content)
|
||||
if err == nil {
|
||||
content.Markdown = md
|
||||
}
|
||||
}
|
||||
|
||||
if req.Options != nil && req.Options.IncludeHTML {
|
||||
content.HTML = article.Content
|
||||
}
|
||||
|
||||
return &ExtractResponse{
|
||||
Success: true,
|
||||
Content: content,
|
||||
Meta: ExtractMeta{
|
||||
URL: req.URL,
|
||||
Duration: time.Since(start).Milliseconds(),
|
||||
Cached: false,
|
||||
ContentType: "text/html",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// BulkExtract processes multiple URLs with limited concurrency.
|
||||
func (e *Extractor) BulkExtract(ctx context.Context, req *BulkExtractRequest) *BulkExtractResponse {
|
||||
start := time.Now()
|
||||
concurrency := 5
|
||||
if req.Concurrency > 0 && req.Concurrency <= 10 {
|
||||
concurrency = req.Concurrency
|
||||
}
|
||||
|
||||
results := make([]BulkExtractResult, len(req.URLs))
|
||||
|
||||
// Process in batches
|
||||
for i := 0; i < len(req.URLs); i += concurrency {
|
||||
end := i + concurrency
|
||||
if end > len(req.URLs) {
|
||||
end = len(req.URLs)
|
||||
}
|
||||
|
||||
type indexedResult struct {
|
||||
index int
|
||||
result *ExtractResponse
|
||||
}
|
||||
|
||||
ch := make(chan indexedResult, end-i)
|
||||
for j := i; j < end; j++ {
|
||||
go func(idx int, u string) {
|
||||
r := e.Extract(ctx, &ExtractRequest{URL: u, Options: req.Options})
|
||||
ch <- indexedResult{index: idx, result: r}
|
||||
}(j, req.URLs[j])
|
||||
}
|
||||
|
||||
for j := i; j < end; j++ {
|
||||
ir := <-ch
|
||||
results[ir.index] = BulkExtractResult{
|
||||
URL: req.URLs[ir.index],
|
||||
Success: ir.result.Success,
|
||||
Content: ir.result.Content,
|
||||
Error: ir.result.Error,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
successful := 0
|
||||
failed := 0
|
||||
for _, r := range results {
|
||||
if r.Success {
|
||||
successful++
|
||||
} else {
|
||||
failed++
|
||||
}
|
||||
}
|
||||
|
||||
return &BulkExtractResponse{
|
||||
Results: results,
|
||||
Meta: BulkMeta{
|
||||
Total: len(req.URLs),
|
||||
Successful: successful,
|
||||
Failed: failed,
|
||||
Duration: time.Since(start).Milliseconds(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// BuildCacheKey creates a cache key for extraction results.
|
||||
func BuildCacheKey(rawURL string) string {
|
||||
return "extract:" + rawURL
|
||||
}
|
||||
|
||||
func errorResponse(rawURL, errMsg string, start time.Time) *ExtractResponse {
|
||||
return &ExtractResponse{
|
||||
Success: false,
|
||||
Error: errMsg,
|
||||
Meta: ExtractMeta{
|
||||
URL: rawURL,
|
||||
Duration: time.Since(start).Milliseconds(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
reScript = regexp.MustCompile(`(?is)<script[^>]*>.*?</script>`)
|
||||
reStyle = regexp.MustCompile(`(?is)<style[^>]*>.*?</style>`)
|
||||
reTags = regexp.MustCompile(`<[^>]+>`)
|
||||
reWhitespace = regexp.MustCompile(`\s+`)
|
||||
)
|
||||
|
||||
func cleanText(html string) string {
|
||||
text := reScript.ReplaceAllString(html, "")
|
||||
text = reStyle.ReplaceAllString(text, "")
|
||||
text = reTags.ReplaceAllString(text, "")
|
||||
text = reWhitespace.ReplaceAllString(text, " ")
|
||||
return strings.TrimSpace(text)
|
||||
}
|
||||
|
||||
func countWords(text string) int {
|
||||
fields := strings.Fields(text)
|
||||
return len(fields)
|
||||
}
|
||||
|
||||
func formatTime(t *time.Time) string {
|
||||
if t == nil || t.IsZero() {
|
||||
return ""
|
||||
}
|
||||
return t.Format(time.RFC3339)
|
||||
}
|
||||
|
||||
24
services/mana-search-go/internal/handler/common.go
Normal file
24
services/mana-search-go/internal/handler/common.go
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, data any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
json.NewEncoder(w).Encode(data)
|
||||
}
|
||||
|
||||
func writeError(w http.ResponseWriter, status int, message string) {
|
||||
writeJSON(w, status, map[string]any{
|
||||
"success": false,
|
||||
"error": map[string]any{
|
||||
"statusCode": status,
|
||||
"message": message,
|
||||
"timestamp": time.Now().UTC().Format(time.RFC3339),
|
||||
},
|
||||
})
|
||||
}
|
||||
127
services/mana-search-go/internal/handler/extract.go
Normal file
127
services/mana-search-go/internal/handler/extract.go
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/manacore/mana-search/internal/cache"
|
||||
"github.com/manacore/mana-search/internal/config"
|
||||
"github.com/manacore/mana-search/internal/extract"
|
||||
"github.com/manacore/mana-search/internal/metrics"
|
||||
)
|
||||
|
||||
type ExtractHandler struct {
|
||||
extractor *extract.Extractor
|
||||
cache *cache.Cache
|
||||
metrics *metrics.Metrics
|
||||
cfg *config.Config
|
||||
}
|
||||
|
||||
func NewExtractHandler(extractor *extract.Extractor, c *cache.Cache, m *metrics.Metrics, cfg *config.Config) *ExtractHandler {
|
||||
return &ExtractHandler{
|
||||
extractor: extractor,
|
||||
cache: c,
|
||||
metrics: m,
|
||||
cfg: cfg,
|
||||
}
|
||||
}
|
||||
|
||||
// Extract handles POST /api/v1/extract
|
||||
func (h *ExtractHandler) Extract(w http.ResponseWriter, r *http.Request) {
|
||||
start := time.Now()
|
||||
|
||||
var req extract.ExtractRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid request body")
|
||||
return
|
||||
}
|
||||
|
||||
if req.URL == "" {
|
||||
writeError(w, http.StatusBadRequest, "url is required")
|
||||
return
|
||||
}
|
||||
if _, err := url.ParseRequestURI(req.URL); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "url must be a valid URL")
|
||||
return
|
||||
}
|
||||
|
||||
// Validate options
|
||||
if req.Options != nil {
|
||||
if req.Options.MaxLength > 0 && (req.Options.MaxLength < 100 || req.Options.MaxLength > 100000) {
|
||||
writeError(w, http.StatusBadRequest, "maxLength must be between 100 and 100000")
|
||||
return
|
||||
}
|
||||
if req.Options.Timeout > 0 && (req.Options.Timeout < 1000 || req.Options.Timeout > 30000) {
|
||||
writeError(w, http.StatusBadRequest, "timeout must be between 1000 and 30000")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
cacheKey := extract.BuildCacheKey(req.URL)
|
||||
|
||||
// Check cache
|
||||
if data, ok := h.cache.Get(r.Context(), cacheKey); ok {
|
||||
var cached extract.ExtractResponse
|
||||
if err := json.Unmarshal(data, &cached); err == nil {
|
||||
cached.Meta.Cached = true
|
||||
duration := time.Since(start).Seconds()
|
||||
h.metrics.RecordRequest("extract", "200", duration)
|
||||
writeJSON(w, http.StatusOK, cached)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Extract content
|
||||
resp := h.extractor.Extract(r.Context(), &req)
|
||||
|
||||
// Cache successful results
|
||||
if resp.Success {
|
||||
ttl := time.Duration(h.cfg.CacheExtractTTL) * time.Second
|
||||
h.cache.Set(r.Context(), cacheKey, resp, ttl)
|
||||
}
|
||||
|
||||
status := "200"
|
||||
if !resp.Success {
|
||||
status = "500"
|
||||
}
|
||||
duration := time.Since(start).Seconds()
|
||||
h.metrics.RecordRequest("extract", status, duration)
|
||||
|
||||
writeJSON(w, http.StatusOK, resp)
|
||||
}
|
||||
|
||||
// BulkExtract handles POST /api/v1/extract/bulk
|
||||
func (h *ExtractHandler) BulkExtract(w http.ResponseWriter, r *http.Request) {
|
||||
start := time.Now()
|
||||
|
||||
var req extract.BulkExtractRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid request body")
|
||||
return
|
||||
}
|
||||
|
||||
if len(req.URLs) == 0 {
|
||||
writeError(w, http.StatusBadRequest, "urls is required")
|
||||
return
|
||||
}
|
||||
if len(req.URLs) > 20 {
|
||||
writeError(w, http.StatusBadRequest, "maximum 20 URLs allowed")
|
||||
return
|
||||
}
|
||||
|
||||
for _, u := range req.URLs {
|
||||
if _, err := url.ParseRequestURI(u); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid URL: "+u)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
resp := h.extractor.BulkExtract(r.Context(), &req)
|
||||
|
||||
duration := time.Since(start).Seconds()
|
||||
h.metrics.RecordRequest("extract_bulk", "200", duration)
|
||||
|
||||
writeJSON(w, http.StatusOK, resp)
|
||||
}
|
||||
51
services/mana-search-go/internal/handler/health.go
Normal file
51
services/mana-search-go/internal/handler/health.go
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/manacore/mana-search/internal/cache"
|
||||
"github.com/manacore/mana-search/internal/search"
|
||||
)
|
||||
|
||||
type HealthHandler struct {
|
||||
provider *search.SearxngProvider
|
||||
cache *cache.Cache
|
||||
startTime time.Time
|
||||
}
|
||||
|
||||
func NewHealthHandler(provider *search.SearxngProvider, c *cache.Cache) *HealthHandler {
|
||||
return &HealthHandler{
|
||||
provider: provider,
|
||||
cache: c,
|
||||
startTime: time.Now(),
|
||||
}
|
||||
}
|
||||
|
||||
// Health handles GET /health
|
||||
func (h *HealthHandler) Health(w http.ResponseWriter, r *http.Request) {
|
||||
sxStatus, sxLatency := h.provider.HealthCheck(r.Context())
|
||||
redisHealth := h.cache.HealthCheck(r.Context())
|
||||
|
||||
overall := "ok"
|
||||
if sxStatus == "error" && redisHealth.Status == "error" {
|
||||
overall = "error"
|
||||
} else if sxStatus == "error" || redisHealth.Status == "error" || redisHealth.Status == "disabled" {
|
||||
overall = "degraded"
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"status": overall,
|
||||
"service": "mana-search",
|
||||
"version": "1.0.0",
|
||||
"uptime": time.Since(h.startTime).Seconds(),
|
||||
"timestamp": time.Now().UTC().Format(time.RFC3339),
|
||||
"components": map[string]any{
|
||||
"searxng": map[string]any{
|
||||
"status": sxStatus,
|
||||
"latency": sxLatency,
|
||||
},
|
||||
"redis": redisHealth,
|
||||
},
|
||||
})
|
||||
}
|
||||
156
services/mana-search-go/internal/handler/search.go
Normal file
156
services/mana-search-go/internal/handler/search.go
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/manacore/mana-search/internal/cache"
|
||||
"github.com/manacore/mana-search/internal/config"
|
||||
"github.com/manacore/mana-search/internal/metrics"
|
||||
"github.com/manacore/mana-search/internal/search"
|
||||
)
|
||||
|
||||
type SearchHandler struct {
|
||||
provider *search.SearxngProvider
|
||||
cache *cache.Cache
|
||||
metrics *metrics.Metrics
|
||||
cfg *config.Config
|
||||
}
|
||||
|
||||
func NewSearchHandler(provider *search.SearxngProvider, c *cache.Cache, m *metrics.Metrics, cfg *config.Config) *SearchHandler {
|
||||
return &SearchHandler{
|
||||
provider: provider,
|
||||
cache: c,
|
||||
metrics: m,
|
||||
cfg: cfg,
|
||||
}
|
||||
}
|
||||
|
||||
// Search handles POST /api/v1/search
|
||||
func (h *SearchHandler) Search(w http.ResponseWriter, r *http.Request) {
|
||||
start := time.Now()
|
||||
h.metrics.ActiveSearches.Inc()
|
||||
defer h.metrics.ActiveSearches.Dec()
|
||||
|
||||
var req search.SearchRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid request body")
|
||||
return
|
||||
}
|
||||
|
||||
if req.Query == "" {
|
||||
writeError(w, http.StatusBadRequest, "query is required")
|
||||
return
|
||||
}
|
||||
|
||||
// Validate options
|
||||
if req.Options != nil {
|
||||
if req.Options.Limit < 0 || req.Options.Limit > 50 {
|
||||
writeError(w, http.StatusBadRequest, "limit must be between 1 and 50")
|
||||
return
|
||||
}
|
||||
if req.Options.SafeSearch < 0 || req.Options.SafeSearch > 2 {
|
||||
writeError(w, http.StatusBadRequest, "safeSearch must be 0, 1, or 2")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
cacheKey := search.BuildCacheKey(&req)
|
||||
|
||||
// Check cache
|
||||
if req.Cache.IsEnabled() {
|
||||
if data, ok := h.cache.Get(r.Context(), cacheKey); ok {
|
||||
var cached search.SearchResponse
|
||||
if err := json.Unmarshal(data, &cached); err == nil {
|
||||
cached.Meta.Cached = true
|
||||
cached.Meta.CacheKey = cacheKey
|
||||
duration := time.Since(start).Seconds()
|
||||
h.metrics.RecordRequest("search", "200", duration)
|
||||
writeJSON(w, http.StatusOK, cached)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Query SearXNG
|
||||
results, err := h.provider.Search(r.Context(), &req)
|
||||
if err != nil {
|
||||
slog.Error("search failed", "error", err, "query", req.Query)
|
||||
duration := time.Since(start).Seconds()
|
||||
h.metrics.RecordRequest("search", "502", duration)
|
||||
writeError(w, http.StatusBadGateway, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Collect unique engines
|
||||
engineSet := make(map[string]bool)
|
||||
for _, r := range results {
|
||||
engineSet[r.Engine] = true
|
||||
}
|
||||
engines := make([]string, 0, len(engineSet))
|
||||
for e := range engineSet {
|
||||
engines = append(engines, e)
|
||||
}
|
||||
|
||||
resp := search.SearchResponse{
|
||||
Results: results,
|
||||
Meta: search.SearchMeta{
|
||||
Query: req.Query,
|
||||
TotalResults: len(results),
|
||||
Engines: engines,
|
||||
Duration: time.Since(start).Milliseconds(),
|
||||
Cached: false,
|
||||
CacheKey: cacheKey,
|
||||
},
|
||||
}
|
||||
|
||||
// Cache result
|
||||
if req.Cache.IsEnabled() {
|
||||
ttl := time.Duration(h.cfg.CacheSearchTTL) * time.Second
|
||||
if req.Cache != nil && req.Cache.TTL > 0 {
|
||||
ttl = time.Duration(req.Cache.TTL) * time.Second
|
||||
}
|
||||
h.cache.Set(r.Context(), cacheKey, resp, ttl)
|
||||
}
|
||||
|
||||
duration := time.Since(start).Seconds()
|
||||
h.metrics.RecordRequest("search", "200", duration)
|
||||
writeJSON(w, http.StatusOK, resp)
|
||||
}
|
||||
|
||||
// Engines handles GET /api/v1/search/engines
|
||||
func (h *SearchHandler) Engines(w http.ResponseWriter, r *http.Request) {
|
||||
engines := h.provider.GetEngines(r.Context())
|
||||
writeJSON(w, http.StatusOK, map[string]any{"engines": engines})
|
||||
}
|
||||
|
||||
// Health handles GET /api/v1/search/health
|
||||
func (h *SearchHandler) Health(w http.ResponseWriter, r *http.Request) {
|
||||
sxStatus, sxLatency := h.provider.HealthCheck(r.Context())
|
||||
redisHealth := h.cache.HealthCheck(r.Context())
|
||||
cacheStats := h.cache.Stats()
|
||||
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"searxng": map[string]any{
|
||||
"status": sxStatus,
|
||||
"latency": sxLatency,
|
||||
},
|
||||
"redis": redisHealth,
|
||||
"cache": cacheStats,
|
||||
})
|
||||
}
|
||||
|
||||
// ClearCache handles DELETE /api/v1/search/cache
|
||||
func (h *SearchHandler) ClearCache(w http.ResponseWriter, r *http.Request) {
|
||||
deleted, err := h.cache.Clear(r.Context())
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "failed to clear cache")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"cleared": true,
|
||||
"keysRemoved": deleted,
|
||||
})
|
||||
}
|
||||
55
services/mana-search-go/internal/metrics/metrics.go
Normal file
55
services/mana-search-go/internal/metrics/metrics.go
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
package metrics
|
||||
|
||||
import (
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
)
|
||||
|
||||
type Metrics struct {
|
||||
Requests *prometheus.CounterVec
|
||||
Latency *prometheus.HistogramVec
|
||||
CacheHits prometheus.Counter
|
||||
CacheMisses prometheus.Counter
|
||||
EngineStatus *prometheus.GaugeVec
|
||||
ActiveSearches prometheus.Gauge
|
||||
}
|
||||
|
||||
func New() *Metrics {
|
||||
return &Metrics{
|
||||
Requests: promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "mana_search_requests_total",
|
||||
Help: "Total number of requests",
|
||||
}, []string{"endpoint", "status"}),
|
||||
|
||||
Latency: promauto.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "mana_search_latency_seconds",
|
||||
Help: "Request latency in seconds",
|
||||
Buckets: []float64{0.1, 0.25, 0.5, 1, 2, 5, 10},
|
||||
}, []string{"endpoint"}),
|
||||
|
||||
CacheHits: promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "mana_search_cache_hits_total",
|
||||
Help: "Total cache hits",
|
||||
}),
|
||||
|
||||
CacheMisses: promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "mana_search_cache_misses_total",
|
||||
Help: "Total cache misses",
|
||||
}),
|
||||
|
||||
EngineStatus: promauto.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Name: "mana_search_engine_status",
|
||||
Help: "Search engine status (1=ok, 0=error)",
|
||||
}, []string{"engine"}),
|
||||
|
||||
ActiveSearches: promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Name: "mana_search_active_searches",
|
||||
Help: "Number of active search requests",
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Metrics) RecordRequest(endpoint string, status string, durationSeconds float64) {
|
||||
m.Requests.WithLabelValues(endpoint, status).Inc()
|
||||
m.Latency.WithLabelValues(endpoint).Observe(durationSeconds)
|
||||
}
|
||||
295
services/mana-search-go/internal/search/searxng.go
Normal file
295
services/mana-search-go/internal/search/searxng.go
Normal file
|
|
@ -0,0 +1,295 @@
|
|||
package search
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/manacore/mana-search/internal/config"
|
||||
)
|
||||
|
||||
// SearXNG provider handles communication with the SearXNG meta-search engine.
|
||||
type SearxngProvider struct {
|
||||
baseURL string
|
||||
timeout time.Duration
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func NewSearxngProvider(cfg *config.Config) *SearxngProvider {
|
||||
timeout := time.Duration(cfg.SearxngTimeout) * time.Millisecond
|
||||
return &SearxngProvider{
|
||||
baseURL: cfg.SearxngURL,
|
||||
timeout: timeout,
|
||||
client: &http.Client{
|
||||
Timeout: timeout,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// SearchRequest represents the incoming search request.
|
||||
type SearchRequest struct {
|
||||
Query string `json:"query"`
|
||||
Options *SearchOptions `json:"options,omitempty"`
|
||||
Cache *CacheOptions `json:"cache,omitempty"`
|
||||
}
|
||||
|
||||
type SearchOptions struct {
|
||||
Categories []string `json:"categories,omitempty"`
|
||||
Engines []string `json:"engines,omitempty"`
|
||||
Language string `json:"language,omitempty"`
|
||||
TimeRange string `json:"timeRange,omitempty"`
|
||||
SafeSearch int `json:"safeSearch,omitempty"`
|
||||
Limit int `json:"limit,omitempty"`
|
||||
}
|
||||
|
||||
type CacheOptions struct {
|
||||
Enabled *bool `json:"enabled,omitempty"`
|
||||
TTL int `json:"ttl,omitempty"`
|
||||
}
|
||||
|
||||
func (c *CacheOptions) IsEnabled() bool {
|
||||
if c == nil || c.Enabled == nil {
|
||||
return true
|
||||
}
|
||||
return *c.Enabled
|
||||
}
|
||||
|
||||
// SearchResponse is returned to the client.
|
||||
type SearchResponse struct {
|
||||
Results []SearchResult `json:"results"`
|
||||
Meta SearchMeta `json:"meta"`
|
||||
}
|
||||
|
||||
type SearchResult struct {
|
||||
URL string `json:"url"`
|
||||
Title string `json:"title"`
|
||||
Snippet string `json:"snippet"`
|
||||
Engine string `json:"engine"`
|
||||
Score float64 `json:"score"`
|
||||
PublishedDate string `json:"publishedDate,omitempty"`
|
||||
Thumbnail string `json:"thumbnail,omitempty"`
|
||||
Category string `json:"category"`
|
||||
}
|
||||
|
||||
type SearchMeta struct {
|
||||
Query string `json:"query"`
|
||||
TotalResults int `json:"totalResults"`
|
||||
Engines []string `json:"engines"`
|
||||
Duration int64 `json:"duration"`
|
||||
Cached bool `json:"cached"`
|
||||
CacheKey string `json:"cacheKey,omitempty"`
|
||||
}
|
||||
|
||||
// searxngResponse is the raw response from SearXNG.
|
||||
type searxngResponse struct {
|
||||
Results []searxngResult `json:"results"`
|
||||
}
|
||||
|
||||
type searxngResult struct {
|
||||
URL string `json:"url"`
|
||||
Title string `json:"title"`
|
||||
Content string `json:"content"`
|
||||
Engine string `json:"engine"`
|
||||
Score float64 `json:"score"`
|
||||
PublishedDate string `json:"publishedDate,omitempty"`
|
||||
Thumbnail string `json:"thumbnail,omitempty"`
|
||||
Category string `json:"category"`
|
||||
}
|
||||
|
||||
// Search queries SearXNG and returns normalized, scored results.
|
||||
func (p *SearxngProvider) Search(ctx context.Context, req *SearchRequest) ([]SearchResult, error) {
|
||||
params := url.Values{}
|
||||
params.Set("q", req.Query)
|
||||
params.Set("format", "json")
|
||||
|
||||
if req.Options != nil {
|
||||
if len(req.Options.Categories) > 0 {
|
||||
params.Set("categories", strings.Join(req.Options.Categories, ","))
|
||||
}
|
||||
if len(req.Options.Engines) > 0 {
|
||||
params.Set("engines", strings.Join(req.Options.Engines, ","))
|
||||
}
|
||||
if req.Options.Language != "" {
|
||||
params.Set("language", req.Options.Language)
|
||||
}
|
||||
if req.Options.TimeRange != "" {
|
||||
params.Set("time_range", req.Options.TimeRange)
|
||||
}
|
||||
params.Set("safesearch", fmt.Sprintf("%d", req.Options.SafeSearch))
|
||||
}
|
||||
|
||||
reqURL := fmt.Sprintf("%s/search?%s", p.baseURL, params.Encode())
|
||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
|
||||
resp, err := p.client.Do(httpReq)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil, fmt.Errorf("searxng timeout")
|
||||
}
|
||||
return nil, fmt.Errorf("searxng unavailable: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return nil, fmt.Errorf("searxng returned %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
var sxResp searxngResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&sxResp); err != nil {
|
||||
return nil, fmt.Errorf("decode searxng response: %w", err)
|
||||
}
|
||||
|
||||
return normalizeResults(sxResp.Results, req.Options), nil
|
||||
}
|
||||
|
||||
// HealthCheck pings SearXNG.
|
||||
func (p *SearxngProvider) HealthCheck(ctx context.Context) (string, int64) {
|
||||
start := time.Now()
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, p.baseURL+"/healthz", nil)
|
||||
resp, err := p.client.Do(req)
|
||||
latency := time.Since(start).Milliseconds()
|
||||
|
||||
if err != nil || resp.StatusCode != http.StatusOK {
|
||||
return "error", latency
|
||||
}
|
||||
resp.Body.Close()
|
||||
return "ok", latency
|
||||
}
|
||||
|
||||
// GetEngines fetches available engines from SearXNG config.
|
||||
func (p *SearxngProvider) GetEngines(ctx context.Context) []string {
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, p.baseURL+"/config", nil)
|
||||
resp, err := p.client.Do(req)
|
||||
if err != nil {
|
||||
slog.Warn("failed to fetch searxng engines", "error", err)
|
||||
return nil
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
var cfg struct {
|
||||
Engines map[string]any `json:"engines"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&cfg); err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
engines := make([]string, 0, len(cfg.Engines))
|
||||
for name := range cfg.Engines {
|
||||
engines = append(engines, name)
|
||||
}
|
||||
sort.Strings(engines)
|
||||
return engines
|
||||
}
|
||||
|
||||
// BuildCacheKey creates a deterministic cache key from the search request.
|
||||
func BuildCacheKey(req *SearchRequest) string {
|
||||
var categories, engines, language, timeRange string
|
||||
safeSearch := 0
|
||||
|
||||
if req.Options != nil {
|
||||
cats := make([]string, len(req.Options.Categories))
|
||||
copy(cats, req.Options.Categories)
|
||||
sort.Strings(cats)
|
||||
categories = strings.Join(cats, ",")
|
||||
|
||||
engs := make([]string, len(req.Options.Engines))
|
||||
copy(engs, req.Options.Engines)
|
||||
sort.Strings(engs)
|
||||
engines = strings.Join(engs, ",")
|
||||
|
||||
language = strings.ToLower(req.Options.Language)
|
||||
timeRange = req.Options.TimeRange
|
||||
safeSearch = req.Options.SafeSearch
|
||||
}
|
||||
|
||||
return fmt.Sprintf("search:%s:%s:%s:%s:%s:%d",
|
||||
strings.ToLower(req.Query), categories, engines, language, timeRange, safeSearch)
|
||||
}
|
||||
|
||||
// Trusted domains get a score boost.
|
||||
var trustedDomains = map[string]bool{
|
||||
"wikipedia.org": true,
|
||||
"github.com": true,
|
||||
"stackoverflow.com": true,
|
||||
}
|
||||
|
||||
func normalizeResults(raw []searxngResult, opts *SearchOptions) []SearchResult {
|
||||
seen := make(map[string]bool)
|
||||
var results []SearchResult
|
||||
|
||||
for _, r := range raw {
|
||||
if seen[r.URL] {
|
||||
continue
|
||||
}
|
||||
seen[r.URL] = true
|
||||
|
||||
score := scoreResult(r)
|
||||
results = append(results, SearchResult{
|
||||
URL: r.URL,
|
||||
Title: r.Title,
|
||||
Snippet: r.Content,
|
||||
Engine: r.Engine,
|
||||
Score: score,
|
||||
PublishedDate: r.PublishedDate,
|
||||
Thumbnail: r.Thumbnail,
|
||||
Category: r.Category,
|
||||
})
|
||||
}
|
||||
|
||||
sort.Slice(results, func(i, j int) bool {
|
||||
return results[i].Score > results[j].Score
|
||||
})
|
||||
|
||||
limit := 10
|
||||
if opts != nil && opts.Limit > 0 && opts.Limit <= 50 {
|
||||
limit = opts.Limit
|
||||
}
|
||||
if len(results) > limit {
|
||||
results = results[:limit]
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
func scoreResult(r searxngResult) float64 {
|
||||
score := 0.5
|
||||
|
||||
if len(r.Content) > 100 {
|
||||
score += 0.1
|
||||
}
|
||||
|
||||
// Check if URL is from a trusted domain
|
||||
if u, err := url.Parse(r.URL); err == nil {
|
||||
host := strings.ToLower(u.Hostname())
|
||||
for domain := range trustedDomains {
|
||||
if strings.HasSuffix(host, domain) {
|
||||
score += 0.15
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(r.URL) > 200 {
|
||||
score -= 0.05
|
||||
}
|
||||
|
||||
return math.Max(0, math.Min(1, score))
|
||||
}
|
||||
10
services/mana-search-go/package.json
Normal file
10
services/mana-search-go/package.json
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
{
|
||||
"name": "@manacore/mana-search-go",
|
||||
"version": "1.0.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "go run ./cmd/server",
|
||||
"build": "go build -o bin/mana-search ./cmd/server",
|
||||
"test": "go test ./..."
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue