diff --git a/CLAUDE.md b/CLAUDE.md index d335fcb3b..4a949f391 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -132,15 +132,14 @@ manacore-monorepo/ │ ├── mana-core-auth/ # Central authentication service │ ├── mana-credits/ # Credit system (Hono + Bun, extracted from auth) │ ├── mana-user/ # User settings, tags, storage (Hono + Bun, extracted from auth) -│ ├── mana-search/ # Central search & content extraction (NestJS, legacy) -│ ├── mana-search-go/ # Central search & content extraction (Go, active) -│ ├── mana-crawler/ # Web crawler service +│ ├── mana-subscriptions/ # Subscription billing (Hono + Bun, extracted from auth) +│ ├── mana-search-go/ # Central search & content extraction (Go) +│ ├── mana-crawler-go/ # Web crawler service (Go) │ ├── mana-llm/ # Central LLM abstraction service │ ├── mana-landing-builder/# Org landing page builder (Astro → Cloudflare Pages) │ ├── mana-media/ # Central media platform (CAS, thumbnails) -│ ├── mana-api-gateway/ # API gateway with rate limiting -│ ├── mana-notify/ # Notification service (NestJS, legacy) -│ ├── mana-notify-go/ # Notification service (Go, active) +│ ├── mana-api-gateway-go/ # API gateway with rate limiting (Go) +│ ├── mana-notify-go/ # Notification service (Go) │ ├── mana-image-gen/ # Local AI image generation (FLUX) │ ├── mana-stt/ # Speech-to-text service │ ├── mana-tts/ # Text-to-speech service @@ -449,14 +448,11 @@ GET /metrics #### Starting the Service ```bash -# Start SearXNG + Redis (for local NestJS development) -cd services/mana-search && docker-compose -f docker-compose.dev.yml up -d +# Start SearXNG + Redis for local development +cd services/mana-search-go && docker-compose -f docker-compose.dev.yml up -d -# Start NestJS API -pnpm --filter @mana-search/service dev - -# Or start everything via Docker -cd services/mana-search && docker-compose up -d +# Start Go search service +cd services/mana-search-go && go run ./cmd/server ``` #### Environment Variables @@ -936,9 +932,9 @@ Each project has its own `CLAUDE.md` with detailed information: - `apps/chat/CLAUDE.md` - Chat API endpoints, AI models - `apps/picture/CLAUDE.md` - AI image generation - `services/mana-core-auth/` - Central authentication service -- `services/mana-search/CLAUDE.md` - Search & content extraction service (NestJS, legacy) -- `services/mana-search-go/CLAUDE.md` - Search & content extraction service (Go, active) -- `services/mana-crawler/CLAUDE.md` - Web crawler service +- `services/mana-search-go/CLAUDE.md` - Search & content extraction service (Go) +- `services/mana-crawler-go/CLAUDE.md` - Web crawler service (Go) +- `services/mana-notify-go/CLAUDE.md` - Notification service (Go) - `services/mana-llm/CLAUDE.md` - Central LLM abstraction service - `services/mana-landing-builder/CLAUDE.md` - Org landing page builder service diff --git a/docker-compose.macmini.yml b/docker-compose.macmini.yml index d90075db0..f5f2a7017 100644 --- a/docker-compose.macmini.yml +++ b/docker-compose.macmini.yml @@ -325,8 +325,8 @@ services: container_name: mana-core-searxng restart: always volumes: - - ./services/mana-search/searxng/settings.yml:/etc/searxng/settings.yml:ro - - ./services/mana-search/searxng/limiter.toml:/etc/searxng/limiter.toml:ro + - ./services/mana-search-go/searxng/settings.yml:/etc/searxng/settings.yml:ro + - ./services/mana-search-go/searxng/limiter.toml:/etc/searxng/limiter.toml:ro environment: SEARXNG_BASE_URL: http://searxng:8080 SEARXNG_SECRET: ${SEARXNG_SECRET:-change-me-searxng-secret} diff --git a/services/mana-crawler/.env.example b/services/mana-crawler/.env.example deleted file mode 100644 index 04458b8a0..000000000 --- a/services/mana-crawler/.env.example +++ /dev/null @@ -1,24 +0,0 @@ -# Server -PORT=3023 -NODE_ENV=development - -# Database -DATABASE_URL=postgresql://manacore:devpassword@localhost:5432/manacore - -# Redis (Queue) -REDIS_HOST=localhost -REDIS_PORT=6379 -REDIS_PASSWORD= - -# Crawling -CRAWLER_USER_AGENT=ManaCoreCrawler/1.0 (+https://manacore.io/bot) -CRAWLER_DEFAULT_RATE_LIMIT=2 -CRAWLER_DEFAULT_MAX_DEPTH=3 -CRAWLER_DEFAULT_MAX_PAGES=100 -CRAWLER_TIMEOUT=30000 - -# External Services (optional - for single-page extraction fallback) -MANA_SEARCH_URL=http://localhost:3021 - -# CORS -CORS_ORIGINS=http://localhost:3000,http://localhost:5173,http://localhost:8081 diff --git a/services/mana-crawler/.gitignore b/services/mana-crawler/.gitignore deleted file mode 100644 index 33af69cc1..000000000 --- a/services/mana-crawler/.gitignore +++ /dev/null @@ -1,20 +0,0 @@ -# Dependencies -node_modules - -# Build -dist - -# Environment -.env -.env.local - -# IDE -.idea -.vscode - -# Debug -*.log -npm-debug.log* - -# Test -coverage diff --git a/services/mana-crawler/CLAUDE.md b/services/mana-crawler/CLAUDE.md deleted file mode 100644 index 1a89999d0..000000000 --- a/services/mana-crawler/CLAUDE.md +++ /dev/null @@ -1,297 +0,0 @@ -# Mana Crawler Service - -Web crawler microservice for systematic website crawling and content extraction. - -## Overview - -- **Port**: 3023 -- **Technology**: NestJS + BullMQ + Cheerio + PostgreSQL + Redis -- **Purpose**: Crawl websites, extract structured content, and queue-based processing - -## Architecture - -``` -┌─────────────────────────────────────────────────────────────┐ -│ mana-crawler (Port 3023) │ -│ │ -│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ -│ │ Crawl API │ │ Queue │ │ Parser │ │ -│ │ Controller │──│ Service │──│ Service │ │ -│ └─────────────┘ │ (BullMQ) │ │ (Cheerio) │ │ -│ └─────────────┘ └─────────────┘ │ -│ │ │ │ -│ ┌─────┴────────────────┴─────┐ │ -│ │ Storage Service │ │ -│ │ (PostgreSQL + Redis) │ │ -│ └─────────────────────────────┘ │ -└─────────────────────────────────────────────────────────────┘ -``` - -## Quick Start - -### Development - -```bash -# 1. Start Redis and PostgreSQL (from monorepo root) -pnpm docker:up - -# 2. Install dependencies -pnpm install - -# 3. Push database schema -pnpm db:push - -# 4. Start in development mode -pnpm dev -``` - -### Production - -```bash -pnpm build -pnpm start -``` - -## API Endpoints - -### Crawl Jobs - -| Method | Endpoint | Description | -|--------|----------|-------------| -| POST | `/api/v1/crawl` | Start a new crawl job | -| GET | `/api/v1/crawl/:jobId` | Get job status | -| GET | `/api/v1/crawl/:jobId/results` | Get crawl results (paginated) | -| DELETE | `/api/v1/crawl/:jobId` | Cancel a crawl job | -| POST | `/api/v1/crawl/:jobId/pause` | Pause a running job | -| POST | `/api/v1/crawl/:jobId/resume` | Resume a paused job | - -### Instant Extract - -| Method | Endpoint | Description | -|--------|----------|-------------| -| POST | `/api/v1/extract` | Extract single page (proxy to mana-search) | - -### System - -| Method | Endpoint | Description | -|--------|----------|-------------| -| GET | `/health` | Health check | -| GET | `/metrics` | Prometheus metrics | -| GET | `/queue/dashboard` | Bull Board dashboard | - -## Usage Examples - -### Start a Crawl Job - -```bash -curl -X POST http://localhost:3023/api/v1/crawl \ - -H "Content-Type: application/json" \ - -d '{ - "startUrl": "https://docs.example.com", - "config": { - "maxDepth": 3, - "maxPages": 500, - "respectRobots": true, - "rateLimit": 2, - "includePatterns": ["/docs/*"], - "excludePatterns": ["/api/*", "*.pdf"], - "selectors": { - "content": "article.main-content", - "title": "h1.page-title" - }, - "output": { - "format": "markdown", - "includeScreenshots": false - } - } - }' - -# Response: -# { -# "jobId": "uuid", -# "status": "pending", -# "estimatedPages": 500, -# "queuePosition": 3 -# } -``` - -### Check Job Status - -```bash -curl http://localhost:3023/api/v1/crawl/{jobId} - -# Response: -# { -# "jobId": "uuid", -# "status": "running", -# "progress": { -# "discovered": 245, -# "crawled": 127, -# "failed": 3, -# "queued": 115 -# }, -# "startedAt": "2024-01-29T12:00:00Z", -# "averagePageTime": 450 -# } -``` - -### Get Results - -```bash -curl "http://localhost:3023/api/v1/crawl/{jobId}/results?page=1&limit=50" - -# Response: -# { -# "results": [...], -# "pagination": { -# "page": 1, -# "limit": 50, -# "total": 127 -# } -# } -``` - -## Environment Variables - -| Variable | Default | Description | -|----------|---------|-------------| -| `PORT` | 3023 | API port | -| `DATABASE_URL` | - | PostgreSQL connection URL | -| `REDIS_HOST` | localhost | Redis host | -| `REDIS_PORT` | 6379 | Redis port | -| `CRAWLER_USER_AGENT` | ManaCoreCrawler/1.0 | Crawler user agent | -| `CRAWLER_DEFAULT_RATE_LIMIT` | 2 | Default requests/second | -| `CRAWLER_DEFAULT_MAX_DEPTH` | 3 | Default max crawl depth | -| `CRAWLER_DEFAULT_MAX_PAGES` | 100 | Default max pages per job | -| `CRAWLER_TIMEOUT` | 30000 | Request timeout (ms) | -| `MANA_SEARCH_URL` | http://localhost:3021 | mana-search URL (for extract fallback) | - -## Development Commands - -```bash -# Install dependencies -pnpm install - -# Start development server -pnpm dev - -# Build for production -pnpm build - -# Start production server -pnpm start - -# Type checking -pnpm type-check - -# Linting -pnpm lint - -# Database commands -pnpm db:push # Push schema to database -pnpm db:generate # Generate migrations -pnpm db:migrate # Run migrations -pnpm db:studio # Open Drizzle Studio -``` - -## Database Schema - -The crawler uses its own schema (`crawler`) in the shared ManaCore database: - -- `crawler.crawl_jobs` - Crawl job configuration and status -- `crawler.crawl_results` - Individual page results - -## Queue System - -Uses BullMQ with Redis for job processing: - -- **Queue Name**: `crawl` -- **Concurrency**: Configurable (default: 5) -- **Retry**: 3 attempts with exponential backoff -- **Dashboard**: Available at `/queue/dashboard` - -## Robots.txt Compliance - -The crawler respects robots.txt by default: -- Checks robots.txt before crawling each domain -- Caches robots.txt rules in Redis (24h TTL) -- Can be disabled per-job with `respectRobots: false` - -## Rate Limiting - -Built-in rate limiting to be a good citizen: -- Per-domain rate limiting -- Configurable delay between requests -- Default: 2 requests/second/domain - -## Project Structure - -``` -services/mana-crawler/ -├── src/ -│ ├── main.ts # Application entry point -│ ├── app.module.ts # Root module -│ ├── config/ -│ │ └── configuration.ts # App configuration -│ ├── db/ -│ │ ├── schema/ # Drizzle schemas -│ │ ├── database.module.ts # Database provider -│ │ └── connection.ts # DB connection -│ ├── crawler/ # Crawl job management -│ │ ├── crawler.controller.ts -│ │ ├── crawler.service.ts -│ │ └── dto/ -│ ├── queue/ # BullMQ queue processing -│ │ ├── queue.module.ts -│ │ └── processors/ -│ ├── parser/ # HTML parsing (Cheerio) -│ ├── robots/ # robots.txt handling -│ ├── cache/ # Redis caching -│ ├── metrics/ # Prometheus metrics -│ └── health/ # Health check -├── drizzle.config.ts -├── package.json -├── tsconfig.json -└── Dockerfile -``` - -## Integration with Other Services - -### mana-search -The crawler can use mana-search for single-page extraction as a fallback: -```typescript -POST http://mana-search:3021/api/v1/extract -``` - -### mana-api-gateway -The crawler can be exposed via the API gateway for monetization: -``` -POST /v1/crawler/start → 5 Credits/Job + 1 Credit/100 pages -GET /v1/crawler/:id → 0 Credits -``` - -## Troubleshooting - -### Redis connection issues - -```bash -# Check Redis -docker exec mana-redis redis-cli ping - -# Check queue status -curl http://localhost:3023/queue/dashboard -``` - -### Jobs stuck in pending - -Check that: -1. Redis is running -2. The queue processor is active -3. No rate limit issues - -### High memory usage - -The crawler loads pages into memory for parsing. For large crawls: -- Reduce `maxPages` per job -- Increase job concurrency instead -- Monitor with `/metrics` diff --git a/services/mana-crawler/Dockerfile b/services/mana-crawler/Dockerfile deleted file mode 100644 index a8c6413a5..000000000 --- a/services/mana-crawler/Dockerfile +++ /dev/null @@ -1,56 +0,0 @@ -# syntax=docker/dockerfile:1 -# Build stage -FROM node:20-alpine AS builder - -RUN corepack enable && corepack prepare pnpm@9.15.0 --activate - -WORKDIR /app - -# Copy workspace files -COPY pnpm-workspace.yaml ./ -COPY pnpm-lock.yaml ./ -COPY package.json ./ - -# Copy service files -COPY services/mana-crawler/package.json ./services/mana-crawler/ - -# Copy shared packages -COPY packages/shared-drizzle-config/package.json ./packages/shared-drizzle-config/ - -# Install dependencies -RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --frozen-lockfile - -# Copy source code -COPY services/mana-crawler ./services/mana-crawler -COPY packages/shared-drizzle-config ./packages/shared-drizzle-config - -# Build -WORKDIR /app/services/mana-crawler -RUN pnpm build - -# Production stage -FROM node:20-alpine AS runner - -RUN corepack enable && corepack prepare pnpm@9.15.0 --activate - -WORKDIR /app - -# Copy package files -COPY --from=builder /app/pnpm-workspace.yaml ./ -COPY --from=builder /app/pnpm-lock.yaml ./ -COPY --from=builder /app/package.json ./ -COPY --from=builder /app/services/mana-crawler/package.json ./services/mana-crawler/ -COPY --from=builder /app/packages/shared-drizzle-config/package.json ./packages/shared-drizzle-config/ - -# Install production dependencies only -RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --frozen-lockfile --prod - -# Copy built files -COPY --from=builder /app/services/mana-crawler/dist ./services/mana-crawler/dist -COPY --from=builder /app/packages/shared-drizzle-config/dist ./packages/shared-drizzle-config/dist - -WORKDIR /app/services/mana-crawler - -EXPOSE 3023 - -CMD ["node", "dist/main"] diff --git a/services/mana-crawler/drizzle.config.ts b/services/mana-crawler/drizzle.config.ts deleted file mode 100644 index 5741a38eb..000000000 --- a/services/mana-crawler/drizzle.config.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { createDrizzleConfig } from '@manacore/shared-drizzle-config'; - -export default createDrizzleConfig({ - dbName: 'manacore', - schemaFilter: ['crawler'], -}); diff --git a/services/mana-crawler/nest-cli.json b/services/mana-crawler/nest-cli.json deleted file mode 100644 index 95538fb90..000000000 --- a/services/mana-crawler/nest-cli.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "$schema": "https://json.schemastore.org/nest-cli", - "collection": "@nestjs/schematics", - "sourceRoot": "src", - "compilerOptions": { - "deleteOutDir": true - } -} diff --git a/services/mana-crawler/package.json b/services/mana-crawler/package.json deleted file mode 100644 index cf7b093f0..000000000 --- a/services/mana-crawler/package.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "name": "@manacore/mana-crawler", - "version": "0.1.0", - "description": "Web crawler microservice for systematic website crawling and content extraction", - "private": true, - "license": "UNLICENSED", - "scripts": { - "build": "nest build", - "dev": "nest start --watch", - "start": "node dist/main", - "start:dev": "nest start --watch", - "start:debug": "nest start --debug --watch", - "start:prod": "node dist/main", - "lint": "eslint \"{src,test}/**/*.ts\" --fix", - "type-check": "tsc --noEmit", - "test": "jest", - "test:watch": "jest --watch", - "test:cov": "jest --coverage", - "db:push": "drizzle-kit push", - "db:generate": "drizzle-kit generate", - "db:migrate": "drizzle-kit migrate", - "db:studio": "drizzle-kit studio" - }, - "dependencies": { - "@bull-board/api": "^6.6.0", - "@bull-board/express": "^6.6.0", - "@bull-board/nestjs": "^6.6.0", - "@nestjs/bullmq": "^10.2.3", - "@nestjs/common": "^10.4.17", - "@nestjs/config": "^3.3.0", - "@nestjs/core": "^10.4.17", - "@nestjs/platform-express": "^10.4.17", - "bullmq": "^5.34.8", - "cheerio": "^1.0.0", - "class-transformer": "^0.5.1", - "class-validator": "^0.14.1", - "drizzle-orm": "^0.38.4", - "ioredis": "^5.4.2", - "postgres": "^3.4.5", - "prom-client": "^15.1.3", - "reflect-metadata": "^0.2.2", - "robots-parser": "^3.0.1", - "rxjs": "^7.8.1", - "turndown": "^7.2.0" - }, - "devDependencies": { - "@manacore/shared-drizzle-config": "workspace:*", - "@nestjs/cli": "^10.4.9", - "@nestjs/schematics": "^10.2.3", - "@nestjs/testing": "^10.4.17", - "@types/express": "^5.0.0", - "@types/jest": "^29.5.14", - "@types/node": "^22.10.5", - "@types/turndown": "^5.0.5", - "drizzle-kit": "^0.30.4", - "jest": "^29.7.0", - "ts-jest": "^29.2.5", - "ts-node": "^10.9.2", - "typescript": "^5.7.2" - } -} diff --git a/services/mana-crawler/src/app.module.ts b/services/mana-crawler/src/app.module.ts deleted file mode 100644 index 4ae3f1944..000000000 --- a/services/mana-crawler/src/app.module.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { Module } from '@nestjs/common'; -import { ConfigModule } from '@nestjs/config'; -import { BullModule } from '@nestjs/bullmq'; -import { BullBoardModule } from '@bull-board/nestjs'; -import { ExpressAdapter } from '@bull-board/express'; -import configuration from './config/configuration'; -import { DatabaseModule } from './db/database.module'; -import { HealthModule } from './health/health.module'; -import { MetricsModule } from './metrics/metrics.module'; -import { CacheModule } from './cache/cache.module'; -import { CrawlerModule } from './crawler/crawler.module'; -import { QueueModule } from './queue/queue.module'; -import { ProcessorModule } from './queue/processor.module'; -import { ParserModule } from './parser/parser.module'; -import { RobotsModule } from './robots/robots.module'; - -@Module({ - imports: [ - ConfigModule.forRoot({ - isGlobal: true, - load: [configuration], - }), - BullModule.forRootAsync({ - useFactory: () => ({ - connection: { - host: process.env.REDIS_HOST || 'localhost', - port: parseInt(process.env.REDIS_PORT || '6379', 10), - password: process.env.REDIS_PASSWORD || undefined, - }, - }), - }), - BullBoardModule.forRoot({ - route: '/queue/dashboard', - adapter: ExpressAdapter, - }), - DatabaseModule, - HealthModule, - MetricsModule, - CacheModule, - RobotsModule, - ParserModule, - QueueModule, - ProcessorModule, - CrawlerModule, - ], -}) -export class AppModule {} diff --git a/services/mana-crawler/src/cache/cache.module.ts b/services/mana-crawler/src/cache/cache.module.ts deleted file mode 100644 index 618591395..000000000 --- a/services/mana-crawler/src/cache/cache.module.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { Module } from '@nestjs/common'; -import { CacheService } from './cache.service'; -import { MetricsModule } from '../metrics/metrics.module'; - -@Module({ - imports: [MetricsModule], - providers: [CacheService], - exports: [CacheService], -}) -export class CacheModule {} diff --git a/services/mana-crawler/src/cache/cache.service.ts b/services/mana-crawler/src/cache/cache.service.ts deleted file mode 100644 index 2e4e60ef7..000000000 --- a/services/mana-crawler/src/cache/cache.service.ts +++ /dev/null @@ -1,152 +0,0 @@ -import { Injectable, Logger, OnModuleInit, OnModuleDestroy } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import Redis from 'ioredis'; -import { MetricsService } from '../metrics/metrics.service'; - -@Injectable() -export class CacheService implements OnModuleInit, OnModuleDestroy { - private readonly logger = new Logger(CacheService.name); - private client: Redis | null = null; - private readonly keyPrefix: string; - - private stats = { - hits: 0, - misses: 0, - }; - - constructor( - private readonly configService: ConfigService, - private readonly metricsService: MetricsService, - ) { - this.keyPrefix = this.configService.get('redis.keyPrefix', 'mana-crawler:'); - } - - async onModuleInit() { - const host = this.configService.get('redis.host', 'localhost'); - const port = this.configService.get('redis.port', 6379); - const password = this.configService.get('redis.password'); - - try { - this.client = new Redis({ - host, - port, - password, - retryStrategy: (times) => { - if (times > 3) { - this.logger.warn('Redis connection failed, running without cache'); - return null; - } - return Math.min(times * 200, 2000); - }, - maxRetriesPerRequest: 1, - }); - - this.client.on('error', (err) => { - this.logger.error(`Redis error: ${err.message}`); - }); - - this.client.on('connect', () => { - this.logger.log(`Connected to Redis at ${host}:${port}`); - }); - - await this.client.ping(); - } catch (error) { - this.logger.warn(`Could not connect to Redis: ${error}. Running without cache.`); - this.client = null; - } - } - - async onModuleDestroy() { - if (this.client) { - await this.client.quit(); - } - } - - private buildKey(key: string): string { - return `${this.keyPrefix}${key}`; - } - - async get(key: string): Promise { - if (!this.client) return null; - - try { - const data = await this.client.get(this.buildKey(key)); - if (data) { - this.stats.hits++; - this.metricsService.recordCacheHit(); - return JSON.parse(data); - } - this.stats.misses++; - this.metricsService.recordCacheMiss(); - return null; - } catch (error) { - this.logger.error(`Cache get error: ${error}`); - return null; - } - } - - async set(key: string, value: unknown, ttlSeconds: number): Promise { - if (!this.client) return; - - try { - await this.client.setex(this.buildKey(key), ttlSeconds, JSON.stringify(value)); - } catch (error) { - this.logger.error(`Cache set error: ${error}`); - } - } - - async delete(key: string): Promise { - if (!this.client) return; - - try { - await this.client.del(this.buildKey(key)); - } catch (error) { - this.logger.error(`Cache delete error: ${error}`); - } - } - - async clear(pattern?: string): Promise { - if (!this.client) return 0; - - try { - const searchPattern = pattern - ? `${this.keyPrefix}${pattern}` - : `${this.keyPrefix}*`; - const keys = await this.client.keys(searchPattern); - if (keys.length > 0) { - await this.client.del(...keys); - } - return keys.length; - } catch (error) { - this.logger.error(`Cache clear error: ${error}`); - return 0; - } - } - - getStats() { - const total = this.stats.hits + this.stats.misses; - return { - hits: this.stats.hits, - misses: this.stats.misses, - hitRate: total > 0 ? this.stats.hits / total : 0, - }; - } - - async healthCheck(): Promise<{ status: string; latency: number }> { - if (!this.client) { - return { status: 'disabled', latency: 0 }; - } - - const start = Date.now(); - try { - await this.client.ping(); - return { status: 'ok', latency: Date.now() - start }; - } catch { - return { status: 'error', latency: Date.now() - start }; - } - } - - isConnected(): boolean { - return this.client !== null && this.client.status === 'ready'; - } -} diff --git a/services/mana-crawler/src/common/filters/http-exception.filter.ts b/services/mana-crawler/src/common/filters/http-exception.filter.ts deleted file mode 100644 index b86fb7885..000000000 --- a/services/mana-crawler/src/common/filters/http-exception.filter.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { - ExceptionFilter, - Catch, - ArgumentsHost, - HttpException, - HttpStatus, - Logger, -} from '@nestjs/common'; -import { Response } from 'express'; - -@Catch() -export class HttpExceptionFilter implements ExceptionFilter { - private readonly logger = new Logger(HttpExceptionFilter.name); - - catch(exception: unknown, host: ArgumentsHost) { - const ctx = host.switchToHttp(); - const response = ctx.getResponse(); - - let status: number; - let message: string; - let error: string; - - if (exception instanceof HttpException) { - status = exception.getStatus(); - const exceptionResponse = exception.getResponse(); - message = - typeof exceptionResponse === 'string' - ? exceptionResponse - : (exceptionResponse as { message?: string }).message || - exception.message; - error = exception.name; - } else if (exception instanceof Error) { - status = HttpStatus.INTERNAL_SERVER_ERROR; - message = exception.message; - error = 'Internal Server Error'; - this.logger.error(`Unhandled exception: ${exception.message}`, exception.stack); - } else { - status = HttpStatus.INTERNAL_SERVER_ERROR; - message = 'An unexpected error occurred'; - error = 'Internal Server Error'; - this.logger.error(`Unknown exception type: ${exception}`); - } - - response.status(status).json({ - statusCode: status, - error, - message, - timestamp: new Date().toISOString(), - }); - } -} diff --git a/services/mana-crawler/src/config/configuration.ts b/services/mana-crawler/src/config/configuration.ts deleted file mode 100644 index 76c7957c1..000000000 --- a/services/mana-crawler/src/config/configuration.ts +++ /dev/null @@ -1,49 +0,0 @@ -export default () => ({ - port: parseInt(process.env.PORT || '3023', 10), - nodeEnv: process.env.NODE_ENV || 'development', - - cors: { - origins: process.env.CORS_ORIGINS?.split(',') || [ - 'http://localhost:3000', - 'http://localhost:5173', - 'http://localhost:8081', - ], - }, - - database: { - url: - process.env.DATABASE_URL || - 'postgresql://manacore:devpassword@localhost:5432/manacore', - }, - - redis: { - host: process.env.REDIS_HOST || 'localhost', - port: parseInt(process.env.REDIS_PORT || '6379', 10), - password: process.env.REDIS_PASSWORD, - keyPrefix: 'mana-crawler:', - }, - - crawler: { - userAgent: - process.env.CRAWLER_USER_AGENT || - 'ManaCoreCrawler/1.0 (+https://manacore.io/bot)', - defaultRateLimit: parseFloat(process.env.CRAWLER_DEFAULT_RATE_LIMIT || '2'), - defaultMaxDepth: parseInt(process.env.CRAWLER_DEFAULT_MAX_DEPTH || '3', 10), - defaultMaxPages: parseInt(process.env.CRAWLER_DEFAULT_MAX_PAGES || '100', 10), - timeout: parseInt(process.env.CRAWLER_TIMEOUT || '30000', 10), - }, - - queue: { - concurrency: parseInt(process.env.QUEUE_CONCURRENCY || '5', 10), - maxRetries: parseInt(process.env.QUEUE_MAX_RETRIES || '3', 10), - }, - - cache: { - robotsTtl: parseInt(process.env.CACHE_ROBOTS_TTL || '86400', 10), // 24 hours - resultsTtl: parseInt(process.env.CACHE_RESULTS_TTL || '3600', 10), // 1 hour - }, - - externalServices: { - manaSearchUrl: process.env.MANA_SEARCH_URL || 'http://localhost:3021', - }, -}); diff --git a/services/mana-crawler/src/crawler/crawler.controller.ts b/services/mana-crawler/src/crawler/crawler.controller.ts deleted file mode 100644 index 454845146..000000000 --- a/services/mana-crawler/src/crawler/crawler.controller.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { - Controller, - Post, - Get, - Delete, - Body, - Param, - Query, - ParseUUIDPipe, - ParseIntPipe, - DefaultValuePipe, - HttpCode, - HttpStatus, -} from '@nestjs/common'; -import { CrawlerService } from './crawler.service'; -import { StartCrawlDto } from './dto/start-crawl.dto'; -import { CrawlJobResponse, CrawlResultResponse, PaginatedResults } from './dto/crawl-response.dto'; - -@Controller('crawl') -export class CrawlerController { - constructor(private readonly crawlerService: CrawlerService) {} - - @Post() - async startCrawl(@Body() dto: StartCrawlDto): Promise { - return this.crawlerService.startCrawl(dto); - } - - @Get() - async listJobs( - @Query('page', new DefaultValuePipe(1), ParseIntPipe) page: number, - @Query('limit', new DefaultValuePipe(20), ParseIntPipe) limit: number, - @Query('status') status?: string, - ): Promise> { - return this.crawlerService.listJobs(page, limit, status); - } - - @Get(':jobId') - async getJob( - @Param('jobId', ParseUUIDPipe) jobId: string, - ): Promise { - return this.crawlerService.getJob(jobId); - } - - @Get(':jobId/results') - async getJobResults( - @Param('jobId', ParseUUIDPipe) jobId: string, - @Query('page', new DefaultValuePipe(1), ParseIntPipe) page: number, - @Query('limit', new DefaultValuePipe(50), ParseIntPipe) limit: number, - ): Promise> { - return this.crawlerService.getJobResults(jobId, page, limit); - } - - @Delete(':jobId') - @HttpCode(HttpStatus.NO_CONTENT) - async cancelJob( - @Param('jobId', ParseUUIDPipe) jobId: string, - ): Promise { - return this.crawlerService.cancelJob(jobId); - } - - @Post(':jobId/pause') - async pauseJob( - @Param('jobId', ParseUUIDPipe) jobId: string, - ): Promise { - return this.crawlerService.pauseJob(jobId); - } - - @Post(':jobId/resume') - async resumeJob( - @Param('jobId', ParseUUIDPipe) jobId: string, - ): Promise { - return this.crawlerService.resumeJob(jobId); - } -} diff --git a/services/mana-crawler/src/crawler/crawler.module.ts b/services/mana-crawler/src/crawler/crawler.module.ts deleted file mode 100644 index 7fc3eb44a..000000000 --- a/services/mana-crawler/src/crawler/crawler.module.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { Module } from '@nestjs/common'; -import { CrawlerController } from './crawler.controller'; -import { CrawlerService } from './crawler.service'; -import { QueueModule } from '../queue/queue.module'; -import { MetricsModule } from '../metrics/metrics.module'; - -@Module({ - imports: [QueueModule, MetricsModule], - controllers: [CrawlerController], - providers: [CrawlerService], - exports: [CrawlerService], -}) -export class CrawlerModule {} diff --git a/services/mana-crawler/src/crawler/crawler.service.ts b/services/mana-crawler/src/crawler/crawler.service.ts deleted file mode 100644 index b03adc9c3..000000000 --- a/services/mana-crawler/src/crawler/crawler.service.ts +++ /dev/null @@ -1,337 +0,0 @@ -import { Injectable, Logger, Inject, NotFoundException, BadRequestException } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { eq, desc, count } from 'drizzle-orm'; -import { DATABASE_CONNECTION } from '../db/database.module'; -import { crawlJobs, crawlResults, type NewCrawlJob, type CrawlJob, type CrawlResult } from '../db/schema'; -import { QueueService } from '../queue/queue.service'; -import { MetricsService } from '../metrics/metrics.service'; -import { StartCrawlDto } from './dto/start-crawl.dto'; -import { - CrawlJobResponse, - CrawlResultResponse, - PaginatedResults, - toCrawlJobResponse, - toCrawlResultResponse, -} from './dto/crawl-response.dto'; - -@Injectable() -export class CrawlerService { - private readonly logger = new Logger(CrawlerService.name); - private readonly defaultMaxDepth: number; - private readonly defaultMaxPages: number; - private readonly defaultRateLimit: number; - - constructor( - private readonly configService: ConfigService, - private readonly queueService: QueueService, - private readonly metricsService: MetricsService, - @Inject(DATABASE_CONNECTION) private readonly db: any, - ) { - this.defaultMaxDepth = this.configService.get('crawler.defaultMaxDepth', 3); - this.defaultMaxPages = this.configService.get('crawler.defaultMaxPages', 100); - this.defaultRateLimit = this.configService.get('crawler.defaultRateLimit', 2); - } - - async startCrawl(dto: StartCrawlDto, userId?: string, apiKeyId?: string): Promise { - const startUrl = new URL(dto.startUrl); - const domain = startUrl.hostname; - - const config = dto.config || {}; - - const newJob: NewCrawlJob = { - startUrl: dto.startUrl, - domain, - maxDepth: config.maxDepth ?? this.defaultMaxDepth, - maxPages: config.maxPages ?? this.defaultMaxPages, - rateLimit: config.rateLimit ?? this.defaultRateLimit, - respectRobots: config.respectRobots ?? true, - includePatterns: config.includePatterns, - excludePatterns: config.excludePatterns, - selectors: config.selectors, - output: config.output, - webhookUrl: dto.webhookUrl, - userId, - apiKeyId, - status: 'pending', - progress: { - discovered: 0, - crawled: 0, - failed: 0, - queued: 0, - }, - }; - - // Insert job into database - const [createdJob] = await this.db - .insert(crawlJobs) - .values(newJob) - .returning(); - - this.logger.log(`Created crawl job ${createdJob.id} for ${domain}`); - - // Add to queue - try { - await this.queueService.addCrawlJob(createdJob); - - // Update status to running - const [updatedJob] = await this.db - .update(crawlJobs) - .set({ - status: 'running', - startedAt: new Date(), - updatedAt: new Date(), - }) - .where(eq(crawlJobs.id, createdJob.id)) - .returning(); - - this.metricsService.setActiveJobs('running', 1); - - return toCrawlJobResponse(updatedJob); - } catch (error) { - // Update status to failed - await this.db - .update(crawlJobs) - .set({ - status: 'failed', - error: error instanceof Error ? error.message : 'Failed to queue job', - updatedAt: new Date(), - }) - .where(eq(crawlJobs.id, createdJob.id)); - - throw error; - } - } - - async getJob(jobId: string): Promise { - const [job] = await this.db - .select() - .from(crawlJobs) - .where(eq(crawlJobs.id, jobId)) - .limit(1); - - if (!job) { - throw new NotFoundException(`Crawl job ${jobId} not found`); - } - - return toCrawlJobResponse(job); - } - - async getJobResults( - jobId: string, - page = 1, - limit = 50, - ): Promise> { - // Verify job exists - const [job] = await this.db - .select() - .from(crawlJobs) - .where(eq(crawlJobs.id, jobId)) - .limit(1); - - if (!job) { - throw new NotFoundException(`Crawl job ${jobId} not found`); - } - - // Get total count - const [{ total }] = await this.db - .select({ total: count() }) - .from(crawlResults) - .where(eq(crawlResults.jobId, jobId)); - - // Get paginated results - const offset = (page - 1) * limit; - const results = await this.db - .select() - .from(crawlResults) - .where(eq(crawlResults.jobId, jobId)) - .orderBy(desc(crawlResults.createdAt)) - .limit(limit) - .offset(offset); - - return { - results: results.map(toCrawlResultResponse), - pagination: { - page, - limit, - total: Number(total), - totalPages: Math.ceil(Number(total) / limit), - }, - }; - } - - async cancelJob(jobId: string): Promise { - const [job] = await this.db - .select() - .from(crawlJobs) - .where(eq(crawlJobs.id, jobId)) - .limit(1); - - if (!job) { - throw new NotFoundException(`Crawl job ${jobId} not found`); - } - - if (['completed', 'failed', 'cancelled'].includes(job.status)) { - throw new BadRequestException(`Cannot cancel job with status: ${job.status}`); - } - - // Cancel queue jobs - await this.queueService.cancelJob(jobId); - - // Update status - const [updatedJob] = await this.db - .update(crawlJobs) - .set({ - status: 'cancelled', - completedAt: new Date(), - updatedAt: new Date(), - }) - .where(eq(crawlJobs.id, jobId)) - .returning(); - - this.logger.log(`Cancelled crawl job ${jobId}`); - - return toCrawlJobResponse(updatedJob); - } - - async pauseJob(jobId: string): Promise { - const [job] = await this.db - .select() - .from(crawlJobs) - .where(eq(crawlJobs.id, jobId)) - .limit(1); - - if (!job) { - throw new NotFoundException(`Crawl job ${jobId} not found`); - } - - if (job.status !== 'running') { - throw new BadRequestException(`Cannot pause job with status: ${job.status}`); - } - - // Pause queue jobs - await this.queueService.pauseJob(jobId); - - // Update status - const [updatedJob] = await this.db - .update(crawlJobs) - .set({ - status: 'paused', - updatedAt: new Date(), - }) - .where(eq(crawlJobs.id, jobId)) - .returning(); - - this.logger.log(`Paused crawl job ${jobId}`); - - return toCrawlJobResponse(updatedJob); - } - - async resumeJob(jobId: string): Promise { - const [job] = await this.db - .select() - .from(crawlJobs) - .where(eq(crawlJobs.id, jobId)) - .limit(1); - - if (!job) { - throw new NotFoundException(`Crawl job ${jobId} not found`); - } - - if (job.status !== 'paused') { - throw new BadRequestException(`Cannot resume job with status: ${job.status}`); - } - - // Re-add to queue - await this.queueService.addCrawlJob(job); - - // Update status - const [updatedJob] = await this.db - .update(crawlJobs) - .set({ - status: 'running', - updatedAt: new Date(), - }) - .where(eq(crawlJobs.id, jobId)) - .returning(); - - this.logger.log(`Resumed crawl job ${jobId}`); - - return toCrawlJobResponse(updatedJob); - } - - async listJobs( - page = 1, - limit = 20, - status?: string, - userId?: string, - ): Promise> { - let query = this.db.select().from(crawlJobs); - - // Build conditions - const conditions = []; - if (status) { - conditions.push(eq(crawlJobs.status, status)); - } - if (userId) { - conditions.push(eq(crawlJobs.userId, userId)); - } - - if (conditions.length > 0) { - query = query.where(conditions[0]); - for (let i = 1; i < conditions.length; i++) { - query = query.where(conditions[i]); - } - } - - // Get total count - let countQuery = this.db.select({ total: count() }).from(crawlJobs); - if (conditions.length > 0) { - countQuery = countQuery.where(conditions[0]); - for (let i = 1; i < conditions.length; i++) { - countQuery = countQuery.where(conditions[i]); - } - } - const [{ total }] = await countQuery; - - // Get paginated results - const offset = (page - 1) * limit; - const jobs = await query - .orderBy(desc(crawlJobs.createdAt)) - .limit(limit) - .offset(offset); - - return { - results: jobs.map(toCrawlJobResponse), - pagination: { - page, - limit, - total: Number(total), - totalPages: Math.ceil(Number(total) / limit), - }, - }; - } - - async deleteJob(jobId: string): Promise { - const [job] = await this.db - .select() - .from(crawlJobs) - .where(eq(crawlJobs.id, jobId)) - .limit(1); - - if (!job) { - throw new NotFoundException(`Crawl job ${jobId} not found`); - } - - // Cancel if running - if (['running', 'pending', 'paused'].includes(job.status)) { - await this.queueService.cancelJob(jobId); - } - - // Delete job (results will be cascade deleted) - await this.db - .delete(crawlJobs) - .where(eq(crawlJobs.id, jobId)); - - this.logger.log(`Deleted crawl job ${jobId}`); - } -} diff --git a/services/mana-crawler/src/crawler/dto/crawl-response.dto.ts b/services/mana-crawler/src/crawler/dto/crawl-response.dto.ts deleted file mode 100644 index 7f5be7f9b..000000000 --- a/services/mana-crawler/src/crawler/dto/crawl-response.dto.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { CrawlJob, CrawlResult, CrawlProgress } from '../../db/schema'; - -export interface CrawlJobResponse { - jobId: string; - status: string; - startUrl: string; - domain: string; - config: { - maxDepth: number; - maxPages: number; - rateLimit: number; - respectRobots: boolean; - includePatterns?: string[]; - excludePatterns?: string[]; - }; - progress: CrawlProgress; - startedAt?: string; - completedAt?: string; - createdAt: string; - error?: string; -} - -export interface CrawlResultResponse { - id: string; - url: string; - parentUrl?: string; - depth: number; - title?: string; - content?: string; - markdown?: string; - links?: string[]; - metadata?: Record; - statusCode?: number; - error?: string; - fetchDurationMs?: number; - parseDurationMs?: number; - contentLength?: number; - createdAt: string; -} - -export interface PaginatedResults { - results: T[]; - pagination: { - page: number; - limit: number; - total: number; - totalPages: number; - }; -} - -export function toCrawlJobResponse(job: CrawlJob): CrawlJobResponse { - return { - jobId: job.id, - status: job.status, - startUrl: job.startUrl, - domain: job.domain, - config: { - maxDepth: job.maxDepth, - maxPages: job.maxPages, - rateLimit: job.rateLimit, - respectRobots: job.respectRobots, - includePatterns: job.includePatterns ?? undefined, - excludePatterns: job.excludePatterns ?? undefined, - }, - progress: job.progress || { discovered: 0, crawled: 0, failed: 0, queued: 0 }, - startedAt: job.startedAt?.toISOString(), - completedAt: job.completedAt?.toISOString(), - createdAt: job.createdAt.toISOString(), - error: job.error ?? undefined, - }; -} - -export function toCrawlResultResponse(result: CrawlResult): CrawlResultResponse { - return { - id: result.id, - url: result.url, - parentUrl: result.parentUrl ?? undefined, - depth: result.depth, - title: result.title ?? undefined, - content: result.content ?? undefined, - markdown: result.markdown ?? undefined, - links: result.links ?? undefined, - metadata: result.metadata ?? undefined, - statusCode: result.statusCode ?? undefined, - error: result.error ?? undefined, - fetchDurationMs: result.fetchDurationMs ?? undefined, - parseDurationMs: result.parseDurationMs ?? undefined, - contentLength: result.contentLength ?? undefined, - createdAt: result.createdAt.toISOString(), - }; -} diff --git a/services/mana-crawler/src/crawler/dto/index.ts b/services/mana-crawler/src/crawler/dto/index.ts deleted file mode 100644 index 1c80e3659..000000000 --- a/services/mana-crawler/src/crawler/dto/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './start-crawl.dto'; -export * from './crawl-response.dto'; diff --git a/services/mana-crawler/src/crawler/dto/start-crawl.dto.ts b/services/mana-crawler/src/crawler/dto/start-crawl.dto.ts deleted file mode 100644 index 2b2b9bbe2..000000000 --- a/services/mana-crawler/src/crawler/dto/start-crawl.dto.ts +++ /dev/null @@ -1,96 +0,0 @@ -import { - IsString, - IsUrl, - IsOptional, - IsInt, - IsBoolean, - IsArray, - Min, - Max, - ValidateNested, - IsObject, - IsEnum, -} from 'class-validator'; -import { Type } from 'class-transformer'; - -export class CrawlSelectorsDto { - @IsOptional() - @IsString() - title?: string; - - @IsOptional() - @IsString() - content?: string; - - @IsOptional() - @IsString() - links?: string; - - @IsOptional() - @IsObject() - custom?: Record; -} - -export class CrawlOutputDto { - @IsOptional() - @IsEnum(['text', 'html', 'markdown']) - format?: 'text' | 'html' | 'markdown'; -} - -export class CrawlConfigDto { - @IsOptional() - @IsInt() - @Min(1) - @Max(10) - maxDepth?: number; - - @IsOptional() - @IsInt() - @Min(1) - @Max(10000) - maxPages?: number; - - @IsOptional() - @IsBoolean() - respectRobots?: boolean; - - @IsOptional() - @IsInt() - @Min(1) - @Max(100) - rateLimit?: number; - - @IsOptional() - @IsArray() - @IsString({ each: true }) - includePatterns?: string[]; - - @IsOptional() - @IsArray() - @IsString({ each: true }) - excludePatterns?: string[]; - - @IsOptional() - @ValidateNested() - @Type(() => CrawlSelectorsDto) - selectors?: CrawlSelectorsDto; - - @IsOptional() - @ValidateNested() - @Type(() => CrawlOutputDto) - output?: CrawlOutputDto; -} - -export class StartCrawlDto { - @IsUrl() - startUrl: string; - - @IsOptional() - @ValidateNested() - @Type(() => CrawlConfigDto) - config?: CrawlConfigDto; - - @IsOptional() - @IsUrl() - webhookUrl?: string; -} diff --git a/services/mana-crawler/src/db/connection.ts b/services/mana-crawler/src/db/connection.ts deleted file mode 100644 index e84b0fa08..000000000 --- a/services/mana-crawler/src/db/connection.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { drizzle } from 'drizzle-orm/postgres-js'; -import postgres from 'postgres'; -import * as schema from './schema'; - -let connection: ReturnType | null = null; -let db: ReturnType | null = null; - -export function getConnection(databaseUrl: string) { - if (!connection) { - connection = postgres(databaseUrl, { - max: 10, - idle_timeout: 20, - connect_timeout: 10, - }); - } - return connection; -} - -export function getDb(databaseUrl: string) { - if (!db) { - const conn = getConnection(databaseUrl); - db = drizzle(conn, { schema }); - } - return db; -} - -export async function closeConnection() { - if (connection) { - await connection.end(); - connection = null; - db = null; - } -} diff --git a/services/mana-crawler/src/db/database.module.ts b/services/mana-crawler/src/db/database.module.ts deleted file mode 100644 index ffc4b6b06..000000000 --- a/services/mana-crawler/src/db/database.module.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { Global, Module } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { getDb } from './connection'; - -export const DATABASE_CONNECTION = 'DATABASE_CONNECTION'; - -@Global() -@Module({ - providers: [ - { - provide: DATABASE_CONNECTION, - useFactory: (configService: ConfigService) => { - const databaseUrl = configService.get('database.url'); - if (!databaseUrl) { - throw new Error('DATABASE_URL is not configured'); - } - return getDb(databaseUrl); - }, - inject: [ConfigService], - }, - ], - exports: [DATABASE_CONNECTION], -}) -export class DatabaseModule {} diff --git a/services/mana-crawler/src/db/schema/crawl-jobs.schema.ts b/services/mana-crawler/src/db/schema/crawl-jobs.schema.ts deleted file mode 100644 index 61fc47e21..000000000 --- a/services/mana-crawler/src/db/schema/crawl-jobs.schema.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { - pgSchema, - uuid, - text, - integer, - boolean, - timestamp, - jsonb, - index, -} from 'drizzle-orm/pg-core'; - -export const crawlerSchema = pgSchema('crawler'); - -export interface CrawlSelectors { - title?: string; - content?: string; - links?: string; - custom?: Record; -} - -export interface CrawlProgress { - discovered: number; - crawled: number; - failed: number; - queued: number; -} - -export interface CrawlOutput { - format?: 'text' | 'html' | 'markdown'; -} - -export const crawlJobs = crawlerSchema.table( - 'crawl_jobs', - { - id: uuid('id').defaultRandom().primaryKey(), - - // Job config - startUrl: text('start_url').notNull(), - domain: text('domain').notNull(), - maxDepth: integer('max_depth').notNull().default(3), - maxPages: integer('max_pages').notNull().default(100), - rateLimit: integer('rate_limit').notNull().default(2), // requests/second - - // URL patterns - includePatterns: jsonb('include_patterns').$type(), - excludePatterns: jsonb('exclude_patterns').$type(), - - // Selectors for extraction - selectors: jsonb('selectors').$type(), - - // Output options - output: jsonb('output').$type(), - - // Robots.txt - respectRobots: boolean('respect_robots').notNull().default(true), - - // Status - status: text('status').notNull().default('pending'), // pending, running, paused, completed, failed, cancelled - progress: jsonb('progress').$type().default({ - discovered: 0, - crawled: 0, - failed: 0, - queued: 0, - }), - error: text('error'), - - // Metadata - userId: text('user_id'), - apiKeyId: uuid('api_key_id'), - webhookUrl: text('webhook_url'), - - // Bull queue job ID - bullJobId: text('bull_job_id'), - - // Timestamps - startedAt: timestamp('started_at', { withTimezone: true }), - completedAt: timestamp('completed_at', { withTimezone: true }), - createdAt: timestamp('created_at', { withTimezone: true }).defaultNow().notNull(), - updatedAt: timestamp('updated_at', { withTimezone: true }).defaultNow().notNull(), - }, - (table) => ({ - statusIdx: index('crawl_jobs_status_idx').on(table.status), - userIdIdx: index('crawl_jobs_user_id_idx').on(table.userId), - domainIdx: index('crawl_jobs_domain_idx').on(table.domain), - }), -); - -export type CrawlJob = typeof crawlJobs.$inferSelect; -export type NewCrawlJob = typeof crawlJobs.$inferInsert; diff --git a/services/mana-crawler/src/db/schema/crawl-results.schema.ts b/services/mana-crawler/src/db/schema/crawl-results.schema.ts deleted file mode 100644 index f6d966f12..000000000 --- a/services/mana-crawler/src/db/schema/crawl-results.schema.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { - uuid, - text, - integer, - timestamp, - jsonb, - index, -} from 'drizzle-orm/pg-core'; -import { crawlerSchema, crawlJobs } from './crawl-jobs.schema'; - -export const crawlResults = crawlerSchema.table( - 'crawl_results', - { - id: uuid('id').defaultRandom().primaryKey(), - jobId: uuid('job_id') - .references(() => crawlJobs.id, { onDelete: 'cascade' }) - .notNull(), - - // Page data - url: text('url').notNull(), - parentUrl: text('parent_url'), - depth: integer('depth').notNull(), - - // Extracted content - title: text('title'), - content: text('content'), - markdown: text('markdown'), - html: text('html'), - metadata: jsonb('metadata').$type>(), - - // Links found - links: jsonb('links').$type(), - - // Status - statusCode: integer('status_code'), - error: text('error'), - - // Performance - fetchDurationMs: integer('fetch_duration_ms'), - parseDurationMs: integer('parse_duration_ms'), - contentLength: integer('content_length'), - - createdAt: timestamp('created_at', { withTimezone: true }).defaultNow().notNull(), - }, - (table) => ({ - jobIdIdx: index('crawl_results_job_id_idx').on(table.jobId), - urlIdx: index('crawl_results_url_idx').on(table.url), - jobUrlUnique: index('crawl_results_job_url_idx').on(table.jobId, table.url), - }), -); - -export type CrawlResult = typeof crawlResults.$inferSelect; -export type NewCrawlResult = typeof crawlResults.$inferInsert; diff --git a/services/mana-crawler/src/db/schema/index.ts b/services/mana-crawler/src/db/schema/index.ts deleted file mode 100644 index 3ea2611b3..000000000 --- a/services/mana-crawler/src/db/schema/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './crawl-jobs.schema'; -export * from './crawl-results.schema'; diff --git a/services/mana-crawler/src/health/health.controller.ts b/services/mana-crawler/src/health/health.controller.ts deleted file mode 100644 index c762edc96..000000000 --- a/services/mana-crawler/src/health/health.controller.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { Controller, Get, Inject } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { CacheService } from '../cache/cache.service'; -import { DATABASE_CONNECTION } from '../db/database.module'; -import { sql } from 'drizzle-orm'; - -@Controller() -export class HealthController { - constructor( - private readonly configService: ConfigService, - private readonly cacheService: CacheService, - @Inject(DATABASE_CONNECTION) private readonly db: any, - ) {} - - @Get('/health') - async health() { - // Check Redis - const redisStatus = await this.cacheService.healthCheck(); - - // Check Database - let dbStatus = { status: 'unknown', latency: 0 }; - try { - const start = Date.now(); - await this.db.execute(sql`SELECT 1`); - dbStatus = { status: 'ok', latency: Date.now() - start }; - } catch { - dbStatus = { status: 'error', latency: 0 }; - } - - const overallStatus = - redisStatus.status === 'ok' && dbStatus.status === 'ok' - ? 'ok' - : redisStatus.status === 'error' && dbStatus.status === 'error' - ? 'error' - : 'degraded'; - - return { - status: overallStatus, - service: 'mana-crawler', - version: '1.0.0', - timestamp: new Date().toISOString(), - components: { - redis: redisStatus, - database: dbStatus, - }, - }; - } -} diff --git a/services/mana-crawler/src/health/health.module.ts b/services/mana-crawler/src/health/health.module.ts deleted file mode 100644 index 2fd33ce4a..000000000 --- a/services/mana-crawler/src/health/health.module.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { Module } from '@nestjs/common'; -import { HealthController } from './health.controller'; -import { CacheModule } from '../cache/cache.module'; - -@Module({ - imports: [CacheModule], - controllers: [HealthController], -}) -export class HealthModule {} diff --git a/services/mana-crawler/src/main.ts b/services/mana-crawler/src/main.ts deleted file mode 100644 index 139eb99c0..000000000 --- a/services/mana-crawler/src/main.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { NestFactory } from '@nestjs/core'; -import { ValidationPipe, Logger } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { AppModule } from './app.module'; -import { HttpExceptionFilter } from './common/filters/http-exception.filter'; - -async function bootstrap() { - const logger = new Logger('Bootstrap'); - - const app = await NestFactory.create(AppModule); - - const configService = app.get(ConfigService); - const port = configService.get('port', 3023); - - // Global prefix - app.setGlobalPrefix('api/v1'); - - // CORS - app.enableCors({ - origin: configService.get('cors.origins', ['http://localhost:*']), - credentials: true, - }); - - // Global pipes - app.useGlobalPipes( - new ValidationPipe({ - whitelist: true, - transform: true, - forbidNonWhitelisted: true, - }), - ); - - // Global filters - app.useGlobalFilters(new HttpExceptionFilter()); - - await app.listen(port); - logger.log(`Mana Crawler Service running on port ${port}`); - logger.log(`Health check: http://localhost:${port}/health`); - logger.log(`Metrics: http://localhost:${port}/metrics`); - logger.log(`Queue Dashboard: http://localhost:${port}/queue/dashboard`); -} - -bootstrap(); diff --git a/services/mana-crawler/src/metrics/metrics.controller.ts b/services/mana-crawler/src/metrics/metrics.controller.ts deleted file mode 100644 index 5679d2eab..000000000 --- a/services/mana-crawler/src/metrics/metrics.controller.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { Controller, Get, Header } from '@nestjs/common'; -import { MetricsService } from './metrics.service'; - -@Controller() -export class MetricsController { - constructor(private readonly metricsService: MetricsService) {} - - @Get('/metrics') - @Header('Content-Type', 'text/plain') - async getMetrics(): Promise { - return this.metricsService.getMetrics(); - } -} diff --git a/services/mana-crawler/src/metrics/metrics.module.ts b/services/mana-crawler/src/metrics/metrics.module.ts deleted file mode 100644 index 5c3b5c90f..000000000 --- a/services/mana-crawler/src/metrics/metrics.module.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { Module } from '@nestjs/common'; -import { MetricsController } from './metrics.controller'; -import { MetricsService } from './metrics.service'; - -@Module({ - controllers: [MetricsController], - providers: [MetricsService], - exports: [MetricsService], -}) -export class MetricsModule {} diff --git a/services/mana-crawler/src/metrics/metrics.service.ts b/services/mana-crawler/src/metrics/metrics.service.ts deleted file mode 100644 index 16e6c20a0..000000000 --- a/services/mana-crawler/src/metrics/metrics.service.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { Injectable, OnModuleInit } from '@nestjs/common'; -import * as client from 'prom-client'; - -@Injectable() -export class MetricsService implements OnModuleInit { - private requestCounter: client.Counter; - private requestDuration: client.Histogram; - private cacheHitCounter: client.Counter; - private cacheMissCounter: client.Counter; - private crawlJobsGauge: client.Gauge; - private pagesProcessedCounter: client.Counter; - private crawlErrorsCounter: client.Counter; - - onModuleInit() { - // Clear default metrics and register new ones - client.register.clear(); - client.collectDefaultMetrics({ prefix: 'mana_crawler_' }); - - this.requestCounter = new client.Counter({ - name: 'mana_crawler_requests_total', - help: 'Total number of requests', - labelNames: ['method', 'endpoint', 'status'], - }); - - this.requestDuration = new client.Histogram({ - name: 'mana_crawler_request_duration_ms', - help: 'Request duration in milliseconds', - labelNames: ['method', 'endpoint'], - buckets: [10, 50, 100, 200, 500, 1000, 2000, 5000], - }); - - this.cacheHitCounter = new client.Counter({ - name: 'mana_crawler_cache_hits_total', - help: 'Total number of cache hits', - }); - - this.cacheMissCounter = new client.Counter({ - name: 'mana_crawler_cache_misses_total', - help: 'Total number of cache misses', - }); - - this.crawlJobsGauge = new client.Gauge({ - name: 'mana_crawler_jobs_active', - help: 'Number of active crawl jobs', - labelNames: ['status'], - }); - - this.pagesProcessedCounter = new client.Counter({ - name: 'mana_crawler_pages_processed_total', - help: 'Total number of pages processed', - labelNames: ['status'], - }); - - this.crawlErrorsCounter = new client.Counter({ - name: 'mana_crawler_errors_total', - help: 'Total number of crawl errors', - labelNames: ['type'], - }); - } - - recordRequest(endpoint: string, status: number, durationMs: number, method = 'GET') { - this.requestCounter.inc({ method, endpoint, status: String(status) }); - this.requestDuration.observe({ method, endpoint }, durationMs); - } - - recordCacheHit() { - this.cacheHitCounter.inc(); - } - - recordCacheMiss() { - this.cacheMissCounter.inc(); - } - - setActiveJobs(status: string, count: number) { - this.crawlJobsGauge.set({ status }, count); - } - - recordPageProcessed(status: 'success' | 'error') { - this.pagesProcessedCounter.inc({ status }); - } - - recordCrawlError(type: string) { - this.crawlErrorsCounter.inc({ type }); - } - - async getMetrics(): Promise { - return client.register.metrics(); - } - - getContentType(): string { - return client.register.contentType; - } -} diff --git a/services/mana-crawler/src/parser/parser.module.ts b/services/mana-crawler/src/parser/parser.module.ts deleted file mode 100644 index cbcb7c2c0..000000000 --- a/services/mana-crawler/src/parser/parser.module.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { Module } from '@nestjs/common'; -import { ParserService } from './parser.service'; - -@Module({ - providers: [ParserService], - exports: [ParserService], -}) -export class ParserModule {} diff --git a/services/mana-crawler/src/parser/parser.service.ts b/services/mana-crawler/src/parser/parser.service.ts deleted file mode 100644 index 6451d794d..000000000 --- a/services/mana-crawler/src/parser/parser.service.ts +++ /dev/null @@ -1,245 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import * as cheerio from 'cheerio'; -import TurndownService from 'turndown'; -import { CrawlSelectors } from '../db/schema'; - -export interface ParsedPage { - title: string; - content: string; - markdown?: string; - html?: string; - links: string[]; - metadata: Record; -} - -export interface ParseOptions { - selectors?: CrawlSelectors; - includeMarkdown?: boolean; - includeHtml?: boolean; - baseUrl: string; -} - -@Injectable() -export class ParserService { - private readonly logger = new Logger(ParserService.name); - private readonly turndown: TurndownService; - - constructor(private readonly configService: ConfigService) { - this.turndown = new TurndownService({ - headingStyle: 'atx', - codeBlockStyle: 'fenced', - bulletListMarker: '-', - }); - - // Custom rules for better Markdown output - this.turndown.addRule('codeBlocks', { - filter: ['pre'], - replacement: (content: string) => `\n\`\`\`\n${content}\n\`\`\`\n`, - }); - - this.turndown.addRule('inlineCode', { - filter: ['code'], - replacement: (content: string) => `\`${content}\``, - }); - - // Remove script and style elements - this.turndown.remove(['script', 'style', 'noscript']); - } - - parse(html: string, options: ParseOptions): ParsedPage { - const $ = cheerio.load(html); - const { selectors, includeMarkdown, includeHtml, baseUrl } = options; - - // Remove unwanted elements - $('script, style, noscript, iframe, svg').remove(); - - // Extract title - const title = this.extractTitle($, selectors?.title); - - // Extract main content - const contentHtml = this.extractContent($, selectors?.content); - const content = this.cleanText(contentHtml); - - // Extract links - const links = this.extractLinks($, baseUrl, selectors?.links); - - // Extract metadata - const metadata = this.extractMetadata($); - - // Extract custom selectors - if (selectors?.custom) { - for (const [key, selector] of Object.entries(selectors.custom)) { - try { - metadata[key] = $(selector).text().trim() || $(selector).attr('content'); - } catch { - this.logger.warn(`Failed to extract custom selector: ${key}`); - } - } - } - - const result: ParsedPage = { - title, - content, - links, - metadata, - }; - - if (includeMarkdown && contentHtml) { - result.markdown = this.turndown.turndown(contentHtml); - } - - if (includeHtml) { - result.html = contentHtml; - } - - return result; - } - - private extractTitle($: cheerio.CheerioAPI, selector?: string): string { - if (selector) { - const customTitle = $(selector).text().trim(); - if (customTitle) return customTitle; - } - - // Try common title patterns - const h1 = $('h1').first().text().trim(); - if (h1) return h1; - - const title = $('title').text().trim(); - if (title) return title; - - const ogTitle = $('meta[property="og:title"]').attr('content'); - if (ogTitle) return ogTitle; - - return ''; - } - - private extractContent($: cheerio.CheerioAPI, selector?: string): string { - if (selector) { - const customContent = $(selector).html(); - if (customContent) return customContent; - } - - // Try common content patterns - const contentSelectors = [ - 'article', - 'main', - '[role="main"]', - '.main-content', - '.content', - '.post-content', - '.article-content', - '.entry-content', - '#content', - '#main', - ]; - - for (const sel of contentSelectors) { - const content = $(sel).html(); - if (content && content.length > 100) { - return content; - } - } - - // Fallback to body - return $('body').html() || ''; - } - - private extractLinks( - $: cheerio.CheerioAPI, - baseUrl: string, - selector?: string, - ): string[] { - const links = new Set(); - const baseUrlObj = new URL(baseUrl); - - const linkSelector = selector || 'a[href]'; - - $(linkSelector).each((_, element) => { - const href = $(element).attr('href'); - if (!href) return; - - try { - // Skip non-http links - if ( - href.startsWith('javascript:') || - href.startsWith('mailto:') || - href.startsWith('tel:') || - href.startsWith('#') - ) { - return; - } - - // Resolve relative URLs - const absoluteUrl = new URL(href, baseUrl); - - // Only include same-origin links (or all if needed) - if (absoluteUrl.origin === baseUrlObj.origin) { - // Remove hash and normalize - absoluteUrl.hash = ''; - links.add(absoluteUrl.href); - } - } catch { - // Invalid URL, skip - } - }); - - return Array.from(links); - } - - private extractMetadata($: cheerio.CheerioAPI): Record { - const metadata: Record = {}; - - // OpenGraph metadata - $('meta[property^="og:"]').each((_, element) => { - const property = $(element).attr('property')?.replace('og:', ''); - const content = $(element).attr('content'); - if (property && content) { - metadata[`og_${property}`] = content; - } - }); - - // Standard meta tags - const description = $('meta[name="description"]').attr('content'); - if (description) metadata.description = description; - - const keywords = $('meta[name="keywords"]').attr('content'); - if (keywords) metadata.keywords = keywords; - - const author = $('meta[name="author"]').attr('content'); - if (author) metadata.author = author; - - const canonical = $('link[rel="canonical"]').attr('href'); - if (canonical) metadata.canonical = canonical; - - // Schema.org JSON-LD - $('script[type="application/ld+json"]').each((_, element) => { - try { - const json = JSON.parse($(element).html() || ''); - if (!metadata.jsonLd) { - metadata.jsonLd = []; - } - (metadata.jsonLd as unknown[]).push(json); - } catch { - // Invalid JSON, skip - } - }); - - return metadata; - } - - private cleanText(html: string): string { - return html - .replace(/)<[^<]*)*<\/script>/gi, '') - .replace(/)<[^<]*)*<\/style>/gi, '') - .replace(/<[^>]+>/g, ' ') - .replace(/ /g, ' ') - .replace(/&/g, '&') - .replace(/</g, '<') - .replace(/>/g, '>') - .replace(/"/g, '"') - .replace(/\s+/g, ' ') - .trim(); - } -} diff --git a/services/mana-crawler/src/queue/constants.ts b/services/mana-crawler/src/queue/constants.ts deleted file mode 100644 index 0c85a9cd6..000000000 --- a/services/mana-crawler/src/queue/constants.ts +++ /dev/null @@ -1 +0,0 @@ -export const CRAWL_QUEUE = 'crawl'; diff --git a/services/mana-crawler/src/queue/processor.module.ts b/services/mana-crawler/src/queue/processor.module.ts deleted file mode 100644 index 0e321c10f..000000000 --- a/services/mana-crawler/src/queue/processor.module.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { Module, forwardRef } from '@nestjs/common'; -import { CrawlProcessor } from './processors/crawl.processor'; -import { ParserModule } from '../parser/parser.module'; -import { RobotsModule } from '../robots/robots.module'; -import { CacheModule } from '../cache/cache.module'; -import { MetricsModule } from '../metrics/metrics.module'; -import { QueueModule } from './queue.module'; -import { CRAWL_QUEUE } from './constants'; - -@Module({ - imports: [ - forwardRef(() => QueueModule), - ParserModule, - RobotsModule, - CacheModule, - MetricsModule, - ], - providers: [CrawlProcessor], -}) -export class ProcessorModule {} diff --git a/services/mana-crawler/src/queue/processors/crawl.processor.ts b/services/mana-crawler/src/queue/processors/crawl.processor.ts deleted file mode 100644 index d354d23f5..000000000 --- a/services/mana-crawler/src/queue/processors/crawl.processor.ts +++ /dev/null @@ -1,350 +0,0 @@ -import { Processor, WorkerHost, OnWorkerEvent, InjectQueue } from '@nestjs/bullmq'; -import { Logger, Inject } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { Job, Queue } from 'bullmq'; -import { eq, sql } from 'drizzle-orm'; -import { CRAWL_QUEUE } from '../constants'; -import { CrawlPageJob } from '../queue.service'; -import { ParserService } from '../../parser/parser.service'; -import { RobotsService } from '../../robots/robots.service'; -import { CacheService } from '../../cache/cache.service'; -import { MetricsService } from '../../metrics/metrics.service'; -import { DATABASE_CONNECTION } from '../../db/database.module'; -import { crawlJobs, crawlResults, type NewCrawlResult } from '../../db/schema'; - -@Processor(CRAWL_QUEUE, { - concurrency: 5, -}) -export class CrawlProcessor extends WorkerHost { - private readonly logger = new Logger(CrawlProcessor.name); - private readonly userAgent: string; - private readonly timeout: number; - private readonly processedUrls = new Map>(); - - constructor( - private readonly configService: ConfigService, - @InjectQueue(CRAWL_QUEUE) private readonly crawlQueue: Queue, - private readonly parserService: ParserService, - private readonly robotsService: RobotsService, - private readonly cacheService: CacheService, - private readonly metricsService: MetricsService, - @Inject(DATABASE_CONNECTION) private readonly db: any, - ) { - super(); - this.userAgent = this.configService.get( - 'crawler.userAgent', - 'ManaCoreCrawler/1.0', - ); - this.timeout = this.configService.get('crawler.timeout', 30000); - } - - async process(job: Job): Promise { - const { jobId, url, parentUrl, depth, config } = job.data; - const startTime = Date.now(); - - this.logger.debug(`Processing ${url} (depth: ${depth}, job: ${jobId})`); - - try { - // Check if job is still active - const [crawlJob] = await this.db - .select() - .from(crawlJobs) - .where(eq(crawlJobs.id, jobId)) - .limit(1); - - if (!crawlJob || ['cancelled', 'paused', 'completed', 'failed'].includes(crawlJob.status)) { - this.logger.debug(`Job ${jobId} is no longer active, skipping`); - return; - } - - // Initialize URL tracking for this job - if (!this.processedUrls.has(jobId)) { - this.processedUrls.set(jobId, new Set()); - } - const processed = this.processedUrls.get(jobId)!; - - // Check if URL already processed - if (processed.has(url)) { - this.logger.debug(`URL already processed: ${url}`); - return; - } - processed.add(url); - - // Check max pages limit - if (crawlJob.progress.crawled >= config.maxPages) { - this.logger.debug(`Max pages reached for job ${jobId}`); - await this.completeJob(jobId); - return; - } - - // Check robots.txt - if (config.respectRobots) { - const robotsCheck = await this.robotsService.checkUrlWithRobots(url); - if (!robotsCheck.allowed) { - this.logger.debug(`URL blocked by robots.txt: ${url}`); - await this.updateProgress(jobId, { failed: 1 }); - return; - } - } - - // Check URL patterns - if (!this.matchesPatterns(url, config.includePatterns, config.excludePatterns)) { - this.logger.debug(`URL doesn't match patterns: ${url}`); - return; - } - - // Fetch the page - const fetchStart = Date.now(); - const response = await fetch(url, { - headers: { - 'User-Agent': this.userAgent, - Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Language': 'en-US,en;q=0.5', - }, - signal: AbortSignal.timeout(this.timeout), - }); - const fetchDuration = Date.now() - fetchStart; - - if (!response.ok) { - await this.saveResult(jobId, { - url, - parentUrl, - depth, - statusCode: response.status, - error: `HTTP ${response.status}`, - fetchDurationMs: fetchDuration, - }); - await this.updateProgress(jobId, { crawled: 1, failed: 1 }); - this.metricsService.recordPageProcessed('error'); - return; - } - - const contentType = response.headers.get('content-type') || ''; - if (!contentType.includes('text/html')) { - this.logger.debug(`Skipping non-HTML content: ${url}`); - return; - } - - const html = await response.text(); - const contentLength = html.length; - - // Parse the page - const parseStart = Date.now(); - const parsed = this.parserService.parse(html, { - selectors: config.selectors, - includeMarkdown: config.output?.format === 'markdown', - includeHtml: config.output?.format === 'html', - baseUrl: url, - }); - const parseDuration = Date.now() - parseStart; - - // Save result - await this.saveResult(jobId, { - url, - parentUrl, - depth, - title: parsed.title, - content: parsed.content, - markdown: parsed.markdown, - html: parsed.html, - links: parsed.links, - metadata: parsed.metadata, - statusCode: response.status, - fetchDurationMs: fetchDuration, - parseDurationMs: parseDuration, - contentLength, - }); - - await this.updateProgress(jobId, { - crawled: 1, - discovered: parsed.links.length, - }); - this.metricsService.recordPageProcessed('success'); - - // Queue discovered links - if (depth < config.maxDepth && crawlJob.progress.crawled < config.maxPages) { - for (const link of parsed.links) { - if (!processed.has(link)) { - try { - const urlHash = Buffer.from(link).toString('base64').slice(0, 32); - await this.crawlQueue.add( - 'crawl-page', - { - jobId, - url: link, - parentUrl: url, - depth: depth + 1, - config, - } as CrawlPageJob, - { - jobId: `${jobId}-${urlHash}`, - delay: Math.floor(1000 / config.rateLimit), - }, - ); - } catch (error) { - // Job might already exist, ignore - } - } - } - } - - this.logger.debug( - `Processed ${url} in ${Date.now() - startTime}ms (${parsed.links.length} links found)`, - ); - } catch (error) { - this.logger.error(`Error processing ${url}: ${error}`); - this.metricsService.recordCrawlError('fetch_error'); - - await this.saveResult(jobId, { - url, - parentUrl, - depth, - error: error instanceof Error ? error.message : 'Unknown error', - fetchDurationMs: Date.now() - startTime, - }); - - await this.updateProgress(jobId, { crawled: 1, failed: 1 }); - throw error; // Let BullMQ handle retries - } - } - - @OnWorkerEvent('completed') - async onCompleted(job: Job) { - // Check if this was the last job for this crawl - const { jobId } = job.data; - const counts = await this.crawlQueue.getJobCounts( - 'waiting', - 'active', - 'delayed', - ); - - if ((counts.waiting ?? 0) === 0 && (counts.active ?? 0) === 0 && (counts.delayed ?? 0) === 0) { - // Check if there are jobs for this specific crawl - const [crawlJob] = await this.db - .select() - .from(crawlJobs) - .where(eq(crawlJobs.id, jobId)) - .limit(1); - - if (crawlJob && crawlJob.status === 'running') { - await this.completeJob(jobId); - } - } - } - - @OnWorkerEvent('failed') - onFailed(job: Job, error: Error) { - this.logger.error(`Job ${job.id} failed: ${error.message}`); - this.metricsService.recordCrawlError('job_failed'); - } - - private matchesPatterns( - url: string, - includePatterns?: string[], - excludePatterns?: string[], - ): boolean { - // Check exclude patterns first - if (excludePatterns?.length) { - for (const pattern of excludePatterns) { - if (this.matchPattern(url, pattern)) { - return false; - } - } - } - - // If no include patterns, allow all - if (!includePatterns?.length) { - return true; - } - - // Check include patterns - for (const pattern of includePatterns) { - if (this.matchPattern(url, pattern)) { - return true; - } - } - - return false; - } - - private matchPattern(url: string, pattern: string): boolean { - // Simple glob pattern matching - const regex = pattern - .replace(/[.*+?^${}()|[\]\\]/g, '\\$&') - .replace(/\\\*/g, '.*') - .replace(/\\\?/g, '.'); - return new RegExp(regex).test(url); - } - - private async saveResult( - jobId: string, - result: Omit, - ): Promise { - try { - await this.db.insert(crawlResults).values({ - jobId, - ...result, - }); - } catch (error) { - this.logger.error(`Failed to save result for ${result.url}: ${error}`); - } - } - - private async updateProgress( - jobId: string, - delta: { discovered?: number; crawled?: number; failed?: number; queued?: number }, - ): Promise { - try { - const updates: string[] = []; - if (delta.discovered) { - updates.push(`'discovered', COALESCE((progress->>'discovered')::int, 0) + ${delta.discovered}`); - } - if (delta.crawled) { - updates.push(`'crawled', COALESCE((progress->>'crawled')::int, 0) + ${delta.crawled}`); - } - if (delta.failed) { - updates.push(`'failed', COALESCE((progress->>'failed')::int, 0) + ${delta.failed}`); - } - if (delta.queued) { - updates.push(`'queued', COALESCE((progress->>'queued')::int, 0) + ${delta.queued}`); - } - - if (updates.length > 0) { - await this.db - .update(crawlJobs) - .set({ - progress: sql`jsonb_build_object( - 'discovered', COALESCE((progress->>'discovered')::int, 0) + ${delta.discovered || 0}, - 'crawled', COALESCE((progress->>'crawled')::int, 0) + ${delta.crawled || 0}, - 'failed', COALESCE((progress->>'failed')::int, 0) + ${delta.failed || 0}, - 'queued', COALESCE((progress->>'queued')::int, 0) + ${delta.queued || 0} - )`, - updatedAt: new Date(), - }) - .where(eq(crawlJobs.id, jobId)); - } - } catch (error) { - this.logger.error(`Failed to update progress for job ${jobId}: ${error}`); - } - } - - private async completeJob(jobId: string): Promise { - try { - await this.db - .update(crawlJobs) - .set({ - status: 'completed', - completedAt: new Date(), - updatedAt: new Date(), - }) - .where(eq(crawlJobs.id, jobId)); - - // Clean up URL tracking - this.processedUrls.delete(jobId); - - this.logger.log(`Crawl job ${jobId} completed`); - } catch (error) { - this.logger.error(`Failed to complete job ${jobId}: ${error}`); - } - } -} diff --git a/services/mana-crawler/src/queue/queue.module.ts b/services/mana-crawler/src/queue/queue.module.ts deleted file mode 100644 index 2ed9eee42..000000000 --- a/services/mana-crawler/src/queue/queue.module.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { Module } from '@nestjs/common'; -import { BullModule } from '@nestjs/bullmq'; -import { BullBoardModule } from '@bull-board/nestjs'; -import { BullMQAdapter } from '@bull-board/api/bullMQAdapter'; -import { QueueService } from './queue.service'; -import { CRAWL_QUEUE } from './constants'; - -// Re-export for convenience -export { CRAWL_QUEUE } from './constants'; - -@Module({ - imports: [ - BullModule.registerQueue({ - name: CRAWL_QUEUE, - defaultJobOptions: { - attempts: 3, - backoff: { - type: 'exponential', - delay: 1000, - }, - removeOnComplete: 100, - removeOnFail: 1000, - }, - }), - BullBoardModule.forFeature({ - name: CRAWL_QUEUE, - adapter: BullMQAdapter, - }), - ], - providers: [QueueService], - exports: [QueueService, BullModule], -}) -export class QueueModule {} diff --git a/services/mana-crawler/src/queue/queue.service.ts b/services/mana-crawler/src/queue/queue.service.ts deleted file mode 100644 index 4def2d1c2..000000000 --- a/services/mana-crawler/src/queue/queue.service.ts +++ /dev/null @@ -1,150 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { InjectQueue } from '@nestjs/bullmq'; -import { Queue, Job } from 'bullmq'; -import { CRAWL_QUEUE } from './constants'; -import { CrawlJob } from '../db/schema'; - -export interface CrawlPageJob { - jobId: string; - url: string; - parentUrl?: string; - depth: number; - config: { - maxDepth: number; - maxPages: number; - rateLimit: number; - respectRobots: boolean; - includePatterns?: string[]; - excludePatterns?: string[]; - selectors?: { - title?: string; - content?: string; - links?: string; - custom?: Record; - }; - output?: { - format?: 'text' | 'html' | 'markdown'; - }; - }; -} - -@Injectable() -export class QueueService { - private readonly logger = new Logger(QueueService.name); - - constructor(@InjectQueue(CRAWL_QUEUE) private readonly crawlQueue: Queue) {} - - async addCrawlJob(crawlJob: CrawlJob): Promise> { - const jobData: CrawlPageJob = { - jobId: crawlJob.id, - url: crawlJob.startUrl, - depth: 0, - config: { - maxDepth: crawlJob.maxDepth, - maxPages: crawlJob.maxPages, - rateLimit: crawlJob.rateLimit, - respectRobots: crawlJob.respectRobots, - includePatterns: crawlJob.includePatterns ?? undefined, - excludePatterns: crawlJob.excludePatterns ?? undefined, - selectors: crawlJob.selectors ?? undefined, - output: crawlJob.output ?? undefined, - }, - }; - - const job = await this.crawlQueue.add('crawl-page', jobData, { - jobId: `${crawlJob.id}-start`, - }); - - this.logger.log(`Added crawl job ${crawlJob.id} to queue`); - return job; - } - - async addPageToQueue( - jobId: string, - url: string, - parentUrl: string, - depth: number, - config: CrawlPageJob['config'], - ): Promise> { - const jobData: CrawlPageJob = { - jobId, - url, - parentUrl, - depth, - config, - }; - - // Use URL hash as job ID to prevent duplicates - const urlHash = Buffer.from(url).toString('base64').slice(0, 32); - const job = await this.crawlQueue.add('crawl-page', jobData, { - jobId: `${jobId}-${urlHash}`, - delay: Math.floor(1000 / config.rateLimit), // Rate limiting delay - }); - - return job; - } - - async getJobCounts(): Promise<{ - waiting: number; - active: number; - completed: number; - failed: number; - delayed: number; - }> { - const counts = await this.crawlQueue.getJobCounts( - 'waiting', - 'active', - 'completed', - 'failed', - 'delayed', - ); - return { - waiting: counts.waiting ?? 0, - active: counts.active ?? 0, - completed: counts.completed ?? 0, - failed: counts.failed ?? 0, - delayed: counts.delayed ?? 0, - }; - } - - async pauseJob(jobId: string): Promise { - // Get all jobs for this crawl job - const jobs = await this.crawlQueue.getJobs(['waiting', 'delayed']); - for (const job of jobs) { - if (job.data.jobId === jobId) { - await job.remove(); - } - } - this.logger.log(`Paused crawl job ${jobId}`); - } - - async cancelJob(jobId: string): Promise { - // Remove all jobs for this crawl job - const jobs = await this.crawlQueue.getJobs([ - 'waiting', - 'delayed', - 'active', - ]); - for (const job of jobs) { - if (job.data.jobId === jobId) { - await job.remove(); - } - } - this.logger.log(`Cancelled crawl job ${jobId}`); - } - - async getQueueStats(): Promise<{ - name: string; - counts: Record; - isPaused: boolean; - }> { - const counts = await this.getJobCounts(); - const isPaused = await this.crawlQueue.isPaused(); - - return { - name: CRAWL_QUEUE, - counts, - isPaused, - }; - } -} diff --git a/services/mana-crawler/src/robots/robots.module.ts b/services/mana-crawler/src/robots/robots.module.ts deleted file mode 100644 index d466223c1..000000000 --- a/services/mana-crawler/src/robots/robots.module.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { Module } from '@nestjs/common'; -import { RobotsService } from './robots.service'; -import { CacheModule } from '../cache/cache.module'; - -@Module({ - imports: [CacheModule], - providers: [RobotsService], - exports: [RobotsService], -}) -export class RobotsModule {} diff --git a/services/mana-crawler/src/robots/robots.service.ts b/services/mana-crawler/src/robots/robots.service.ts deleted file mode 100644 index 392096eb9..000000000 --- a/services/mana-crawler/src/robots/robots.service.ts +++ /dev/null @@ -1,143 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import robotsParser from 'robots-parser'; -import { CacheService } from '../cache/cache.service'; - -interface RobotsData { - allowed: boolean; - crawlDelay?: number; - sitemaps: string[]; -} - -@Injectable() -export class RobotsService { - private readonly logger = new Logger(RobotsService.name); - private readonly userAgent: string; - private readonly cacheTtl: number; - - constructor( - private readonly configService: ConfigService, - private readonly cacheService: CacheService, - ) { - this.userAgent = this.configService.get( - 'crawler.userAgent', - 'ManaCoreCrawler/1.0', - ); - this.cacheTtl = this.configService.get('cache.robotsTtl', 86400); - } - - async isAllowed(url: string): Promise { - try { - const urlObj = new URL(url); - const robotsUrl = `${urlObj.protocol}//${urlObj.host}/robots.txt`; - const cacheKey = `robots:${urlObj.host}`; - - // Check cache first - const cached = await this.cacheService.get(cacheKey); - if (cached !== null) { - return this.checkUrl(cached, url); - } - - // Fetch robots.txt - const robotsData = await this.fetchRobots(robotsUrl, urlObj.host); - - // Cache the result - await this.cacheService.set(cacheKey, robotsData, this.cacheTtl); - - return this.checkUrl(robotsData, url); - } catch (error) { - this.logger.warn(`Error checking robots.txt for ${url}: ${error}`); - // If we can't check, allow by default - return true; - } - } - - async getCrawlDelay(domain: string): Promise { - const cacheKey = `robots:${domain}`; - const cached = await this.cacheService.get(cacheKey); - return cached?.crawlDelay; - } - - async getSitemaps(domain: string): Promise { - const cacheKey = `robots:${domain}`; - const cached = await this.cacheService.get(cacheKey); - return cached?.sitemaps || []; - } - - private async fetchRobots(robotsUrl: string, host: string): Promise { - try { - const response = await fetch(robotsUrl, { - headers: { - 'User-Agent': this.userAgent, - }, - signal: AbortSignal.timeout(5000), - }); - - if (!response.ok) { - // No robots.txt or error - allow all - this.logger.debug(`No robots.txt found for ${host} (${response.status})`); - return { allowed: true, sitemaps: [] }; - } - - const robotsTxt = await response.text(); - const robots = robotsParser(robotsUrl, robotsTxt); - - // Get crawl delay - const crawlDelay = robots.getCrawlDelay(this.userAgent); - - // Get sitemaps - const sitemaps = robots.getSitemaps(); - - return { - allowed: true, // Will be checked per-URL - crawlDelay: crawlDelay ? Number(crawlDelay) : undefined, - sitemaps, - }; - } catch (error) { - this.logger.warn(`Failed to fetch robots.txt from ${robotsUrl}: ${error}`); - return { allowed: true, sitemaps: [] }; - } - } - - private checkUrl(robotsData: RobotsData, url: string): boolean { - // For now, we're caching a simplified version - // In production, you might want to cache the full robots.txt - // and parse it each time for more accurate checking - return robotsData.allowed; - } - - async checkUrlWithRobots(url: string): Promise<{ - allowed: boolean; - crawlDelay?: number; - }> { - try { - const urlObj = new URL(url); - const robotsUrl = `${urlObj.protocol}//${urlObj.host}/robots.txt`; - - const response = await fetch(robotsUrl, { - headers: { - 'User-Agent': this.userAgent, - }, - signal: AbortSignal.timeout(5000), - }); - - if (!response.ok) { - return { allowed: true }; - } - - const robotsTxt = await response.text(); - const robots = robotsParser(robotsUrl, robotsTxt); - - const allowed = robots.isAllowed(url, this.userAgent) ?? true; - const crawlDelay = robots.getCrawlDelay(this.userAgent); - - return { - allowed, - crawlDelay: crawlDelay ? Number(crawlDelay) : undefined, - }; - } catch (error) { - this.logger.warn(`Error checking robots.txt for ${url}: ${error}`); - return { allowed: true }; - } - } -} diff --git a/services/mana-crawler/tsconfig.json b/services/mana-crawler/tsconfig.json deleted file mode 100644 index f02c2417e..000000000 --- a/services/mana-crawler/tsconfig.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "compilerOptions": { - "module": "commonjs", - "declaration": true, - "removeComments": true, - "emitDecoratorMetadata": true, - "experimentalDecorators": true, - "allowSyntheticDefaultImports": true, - "target": "ES2022", - "sourceMap": true, - "outDir": "./dist", - "baseUrl": "./", - "incremental": true, - "skipLibCheck": true, - "strictNullChecks": true, - "noImplicitAny": true, - "strictBindCallApply": true, - "forceConsistentCasingInFileNames": true, - "noFallthroughCasesInSwitch": true, - "esModuleInterop": true, - "resolveJsonModule": true - }, - "include": ["src/**/*"], - "exclude": ["node_modules", "dist"] -} diff --git a/services/mana-notify/CLAUDE.md b/services/mana-notify/CLAUDE.md deleted file mode 100644 index abf8c9147..000000000 --- a/services/mana-notify/CLAUDE.md +++ /dev/null @@ -1,390 +0,0 @@ -# Mana Notify Service - -Central notification microservice for email, push, Matrix, and webhook notifications across all ManaCore apps. - -## Overview - -- **Port**: 3040 -- **Technology**: NestJS + BullMQ + Drizzle ORM + PostgreSQL + Redis -- **Purpose**: Unified notification API with template support and delivery tracking - -## Architecture - -``` -┌─────────────────────────────────────────────────────────────┐ -│ Consumer Apps │ -│ Auth │ Calendar │ Chat │ Picture │ Zitare │ ... │ -└─────────────────────────┬───────────────────────────────────┘ - ▼ -┌─────────────────────────────────────────────────────────────┐ -│ mana-notify (Port 3040) │ -│ │ -│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ -│ │ Notification│ │ Template │ │ Preferences │ │ -│ │ API │ │ Engine │ │ Manager │ │ -│ └──────┬──────┘ └──────┬──────┘ └─────────────┘ │ -│ │ │ │ -│ ▼ ▼ │ -│ ┌──────────────────────────────────────────────┐ │ -│ │ BullMQ Job Queues │ │ -│ │ Email │ Push │ Matrix │ Webhook │ │ -│ └──────────────────────────────────────────────┘ │ -│ │ │ │ │ │ -│ ▼ ▼ ▼ ▼ │ -│ ┌─────────┐ ┌─────────┐ ┌─────────┐ ┌─────────┐ │ -│ │ Brevo │ │ Expo │ │ Matrix │ │ HTTP │ │ -│ │ SMTP │ │ Push │ │ API │ │ Client │ │ -│ └─────────┘ └─────────┘ └─────────┘ └─────────┘ │ -└─────────────────────────────────────────────────────────────┘ -``` - -## Quick Start - -### Development - -```bash -# 1. Start PostgreSQL and Redis (from monorepo root) -pnpm docker:up - -# 2. Install dependencies -pnpm install - -# 3. Push database schema -pnpm db:push - -# 4. Start in development mode -pnpm dev -``` - -### Production - -```bash -pnpm build -pnpm start -``` - -## API Endpoints - -### Notifications (Service-Key Auth: X-Service-Key header) - -| Method | Endpoint | Description | -|--------|----------|-------------| -| POST | `/api/v1/notifications/send` | Send notification immediately | -| POST | `/api/v1/notifications/schedule` | Schedule notification for later | -| POST | `/api/v1/notifications/batch` | Send multiple notifications | -| GET | `/api/v1/notifications/:id` | Get notification status | -| DELETE | `/api/v1/notifications/:id` | Cancel pending notification | - -### Templates (Service-Key Auth) - -| Method | Endpoint | Description | -|--------|----------|-------------| -| GET | `/api/v1/templates` | List all templates | -| GET | `/api/v1/templates/:slug` | Get template by slug | -| POST | `/api/v1/templates` | Create custom template | -| PUT | `/api/v1/templates/:slug` | Update template | -| DELETE | `/api/v1/templates/:slug` | Delete custom template | -| POST | `/api/v1/templates/:slug/preview` | Preview rendered template | -| POST | `/api/v1/templates/preview` | Preview custom template | - -### Devices (JWT Auth: Bearer token) - -| Method | Endpoint | Description | -|--------|----------|-------------| -| POST | `/api/v1/devices/register` | Register push device | -| GET | `/api/v1/devices` | List user's devices | -| DELETE | `/api/v1/devices/:id` | Unregister device | - -### Preferences (JWT Auth) - -| Method | Endpoint | Description | -|--------|----------|-------------| -| GET | `/api/v1/preferences` | Get user preferences | -| PUT | `/api/v1/preferences` | Update preferences | - -### System - -| Method | Endpoint | Description | -|--------|----------|-------------| -| GET | `/health` | Health check | -| GET | `/metrics` | Prometheus metrics | - -## Usage Examples - -### Send Email - -```bash -curl -X POST http://localhost:3040/api/v1/notifications/send \ - -H "Content-Type: application/json" \ - -H "X-Service-Key: your-service-key" \ - -d '{ - "channel": "email", - "appId": "auth", - "template": "auth-password-reset", - "recipient": "user@example.com", - "data": { - "resetUrl": "https://mana.how/reset?token=xxx", - "userName": "Max" - } - }' -``` - -### Send Push Notification - -```bash -curl -X POST http://localhost:3040/api/v1/notifications/send \ - -H "Content-Type: application/json" \ - -H "X-Service-Key: your-service-key" \ - -d '{ - "channel": "push", - "appId": "calendar", - "userId": "user-uuid", - "subject": "Erinnerung", - "body": "Meeting in 15 Minuten", - "data": { "eventId": "event-uuid" } - }' -``` - -### Register Device (User JWT) - -```bash -curl -X POST http://localhost:3040/api/v1/devices/register \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer $USER_JWT" \ - -d '{ - "pushToken": "ExponentPushToken[xxx]", - "platform": "ios", - "deviceName": "iPhone 15" - }' -``` - -### Schedule Notification - -```bash -curl -X POST http://localhost:3040/api/v1/notifications/schedule \ - -H "Content-Type: application/json" \ - -H "X-Service-Key: your-service-key" \ - -d '{ - "channel": "email", - "appId": "calendar", - "template": "calendar-reminder", - "recipient": "user@example.com", - "data": { - "eventTitle": "Team Meeting", - "eventTime": "14:00 Uhr", - "eventUrl": "https://calendar.mana.how/event/xxx" - }, - "scheduledFor": "2024-12-20T13:45:00Z" - }' -``` - -## Environment Variables - -| Variable | Default | Description | -|----------|---------|-------------| -| `PORT` | 3040 | API port | -| `DATABASE_URL` | - | PostgreSQL connection URL | -| `REDIS_HOST` | localhost | Redis host for BullMQ | -| `REDIS_PORT` | 6379 | Redis port | -| `SERVICE_KEY` | dev-service-key | Internal service authentication key | -| `MANA_CORE_AUTH_URL` | http://localhost:3001 | Auth service URL for JWT validation | -| `SMTP_HOST` | smtp-relay.brevo.com | SMTP server host | -| `SMTP_PORT` | 587 | SMTP server port | -| `SMTP_USER` | - | SMTP username | -| `SMTP_PASSWORD` | - | SMTP password | -| `SMTP_FROM` | ManaCore | Default sender address | -| `EXPO_ACCESS_TOKEN` | - | Expo push notification access token | -| `MATRIX_HOMESERVER_URL` | - | Matrix homeserver URL | -| `MATRIX_ACCESS_TOKEN` | - | Matrix bot access token | -| `RATE_LIMIT_EMAIL_PER_MINUTE` | 10 | Email rate limit | -| `RATE_LIMIT_PUSH_PER_MINUTE` | 100 | Push notification rate limit | - -## Development Commands - -```bash -# Install dependencies -pnpm install - -# Start development server -pnpm dev - -# Build for production -pnpm build - -# Start production server -pnpm start - -# Type checking -pnpm type-check - -# Linting -pnpm lint - -# Database commands -pnpm db:push # Push schema to database -pnpm db:generate # Generate migrations -pnpm db:migrate # Run migrations -pnpm db:studio # Open Drizzle Studio -``` - -## Database Schema - -The service uses its own schema (`notify`) in the shared ManaCore database: - -- `notify.notifications` - Notification records with status tracking -- `notify.templates` - Email/push templates with Handlebars support -- `notify.devices` - Registered push notification devices -- `notify.preferences` - User notification preferences -- `notify.delivery_logs` - Delivery attempt logs - -## Default Templates - -| Slug | Channel | Purpose | -|------|---------|---------| -| `auth-password-reset` | email | Password reset email | -| `auth-verification` | email | Email verification | -| `auth-welcome` | email | Welcome email | -| `calendar-reminder` | email | Calendar event reminder | - -## Notification Channels - -### Email (Brevo SMTP) -- Uses Nodemailer with Brevo SMTP relay -- Supports HTML and plain text -- Template rendering with Handlebars - -### Push (Expo) -- Uses Expo Server SDK -- Supports iOS, Android, and web -- Batch sending with automatic chunking - -### Matrix -- Direct Matrix API integration -- Supports formatted (HTML) messages -- For bot notifications - -### Webhook -- HTTP POST/PUT to external URLs -- Configurable headers and timeout -- Retry with exponential backoff - -## Integration with Other Services - -### Usage from NestJS Backend - -```typescript -// Direct HTTP call -const response = await fetch('http://mana-notify:3040/api/v1/notifications/send', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'X-Service-Key': process.env.MANA_NOTIFY_SERVICE_KEY, - }, - body: JSON.stringify({ - channel: 'email', - appId: 'calendar', - template: 'calendar-reminder', - recipient: user.email, - data: { eventTitle, eventTime }, - }), -}); -``` - -### Using the Client SDK - -```typescript -import { NotifyClient } from '@manacore/notify-client'; - -const notify = new NotifyClient({ - serviceUrl: 'http://localhost:3040', - serviceKey: process.env.MANA_NOTIFY_SERVICE_KEY, - appId: 'calendar', -}); - -// Send email -await notify.sendEmail({ - to: 'user@example.com', - template: 'calendar-reminder', - data: { eventTitle: 'Meeting', eventTime: '14:00' }, -}); - -// Send push to user -await notify.sendPush({ - userId: 'user-uuid', - title: 'Reminder', - body: 'Meeting in 15 minutes', - data: { eventId: 'xxx' }, -}); -``` - -## Project Structure - -``` -services/mana-notify/ -├── src/ -│ ├── main.ts # Application entry point -│ ├── app.module.ts # Root module -│ ├── config/ -│ │ └── configuration.ts # App configuration -│ ├── db/ -│ │ ├── schema/ # Drizzle schemas -│ │ ├── database.module.ts # Database provider -│ │ └── connection.ts # DB connection -│ ├── common/ -│ │ ├── filters/ # Exception filters -│ │ └── guards/ # Auth guards -│ ├── queue/ -│ │ ├── queue.module.ts # BullMQ setup -│ │ └── processors/ # Channel processors -│ ├── channels/ -│ │ ├── email/ # Nodemailer service -│ │ ├── push/ # Expo push service -│ │ ├── matrix/ # Matrix API service -│ │ └── webhook/ # HTTP webhook service -│ ├── notifications/ # Core notification API -│ ├── templates/ # Template engine -│ │ └── defaults/ # Default HBS templates -│ ├── devices/ # Device registration -│ ├── preferences/ # User preferences -│ ├── health/ # Health check -│ └── metrics/ # Prometheus metrics -├── drizzle.config.ts -├── package.json -├── tsconfig.json -├── Dockerfile -└── CLAUDE.md -``` - -## Troubleshooting - -### Email not sending - -1. Check SMTP credentials in environment -2. Verify SMTP host/port settings -3. Check logs for error messages - -### Push notifications failing - -1. Verify Expo push tokens are valid -2. Check Expo access token is set -3. Ensure devices are registered - -### Redis connection issues - -```bash -# Check Redis -docker exec mana-notify-redis-dev redis-cli ping - -# Check queue status -curl http://localhost:3040/health -``` - -## Metrics - -Available at `/metrics` in Prometheus format: - -- `mana_notify_notifications_sent_total` - Total notifications sent -- `mana_notify_notifications_failed_total` - Total failed notifications -- `mana_notify_emails_sent_total` - Emails sent by template -- `mana_notify_push_sent_total` - Push notifications by platform -- `mana_notify_notification_latency_seconds` - Processing latency diff --git a/services/mana-notify/Dockerfile b/services/mana-notify/Dockerfile deleted file mode 100644 index 0e50b0827..000000000 --- a/services/mana-notify/Dockerfile +++ /dev/null @@ -1,49 +0,0 @@ -# syntax=docker/dockerfile:1 -FROM node:20-alpine AS base - -# Install pnpm -RUN corepack enable && corepack prepare pnpm@9.15.0 --activate - -WORKDIR /app - -# Build stage -FROM base AS builder - -# Copy workspace files -COPY pnpm-workspace.yaml package.json pnpm-lock.yaml ./ -COPY services/mana-notify/package.json ./services/mana-notify/ -COPY packages/shared-drizzle-config/package.json ./packages/shared-drizzle-config/ - -# Install dependencies -RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install --frozen-lockfile --filter @manacore/mana-notify - -# Copy source code -COPY packages/shared-drizzle-config ./packages/shared-drizzle-config -COPY services/mana-notify ./services/mana-notify - -# Build -WORKDIR /app/services/mana-notify -RUN pnpm build - -# Production stage -FROM base AS production - -WORKDIR /app - -# Copy built files -COPY --from=builder /app/services/mana-notify/dist ./dist -COPY --from=builder /app/services/mana-notify/package.json ./ - -# Copy template files -COPY --from=builder /app/services/mana-notify/src/templates/defaults ./dist/templates/defaults - -# Install production dependencies only -COPY --from=builder /app/node_modules ./node_modules - -# Set environment -ENV NODE_ENV=production -ENV PORT=3040 - -EXPOSE 3040 - -CMD ["node", "dist/main.js"] diff --git a/services/mana-notify/docker-compose.dev.yml b/services/mana-notify/docker-compose.dev.yml deleted file mode 100644 index f3d680e05..000000000 --- a/services/mana-notify/docker-compose.dev.yml +++ /dev/null @@ -1,23 +0,0 @@ -version: '3.8' - -# Development compose for mana-notify -# Provides Redis for BullMQ queue -# PostgreSQL should be running from root docker-compose - -services: - redis: - image: redis:7-alpine - container_name: mana-notify-redis-dev - ports: - - '6379:6379' - volumes: - - redis-data:/data - command: redis-server --appendonly yes - healthcheck: - test: ['CMD', 'redis-cli', 'ping'] - interval: 10s - timeout: 5s - retries: 5 - -volumes: - redis-data: diff --git a/services/mana-notify/drizzle.config.ts b/services/mana-notify/drizzle.config.ts deleted file mode 100644 index 888fc6300..000000000 --- a/services/mana-notify/drizzle.config.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { defineConfig } from 'drizzle-kit'; - -export default defineConfig({ - schema: './src/db/schema/index.ts', - out: './drizzle', - dialect: 'postgresql', - dbCredentials: { - url: process.env.DATABASE_URL || 'postgresql://manacore:devpassword@localhost:5432/manacore', - }, - verbose: true, - strict: true, -}); diff --git a/services/mana-notify/nest-cli.json b/services/mana-notify/nest-cli.json deleted file mode 100644 index 27d87b55d..000000000 --- a/services/mana-notify/nest-cli.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "$schema": "https://json.schemastore.org/nest-cli", - "collection": "@nestjs/schematics", - "sourceRoot": "src", - "compilerOptions": { - "deleteOutDir": true, - "assets": ["templates/defaults/**/*.hbs"], - "watchAssets": true - } -} diff --git a/services/mana-notify/package.json b/services/mana-notify/package.json deleted file mode 100644 index ca72a6056..000000000 --- a/services/mana-notify/package.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "name": "@manacore/mana-notify", - "version": "0.0.1", - "description": "Central notification microservice for email, push, matrix, and webhook notifications", - "private": true, - "license": "UNLICENSED", - "scripts": { - "build": "nest build", - "dev": "nest start --watch", - "start": "node dist/main", - "start:dev": "nest start --watch", - "start:debug": "nest start --debug --watch", - "start:prod": "node dist/main", - "lint": "eslint \"{src,test}/**/*.ts\" --fix", - "type-check": "tsc --noEmit", - "test": "jest", - "test:watch": "jest --watch", - "test:cov": "jest --coverage", - "db:push": "drizzle-kit push", - "db:generate": "drizzle-kit generate", - "db:migrate": "drizzle-kit migrate", - "db:studio": "drizzle-kit studio" - }, - "dependencies": { - "@nestjs/bullmq": "^10.2.3", - "@nestjs/common": "^10.4.17", - "@nestjs/config": "^3.3.0", - "@nestjs/core": "^10.4.17", - "@nestjs/platform-express": "^10.4.17", - "@nestjs/schedule": "^4.1.2", - "bullmq": "^5.34.8", - "class-transformer": "^0.5.1", - "class-validator": "^0.14.1", - "drizzle-orm": "^0.38.4", - "expo-server-sdk": "^3.10.0", - "handlebars": "^4.7.8", - "ioredis": "^5.4.2", - "jose": "^5.9.6", - "nodemailer": "^7.0.3", - "postgres": "^3.4.5", - "prom-client": "^15.1.3", - "reflect-metadata": "^0.2.2", - "rxjs": "^7.8.1" - }, - "devDependencies": { - "@manacore/shared-drizzle-config": "workspace:*", - "@nestjs/cli": "^10.4.9", - "@nestjs/schematics": "^10.2.3", - "@nestjs/testing": "^10.4.17", - "@types/express": "^5.0.0", - "@types/jest": "^29.5.14", - "@types/node": "^22.10.5", - "@types/nodemailer": "^6.4.17", - "drizzle-kit": "^0.30.4", - "jest": "^29.7.0", - "ts-jest": "^29.2.5", - "ts-node": "^10.9.2", - "typescript": "^5.7.2" - } -} diff --git a/services/mana-notify/src/app.module.ts b/services/mana-notify/src/app.module.ts deleted file mode 100644 index f0ce7d04c..000000000 --- a/services/mana-notify/src/app.module.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { Module } from '@nestjs/common'; -import { ConfigModule, ConfigService } from '@nestjs/config'; -import { BullModule } from '@nestjs/bullmq'; -import { ScheduleModule } from '@nestjs/schedule'; -import configuration from './config/configuration'; -import { DatabaseModule } from './db/database.module'; -import { HealthModule } from './health/health.module'; -import { MetricsModule } from './metrics/metrics.module'; -import { QueueModule } from './queue/queue.module'; -import { ChannelsModule } from './channels/channels.module'; -import { NotificationsModule } from './notifications/notifications.module'; -import { TemplatesModule } from './templates/templates.module'; -import { DevicesModule } from './devices/devices.module'; -import { PreferencesModule } from './preferences/preferences.module'; - -@Module({ - imports: [ - ConfigModule.forRoot({ - isGlobal: true, - load: [configuration], - }), - BullModule.forRootAsync({ - imports: [ConfigModule], - useFactory: (configService: ConfigService) => ({ - connection: { - host: configService.get('redis.host', 'localhost'), - port: configService.get('redis.port', 6379), - }, - }), - inject: [ConfigService], - }), - ScheduleModule.forRoot(), - DatabaseModule, - HealthModule, - MetricsModule, - QueueModule, - ChannelsModule, - NotificationsModule, - TemplatesModule, - DevicesModule, - PreferencesModule, - ], -}) -export class AppModule {} diff --git a/services/mana-notify/src/channels/channels.module.ts b/services/mana-notify/src/channels/channels.module.ts deleted file mode 100644 index 5609c2e5c..000000000 --- a/services/mana-notify/src/channels/channels.module.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { Module } from '@nestjs/common'; -import { EmailService } from './email/email.service'; -import { PushService } from './push/push.service'; -import { MatrixService } from './matrix/matrix.service'; -import { WebhookService } from './webhook/webhook.service'; - -@Module({ - providers: [EmailService, PushService, MatrixService, WebhookService], - exports: [EmailService, PushService, MatrixService, WebhookService], -}) -export class ChannelsModule {} diff --git a/services/mana-notify/src/channels/email/email.service.ts b/services/mana-notify/src/channels/email/email.service.ts deleted file mode 100644 index 6cb83be7b..000000000 --- a/services/mana-notify/src/channels/email/email.service.ts +++ /dev/null @@ -1,95 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import * as nodemailer from 'nodemailer'; - -export interface EmailOptions { - to: string; - subject: string; - html: string; - text?: string; - from?: string; - replyTo?: string; -} - -export interface EmailResult { - success: boolean; - messageId?: string; - error?: string; -} - -@Injectable() -export class EmailService { - private readonly logger = new Logger(EmailService.name); - private transporter: nodemailer.Transporter | null = null; - private readonly defaultFrom: string; - - constructor(private readonly configService: ConfigService) { - this.defaultFrom = this.configService.get('smtp.from', 'ManaCore '); - this.initializeTransporter(); - } - - private initializeTransporter(): void { - const host = this.configService.get('smtp.host'); - const port = this.configService.get('smtp.port', 587); - const user = this.configService.get('smtp.user'); - const pass = this.configService.get('smtp.password'); - - if (!user || !pass) { - this.logger.warn('SMTP credentials not configured, emails will be logged only'); - return; - } - - this.transporter = nodemailer.createTransport({ - host, - port, - secure: port === 465, - auth: { - user, - pass, - }, - }); - - this.logger.log(`Email service initialized with SMTP host: ${host}`); - } - - async sendEmail(options: EmailOptions): Promise { - const { to, subject, html, text, from, replyTo } = options; - const sender = from || this.defaultFrom; - - this.logger.debug(`Sending email to: ${to}, subject: ${subject}`); - - if (!this.transporter) { - this.logger.log('[Email] No SMTP configured, logging email content:'); - this.logger.log(` To: ${to}`); - this.logger.log(` Subject: ${subject}`); - this.logger.log(` HTML: ${html.substring(0, 200)}...`); - return { success: false, error: 'SMTP not configured' }; - } - - try { - const result = await this.transporter.sendMail({ - from: sender, - to, - subject, - html, - text: text || this.stripHtml(html), - replyTo, - }); - - this.logger.log(`Email sent successfully, messageId: ${result.messageId}`); - return { success: true, messageId: result.messageId }; - } catch (error) { - const errorMessage = error instanceof Error ? error.message : 'Unknown error'; - this.logger.error(`Failed to send email: ${errorMessage}`); - return { success: false, error: errorMessage }; - } - } - - private stripHtml(html: string): string { - return html.replace(/<[^>]*>/g, ''); - } - - isConfigured(): boolean { - return this.transporter !== null; - } -} diff --git a/services/mana-notify/src/channels/matrix/matrix.service.ts b/services/mana-notify/src/channels/matrix/matrix.service.ts deleted file mode 100644 index e308f667d..000000000 --- a/services/mana-notify/src/channels/matrix/matrix.service.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; - -export interface MatrixMessage { - roomId: string; - body: string; - formattedBody?: string; // HTML formatted - msgtype?: 'text' | 'notice'; -} - -export interface MatrixResult { - success: boolean; - eventId?: string; - error?: string; -} - -@Injectable() -export class MatrixService { - private readonly logger = new Logger(MatrixService.name); - private readonly homeserverUrl: string | null; - private readonly accessToken: string | null; - - constructor(private readonly configService: ConfigService) { - this.homeserverUrl = this.configService.get('matrix.homeserverUrl') || null; - this.accessToken = this.configService.get('matrix.accessToken') || null; - - if (this.isConfigured()) { - this.logger.log(`Matrix service initialized with homeserver: ${this.homeserverUrl}`); - } else { - this.logger.warn('Matrix service not configured'); - } - } - - isConfigured(): boolean { - return !!(this.homeserverUrl && this.accessToken); - } - - async sendMessage(message: MatrixMessage): Promise { - if (!this.isConfigured()) { - return { success: false, error: 'Matrix not configured' }; - } - - const { roomId, body, formattedBody, msgtype = 'text' } = message; - const txnId = `mana_${Date.now()}_${Math.random().toString(36).substring(7)}`; - - const endpoint = `${this.homeserverUrl}/_matrix/client/v3/rooms/${encodeURIComponent(roomId)}/send/m.room.message/${txnId}`; - - const content: Record = { - msgtype: `m.${msgtype}`, - body, - }; - - if (formattedBody) { - content.format = 'org.matrix.custom.html'; - content.formatted_body = formattedBody; - } - - try { - const response = await fetch(endpoint, { - method: 'PUT', - headers: { - Authorization: `Bearer ${this.accessToken}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(content), - }); - - if (!response.ok) { - const errorData = await response.json().catch(() => ({})); - const errorMessage = (errorData as { error?: string }).error || response.statusText; - this.logger.error(`Matrix API error: ${response.status} - ${errorMessage}`); - return { success: false, error: errorMessage }; - } - - const data = (await response.json()) as { event_id?: string }; - this.logger.debug(`Matrix message sent to ${roomId}, eventId: ${data.event_id}`); - - return { success: true, eventId: data.event_id }; - } catch (error) { - const errorMessage = error instanceof Error ? error.message : 'Unknown error'; - this.logger.error(`Failed to send Matrix message: ${errorMessage}`); - return { success: false, error: errorMessage }; - } - } -} diff --git a/services/mana-notify/src/channels/push/push.service.ts b/services/mana-notify/src/channels/push/push.service.ts deleted file mode 100644 index e72070abd..000000000 --- a/services/mana-notify/src/channels/push/push.service.ts +++ /dev/null @@ -1,178 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import Expo, { ExpoPushMessage, ExpoPushTicket, ExpoPushReceipt } from 'expo-server-sdk'; - -export interface PushNotification { - title: string; - body: string; - data?: Record; - sound?: 'default' | null; - badge?: number; - channelId?: string; -} - -export interface PushResult { - success: boolean; - ticketId?: string; - error?: string; -} - -@Injectable() -export class PushService { - private readonly logger = new Logger(PushService.name); - private readonly expo: Expo; - - constructor(private readonly configService: ConfigService) { - const accessToken = this.configService.get('push.expoAccessToken'); - this.expo = new Expo({ - accessToken: accessToken || undefined, - }); - - if (accessToken) { - this.logger.log('Push service initialized with Expo access token'); - } else { - this.logger.warn('Push service initialized without access token (rate limited)'); - } - } - - /** - * Validate if a token is a valid Expo push token - */ - isValidToken(token: string): boolean { - return Expo.isExpoPushToken(token); - } - - /** - * Send push notification to a single token - */ - async sendToToken(token: string, notification: PushNotification): Promise { - if (!this.isValidToken(token)) { - this.logger.warn(`Invalid Expo push token: ${token}`); - return { success: false, error: 'Invalid push token' }; - } - - const message: ExpoPushMessage = { - to: token, - title: notification.title, - body: notification.body, - data: notification.data, - sound: notification.sound ?? 'default', - badge: notification.badge, - channelId: notification.channelId, - }; - - try { - const tickets = await this.expo.sendPushNotificationsAsync([message]); - const ticket = tickets[0]; - - if (ticket.status === 'error') { - this.logger.error(`Push notification error: ${ticket.message}`, ticket.details); - return { success: false, error: ticket.message }; - } - - this.logger.debug( - `Push notification sent successfully to token: ${token.substring(0, 30)}...` - ); - return { - success: true, - ticketId: (ticket as ExpoPushTicket & { id?: string }).id, - }; - } catch (error) { - const errorMessage = error instanceof Error ? error.message : 'Unknown error'; - this.logger.error(`Failed to send push notification: ${errorMessage}`); - return { success: false, error: errorMessage }; - } - } - - /** - * Send push notification to multiple tokens - */ - async sendToTokens( - tokens: string[], - notification: PushNotification - ): Promise> { - const results = new Map(); - const validTokens = tokens.filter((token) => { - const isValid = this.isValidToken(token); - if (!isValid) { - this.logger.warn(`Skipping invalid token: ${token}`); - results.set(token, { success: false, error: 'Invalid token' }); - } - return isValid; - }); - - if (validTokens.length === 0) { - return results; - } - - const messages: ExpoPushMessage[] = validTokens.map((token) => ({ - to: token, - title: notification.title, - body: notification.body, - data: notification.data, - sound: notification.sound ?? 'default', - badge: notification.badge, - channelId: notification.channelId, - })); - - // Chunk messages (Expo has a limit of 100 per batch) - const chunks = this.expo.chunkPushNotifications(messages); - - for (const chunk of chunks) { - try { - const tickets = await this.expo.sendPushNotificationsAsync(chunk); - - tickets.forEach((ticket, index) => { - const token = (chunk[index] as ExpoPushMessage).to as string; - if (ticket.status === 'ok') { - results.set(token, { - success: true, - ticketId: (ticket as ExpoPushTicket & { id?: string }).id, - }); - } else { - this.logger.error(`Push error for ${token}: ${ticket.message}`); - results.set(token, { success: false, error: ticket.message }); - } - }); - } catch (error) { - const errorMessage = error instanceof Error ? error.message : 'Unknown error'; - this.logger.error(`Failed to send push notification batch: ${errorMessage}`); - chunk.forEach((msg) => { - results.set(msg.to as string, { success: false, error: errorMessage }); - }); - } - } - - const successCount = Array.from(results.values()).filter((v) => v.success).length; - this.logger.log(`Push notifications sent: ${successCount}/${tokens.length} successful`); - - return results; - } - - /** - * Check receipts for sent notifications - * Call this after some time to verify delivery - */ - async checkReceipts(ticketIds: string[]): Promise> { - const results = new Map(); - const chunks = this.expo.chunkPushNotificationReceiptIds(ticketIds); - - for (const chunk of chunks) { - try { - const receipts = await this.expo.getPushNotificationReceiptsAsync(chunk); - - for (const [id, receipt] of Object.entries(receipts)) { - results.set(id, receipt); - - if (receipt.status === 'error') { - this.logger.error(`Receipt error for ${id}: ${receipt.message}`, receipt.details); - } - } - } catch (error) { - this.logger.error(`Failed to get push notification receipts: ${error}`); - } - } - - return results; - } -} diff --git a/services/mana-notify/src/channels/webhook/webhook.service.ts b/services/mana-notify/src/channels/webhook/webhook.service.ts deleted file mode 100644 index a3e1ae3fc..000000000 --- a/services/mana-notify/src/channels/webhook/webhook.service.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; - -export interface WebhookPayload { - url: string; - method?: 'POST' | 'PUT'; - headers?: Record; - body: Record; - timeout?: number; -} - -export interface WebhookResult { - success: boolean; - statusCode?: number; - response?: unknown; - error?: string; - durationMs?: number; -} - -@Injectable() -export class WebhookService { - private readonly logger = new Logger(WebhookService.name); - private readonly defaultTimeout = 10000; // 10 seconds - - async send(payload: WebhookPayload): Promise { - const { url, method = 'POST', headers = {}, body, timeout = this.defaultTimeout } = payload; - - const startTime = Date.now(); - - this.logger.debug(`Sending webhook to ${url}`); - - try { - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), timeout); - - const response = await fetch(url, { - method, - headers: { - 'Content-Type': 'application/json', - 'User-Agent': 'ManaNotify/1.0', - ...headers, - }, - body: JSON.stringify(body), - signal: controller.signal, - }); - - clearTimeout(timeoutId); - const durationMs = Date.now() - startTime; - - let responseData: unknown; - try { - responseData = await response.json(); - } catch { - responseData = await response.text(); - } - - if (!response.ok) { - this.logger.warn(`Webhook returned ${response.status}: ${url}`); - return { - success: false, - statusCode: response.status, - response: responseData, - error: `HTTP ${response.status}`, - durationMs, - }; - } - - this.logger.debug(`Webhook sent successfully to ${url} in ${durationMs}ms`); - return { - success: true, - statusCode: response.status, - response: responseData, - durationMs, - }; - } catch (error) { - const durationMs = Date.now() - startTime; - const errorMessage = - error instanceof Error - ? error.name === 'AbortError' - ? 'Request timeout' - : error.message - : 'Unknown error'; - - this.logger.error(`Webhook failed: ${errorMessage}`); - return { - success: false, - error: errorMessage, - durationMs, - }; - } - } -} diff --git a/services/mana-notify/src/common/filters/http-exception.filter.ts b/services/mana-notify/src/common/filters/http-exception.filter.ts deleted file mode 100644 index 406ea78ab..000000000 --- a/services/mana-notify/src/common/filters/http-exception.filter.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { - ExceptionFilter, - Catch, - ArgumentsHost, - HttpException, - HttpStatus, - Logger, -} from '@nestjs/common'; -import { Request, Response } from 'express'; - -@Catch() -export class HttpExceptionFilter implements ExceptionFilter { - private readonly logger = new Logger(HttpExceptionFilter.name); - - catch(exception: unknown, host: ArgumentsHost) { - const ctx = host.switchToHttp(); - const response = ctx.getResponse(); - const request = ctx.getRequest(); - - let status = HttpStatus.INTERNAL_SERVER_ERROR; - let message = 'Internal server error'; - - if (exception instanceof HttpException) { - status = exception.getStatus(); - const exceptionResponse = exception.getResponse(); - message = - typeof exceptionResponse === 'string' - ? exceptionResponse - : (exceptionResponse as any).message || exception.message; - } else if (exception instanceof Error) { - message = exception.message; - this.logger.error(`Unhandled error: ${exception.message}`, exception.stack); - } - - response.status(status).json({ - success: false, - error: { - statusCode: status, - message, - timestamp: new Date().toISOString(), - path: request.url, - }, - }); - } -} diff --git a/services/mana-notify/src/common/guards/jwt-auth.guard.ts b/services/mana-notify/src/common/guards/jwt-auth.guard.ts deleted file mode 100644 index 6f5f934c9..000000000 --- a/services/mana-notify/src/common/guards/jwt-auth.guard.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { - Injectable, - CanActivate, - ExecutionContext, - UnauthorizedException, - Logger, -} from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { Request } from 'express'; -import { jwtVerify, createRemoteJWKSet, JWTPayload } from 'jose'; - -export interface AuthenticatedUser { - userId: string; - email: string; - role?: string; - sessionId?: string; -} - -export interface AuthenticatedRequest extends Request { - user: AuthenticatedUser; -} - -/** - * Guard for user authentication via JWT (validated against mana-core-auth JWKS) - */ -@Injectable() -export class JwtAuthGuard implements CanActivate { - private readonly logger = new Logger(JwtAuthGuard.name); - private readonly authUrl: string; - private jwks: ReturnType | null = null; - - constructor(private readonly configService: ConfigService) { - this.authUrl = this.configService.get('auth.manaCoreAuthUrl', 'http://localhost:3001'); - } - - private getJwks() { - if (!this.jwks) { - this.jwks = createRemoteJWKSet(new URL('/api/v1/auth/jwks', this.authUrl)); - } - return this.jwks; - } - - async canActivate(context: ExecutionContext): Promise { - const request = context.switchToHttp().getRequest(); - const authHeader = request.headers.authorization; - - if (!authHeader || !authHeader.startsWith('Bearer ')) { - throw new UnauthorizedException('Missing or invalid authorization header'); - } - - const token = authHeader.substring(7); - - try { - const { payload } = await jwtVerify(token, this.getJwks(), { - issuer: 'manacore', - audience: 'manacore', - }); - - request.user = this.extractUser(payload); - return true; - } catch (error) { - this.logger.warn(`JWT verification failed: ${error}`); - throw new UnauthorizedException('Invalid or expired token'); - } - } - - private extractUser(payload: JWTPayload): AuthenticatedUser { - return { - userId: payload.sub as string, - email: payload.email as string, - role: payload.role as string | undefined, - sessionId: payload.sid as string | undefined, - }; - } -} diff --git a/services/mana-notify/src/common/guards/service-auth.guard.ts b/services/mana-notify/src/common/guards/service-auth.guard.ts deleted file mode 100644 index 0a83cdb58..000000000 --- a/services/mana-notify/src/common/guards/service-auth.guard.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { - Injectable, - CanActivate, - ExecutionContext, - UnauthorizedException, - Logger, -} from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { Request } from 'express'; - -/** - * Guard for internal service-to-service authentication using X-Service-Key header - */ -@Injectable() -export class ServiceAuthGuard implements CanActivate { - private readonly logger = new Logger(ServiceAuthGuard.name); - private readonly serviceKey: string; - - constructor(private readonly configService: ConfigService) { - this.serviceKey = this.configService.get('auth.serviceKey', 'dev-service-key'); - } - - canActivate(context: ExecutionContext): boolean { - const request = context.switchToHttp().getRequest(); - const providedKey = request.headers['x-service-key'] as string; - - if (!providedKey) { - this.logger.warn('Missing X-Service-Key header'); - throw new UnauthorizedException('Missing service key'); - } - - if (providedKey !== this.serviceKey) { - this.logger.warn('Invalid service key provided'); - throw new UnauthorizedException('Invalid service key'); - } - - return true; - } -} diff --git a/services/mana-notify/src/config/configuration.ts b/services/mana-notify/src/config/configuration.ts deleted file mode 100644 index 46b59cc17..000000000 --- a/services/mana-notify/src/config/configuration.ts +++ /dev/null @@ -1,44 +0,0 @@ -export default () => ({ - port: parseInt(process.env.PORT || '3040', 10), - nodeEnv: process.env.NODE_ENV || 'development', - - database: { - url: process.env.DATABASE_URL || 'postgresql://manacore:devpassword@localhost:5432/manacore', - }, - - redis: { - host: process.env.REDIS_HOST || 'localhost', - port: parseInt(process.env.REDIS_PORT || '6379', 10), - }, - - auth: { - serviceKey: process.env.SERVICE_KEY || 'dev-service-key', - manaCoreAuthUrl: process.env.MANA_CORE_AUTH_URL || 'http://localhost:3001', - }, - - smtp: { - host: process.env.SMTP_HOST || 'smtp-relay.brevo.com', - port: parseInt(process.env.SMTP_PORT || '587', 10), - user: process.env.SMTP_USER, - password: process.env.SMTP_PASSWORD, - from: process.env.SMTP_FROM || 'ManaCore ', - }, - - push: { - expoAccessToken: process.env.EXPO_ACCESS_TOKEN, - }, - - matrix: { - homeserverUrl: process.env.MATRIX_HOMESERVER_URL, - accessToken: process.env.MATRIX_ACCESS_TOKEN, - }, - - rateLimits: { - emailPerMinute: parseInt(process.env.RATE_LIMIT_EMAIL_PER_MINUTE || '10', 10), - pushPerMinute: parseInt(process.env.RATE_LIMIT_PUSH_PER_MINUTE || '100', 10), - }, - - cors: { - origins: process.env.CORS_ORIGINS?.split(',') || ['http://localhost:*'], - }, -}); diff --git a/services/mana-notify/src/db/connection.ts b/services/mana-notify/src/db/connection.ts deleted file mode 100644 index e84b0fa08..000000000 --- a/services/mana-notify/src/db/connection.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { drizzle } from 'drizzle-orm/postgres-js'; -import postgres from 'postgres'; -import * as schema from './schema'; - -let connection: ReturnType | null = null; -let db: ReturnType | null = null; - -export function getConnection(databaseUrl: string) { - if (!connection) { - connection = postgres(databaseUrl, { - max: 10, - idle_timeout: 20, - connect_timeout: 10, - }); - } - return connection; -} - -export function getDb(databaseUrl: string) { - if (!db) { - const conn = getConnection(databaseUrl); - db = drizzle(conn, { schema }); - } - return db; -} - -export async function closeConnection() { - if (connection) { - await connection.end(); - connection = null; - db = null; - } -} diff --git a/services/mana-notify/src/db/database.module.ts b/services/mana-notify/src/db/database.module.ts deleted file mode 100644 index ffc4b6b06..000000000 --- a/services/mana-notify/src/db/database.module.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { Global, Module } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { getDb } from './connection'; - -export const DATABASE_CONNECTION = 'DATABASE_CONNECTION'; - -@Global() -@Module({ - providers: [ - { - provide: DATABASE_CONNECTION, - useFactory: (configService: ConfigService) => { - const databaseUrl = configService.get('database.url'); - if (!databaseUrl) { - throw new Error('DATABASE_URL is not configured'); - } - return getDb(databaseUrl); - }, - inject: [ConfigService], - }, - ], - exports: [DATABASE_CONNECTION], -}) -export class DatabaseModule {} diff --git a/services/mana-notify/src/db/migrate.ts b/services/mana-notify/src/db/migrate.ts deleted file mode 100644 index d3506fcc6..000000000 --- a/services/mana-notify/src/db/migrate.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { drizzle } from 'drizzle-orm/postgres-js'; -import { migrate } from 'drizzle-orm/postgres-js/migrator'; -import postgres from 'postgres'; - -async function runMigrations() { - const databaseUrl = process.env.DATABASE_URL; - - if (!databaseUrl) { - console.error('DATABASE_URL environment variable is not set'); - process.exit(1); - } - - console.log('Running migrations...'); - - const sql = postgres(databaseUrl, { max: 1 }); - const db = drizzle(sql); - - try { - await migrate(db, { migrationsFolder: './drizzle' }); - console.log('Migrations completed successfully'); - } catch (error) { - console.error('Migration failed:', error); - process.exit(1); - } finally { - await sql.end(); - } -} - -runMigrations(); diff --git a/services/mana-notify/src/db/schema/delivery-logs.schema.ts b/services/mana-notify/src/db/schema/delivery-logs.schema.ts deleted file mode 100644 index fd2eaa3af..000000000 --- a/services/mana-notify/src/db/schema/delivery-logs.schema.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { - pgSchema, - uuid, - text, - integer, - boolean, - timestamp, - index, - varchar, -} from 'drizzle-orm/pg-core'; -import { notifySchema, channelEnum, notifications } from './notifications.schema'; - -export const deliveryLogs = notifySchema.table( - 'delivery_logs', - { - id: uuid('id').defaultRandom().primaryKey(), - - // Reference - notificationId: uuid('notification_id') - .notNull() - .references(() => notifications.id, { onDelete: 'cascade' }), - - // Attempt info - attemptNumber: integer('attempt_number').notNull(), - channel: channelEnum('channel').notNull(), - - // Result - success: boolean('success').notNull(), - statusCode: integer('status_code'), - errorMessage: text('error_message'), - - // Provider info - providerId: varchar('provider_id', { length: 255 }), // Expo ticket ID, email message ID, etc. - - // Performance - durationMs: integer('duration_ms'), - - // Timestamp - createdAt: timestamp('created_at', { withTimezone: true }).defaultNow().notNull(), - }, - (table) => ({ - notificationIdIdx: index('delivery_logs_notification_id_idx').on(table.notificationId), - successIdx: index('delivery_logs_success_idx').on(table.success), - createdAtIdx: index('delivery_logs_created_at_idx').on(table.createdAt), - }) -); - -export type DeliveryLog = typeof deliveryLogs.$inferSelect; -export type NewDeliveryLog = typeof deliveryLogs.$inferInsert; diff --git a/services/mana-notify/src/db/schema/devices.schema.ts b/services/mana-notify/src/db/schema/devices.schema.ts deleted file mode 100644 index 5ae164c91..000000000 --- a/services/mana-notify/src/db/schema/devices.schema.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { - pgSchema, - uuid, - text, - varchar, - boolean, - timestamp, - index, - uniqueIndex, -} from 'drizzle-orm/pg-core'; -import { notifySchema } from './notifications.schema'; - -export const devices = notifySchema.table( - 'devices', - { - id: uuid('id').defaultRandom().primaryKey(), - - // Owner - userId: text('user_id').notNull(), - - // Token - pushToken: text('push_token').notNull(), - tokenType: varchar('token_type', { length: 20 }).notNull().default('expo'), // expo, fcm, apns - - // Device Info - platform: varchar('platform', { length: 20 }).notNull(), // ios, android, web - deviceName: varchar('device_name', { length: 100 }), - appId: varchar('app_id', { length: 50 }), // Which app registered this device - - // Status - isActive: boolean('is_active').notNull().default(true), - lastSeenAt: timestamp('last_seen_at', { withTimezone: true }), - - // Timestamps - createdAt: timestamp('created_at', { withTimezone: true }).defaultNow().notNull(), - updatedAt: timestamp('updated_at', { withTimezone: true }).defaultNow().notNull(), - }, - (table) => ({ - userIdIdx: index('devices_user_id_idx').on(table.userId), - pushTokenIdx: uniqueIndex('devices_push_token_idx').on(table.pushToken), - platformIdx: index('devices_platform_idx').on(table.platform), - }) -); - -export type Device = typeof devices.$inferSelect; -export type NewDevice = typeof devices.$inferInsert; diff --git a/services/mana-notify/src/db/schema/index.ts b/services/mana-notify/src/db/schema/index.ts deleted file mode 100644 index 73c75a4e7..000000000 --- a/services/mana-notify/src/db/schema/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -export * from './notifications.schema'; -export * from './templates.schema'; -export * from './devices.schema'; -export * from './preferences.schema'; -export * from './delivery-logs.schema'; diff --git a/services/mana-notify/src/db/schema/notifications.schema.ts b/services/mana-notify/src/db/schema/notifications.schema.ts deleted file mode 100644 index bfdfc47f4..000000000 --- a/services/mana-notify/src/db/schema/notifications.schema.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { - pgSchema, - uuid, - text, - varchar, - integer, - timestamp, - index, - jsonb, -} from 'drizzle-orm/pg-core'; - -export const notifySchema = pgSchema('notify'); - -// Channel enum -export const channelEnum = notifySchema.enum('channel', ['email', 'push', 'matrix', 'webhook']); - -// Status enum -export const statusEnum = notifySchema.enum('notification_status', [ - 'pending', - 'processing', - 'delivered', - 'failed', - 'cancelled', -]); - -// Priority enum -export const priorityEnum = notifySchema.enum('priority', ['low', 'normal', 'high', 'critical']); - -export const notifications = notifySchema.table( - 'notifications', - { - id: uuid('id').defaultRandom().primaryKey(), - - // Target - userId: text('user_id'), - appId: varchar('app_id', { length: 50 }).notNull(), // calendar, chat, auth, etc. - - // Channel & Template - channel: channelEnum('channel').notNull(), - templateId: varchar('template_id', { length: 100 }), - - // Content - subject: varchar('subject', { length: 500 }), - body: text('body'), - data: jsonb('data').$type>(), // Template variables - - // Delivery - status: statusEnum('status').notNull().default('pending'), - priority: priorityEnum('priority').notNull().default('normal'), - scheduledFor: timestamp('scheduled_for', { withTimezone: true }), - recipient: varchar('recipient', { length: 500 }), // Email, Matrix Room, Webhook URL - - // Idempotency - externalId: varchar('external_id', { length: 255 }), - - // Processing - attempts: integer('attempts').notNull().default(0), - deliveredAt: timestamp('delivered_at', { withTimezone: true }), - errorMessage: text('error_message'), - - // Timestamps - createdAt: timestamp('created_at', { withTimezone: true }).defaultNow().notNull(), - updatedAt: timestamp('updated_at', { withTimezone: true }).defaultNow().notNull(), - }, - (table) => ({ - userIdIdx: index('notifications_user_id_idx').on(table.userId), - appIdIdx: index('notifications_app_id_idx').on(table.appId), - statusIdx: index('notifications_status_idx').on(table.status), - scheduledForIdx: index('notifications_scheduled_for_idx').on(table.scheduledFor), - externalIdIdx: index('notifications_external_id_idx').on(table.externalId), - }) -); - -export type Notification = typeof notifications.$inferSelect; -export type NewNotification = typeof notifications.$inferInsert; diff --git a/services/mana-notify/src/db/schema/preferences.schema.ts b/services/mana-notify/src/db/schema/preferences.schema.ts deleted file mode 100644 index b3964b1c3..000000000 --- a/services/mana-notify/src/db/schema/preferences.schema.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { - pgSchema, - uuid, - text, - varchar, - boolean, - timestamp, - uniqueIndex, - jsonb, -} from 'drizzle-orm/pg-core'; -import { notifySchema } from './notifications.schema'; - -export const preferences = notifySchema.table( - 'preferences', - { - id: uuid('id').defaultRandom().primaryKey(), - - // Owner - userId: text('user_id').notNull(), - - // Global settings - emailEnabled: boolean('email_enabled').notNull().default(false), - pushEnabled: boolean('push_enabled').notNull().default(true), - - // Quiet hours - quietHoursEnabled: boolean('quiet_hours_enabled').notNull().default(false), - quietHoursStart: varchar('quiet_hours_start', { length: 5 }), // "22:00" - quietHoursEnd: varchar('quiet_hours_end', { length: 5 }), // "08:00" - timezone: varchar('timezone', { length: 50 }).notNull().default('Europe/Berlin'), - - // Per-category preferences - // e.g., { "calendar": { "reminders": true, "shares": false }, "chat": { "messages": true } } - categoryPreferences: - jsonb('category_preferences').$type>>(), - - // Timestamps - createdAt: timestamp('created_at', { withTimezone: true }).defaultNow().notNull(), - updatedAt: timestamp('updated_at', { withTimezone: true }).defaultNow().notNull(), - }, - (table) => ({ - userIdIdx: uniqueIndex('preferences_user_id_idx').on(table.userId), - }) -); - -export type Preference = typeof preferences.$inferSelect; -export type NewPreference = typeof preferences.$inferInsert; diff --git a/services/mana-notify/src/db/schema/templates.schema.ts b/services/mana-notify/src/db/schema/templates.schema.ts deleted file mode 100644 index 35a7a88d0..000000000 --- a/services/mana-notify/src/db/schema/templates.schema.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { - pgSchema, - uuid, - text, - varchar, - boolean, - timestamp, - index, - jsonb, - uniqueIndex, -} from 'drizzle-orm/pg-core'; -import { notifySchema, channelEnum } from './notifications.schema'; - -export const templates = notifySchema.table( - 'templates', - { - id: uuid('id').defaultRandom().primaryKey(), - - // Identification - slug: varchar('slug', { length: 100 }).notNull(), // e.g. "auth-password-reset" - appId: varchar('app_id', { length: 50 }), // NULL = system template - - // Channel & Content - channel: channelEnum('channel').notNull(), - subject: varchar('subject', { length: 500 }), // Handlebars template - bodyTemplate: text('body_template').notNull(), // Handlebars template - - // Localization - locale: varchar('locale', { length: 10 }).notNull().default('de-DE'), - - // Settings - isActive: boolean('is_active').notNull().default(true), - isSystem: boolean('is_system').notNull().default(false), // System templates cannot be deleted - - // Metadata - variables: jsonb('variables').$type>(), // Expected variables with descriptions - - // Timestamps - createdAt: timestamp('created_at', { withTimezone: true }).defaultNow().notNull(), - updatedAt: timestamp('updated_at', { withTimezone: true }).defaultNow().notNull(), - }, - (table) => ({ - slugLocaleIdx: uniqueIndex('templates_slug_locale_idx').on(table.slug, table.locale), - appIdIdx: index('templates_app_id_idx').on(table.appId), - channelIdx: index('templates_channel_idx').on(table.channel), - }) -); - -export type Template = typeof templates.$inferSelect; -export type NewTemplate = typeof templates.$inferInsert; diff --git a/services/mana-notify/src/devices/devices.controller.ts b/services/mana-notify/src/devices/devices.controller.ts deleted file mode 100644 index a44a76630..000000000 --- a/services/mana-notify/src/devices/devices.controller.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { - Controller, - Get, - Post, - Delete, - Body, - Param, - UseGuards, - HttpCode, - HttpStatus, - Req, -} from '@nestjs/common'; -import { DevicesService, RegisterDeviceDto } from './devices.service'; -import { JwtAuthGuard, AuthenticatedRequest } from '../common/guards/jwt-auth.guard'; -import { Device } from '../db/schema'; -import { IsString, IsOptional } from 'class-validator'; - -class RegisterDeviceRequestDto { - @IsString() - pushToken!: string; - - @IsString() - @IsOptional() - tokenType?: string; - - @IsString() - platform!: string; - - @IsString() - @IsOptional() - deviceName?: string; - - @IsString() - @IsOptional() - appId?: string; -} - -@Controller('devices') -@UseGuards(JwtAuthGuard) -export class DevicesController { - constructor(private readonly devicesService: DevicesService) {} - - @Get() - async listDevices(@Req() req: AuthenticatedRequest): Promise<{ devices: Device[] }> { - const devicesList = await this.devicesService.getByUserId(req.user.userId); - return { devices: devicesList }; - } - - @Post('register') - async register( - @Req() req: AuthenticatedRequest, - @Body() dto: RegisterDeviceRequestDto - ): Promise<{ device: Device }> { - const device = await this.devicesService.register(req.user.userId, dto); - return { device }; - } - - @Delete(':id') - @HttpCode(HttpStatus.NO_CONTENT) - async unregister(@Req() req: AuthenticatedRequest, @Param('id') id: string): Promise { - await this.devicesService.unregister(req.user.userId, id); - } -} diff --git a/services/mana-notify/src/devices/devices.module.ts b/services/mana-notify/src/devices/devices.module.ts deleted file mode 100644 index e79d6bbd5..000000000 --- a/services/mana-notify/src/devices/devices.module.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { Module } from '@nestjs/common'; -import { DevicesService } from './devices.service'; -import { DevicesController } from './devices.controller'; - -@Module({ - providers: [DevicesService], - controllers: [DevicesController], - exports: [DevicesService], -}) -export class DevicesModule {} diff --git a/services/mana-notify/src/devices/devices.service.ts b/services/mana-notify/src/devices/devices.service.ts deleted file mode 100644 index 20e28ab22..000000000 --- a/services/mana-notify/src/devices/devices.service.ts +++ /dev/null @@ -1,141 +0,0 @@ -import { Injectable, Logger, Inject, NotFoundException, ConflictException } from '@nestjs/common'; -import { eq, and } from 'drizzle-orm'; -import { DATABASE_CONNECTION } from '../db/database.module'; -import { devices, type Device, type NewDevice } from '../db/schema'; - -export interface RegisterDeviceDto { - pushToken: string; - tokenType?: string; - platform: string; - deviceName?: string; - appId?: string; -} - -@Injectable() -export class DevicesService { - private readonly logger = new Logger(DevicesService.name); - - constructor(@Inject(DATABASE_CONNECTION) private readonly db: any) {} - - async register(userId: string, dto: RegisterDeviceDto): Promise { - this.logger.debug(`Registering device for user ${userId}`); - - // Check if token already exists - const [existing] = await this.db - .select() - .from(devices) - .where(eq(devices.pushToken, dto.pushToken)) - .limit(1); - - if (existing) { - // If same user, just update - if (existing.userId === userId) { - const [updated] = await this.db - .update(devices) - .set({ - platform: dto.platform, - deviceName: dto.deviceName, - appId: dto.appId, - isActive: true, - lastSeenAt: new Date(), - updatedAt: new Date(), - }) - .where(eq(devices.id, existing.id)) - .returning(); - - this.logger.log(`Updated existing device ${existing.id} for user ${userId}`); - return updated; - } else { - // Token belongs to different user - transfer ownership - const [updated] = await this.db - .update(devices) - .set({ - userId, - platform: dto.platform, - deviceName: dto.deviceName, - appId: dto.appId, - isActive: true, - lastSeenAt: new Date(), - updatedAt: new Date(), - }) - .where(eq(devices.id, existing.id)) - .returning(); - - this.logger.log(`Transferred device ${existing.id} to user ${userId}`); - return updated; - } - } - - // Create new device - const [device] = await this.db - .insert(devices) - .values({ - userId, - pushToken: dto.pushToken, - tokenType: dto.tokenType || 'expo', - platform: dto.platform, - deviceName: dto.deviceName, - appId: dto.appId, - isActive: true, - lastSeenAt: new Date(), - }) - .returning(); - - this.logger.log(`Registered new device ${device.id} for user ${userId}`); - return device; - } - - async unregister(userId: string, deviceId: string): Promise { - const [device] = await this.db - .select() - .from(devices) - .where(and(eq(devices.id, deviceId), eq(devices.userId, userId))) - .limit(1); - - if (!device) { - throw new NotFoundException(`Device ${deviceId} not found`); - } - - await this.db.delete(devices).where(eq(devices.id, deviceId)); - this.logger.log(`Unregistered device ${deviceId} for user ${userId}`); - } - - async getByUserId(userId: string): Promise { - return this.db.select().from(devices).where(eq(devices.userId, userId)); - } - - async getActiveDevicesByUser(userId: string): Promise { - return this.db - .select() - .from(devices) - .where(and(eq(devices.userId, userId), eq(devices.isActive, true))); - } - - async getById(id: string): Promise { - const [device] = await this.db.select().from(devices).where(eq(devices.id, id)).limit(1); - return device || null; - } - - async deactivate(deviceId: string): Promise { - await this.db - .update(devices) - .set({ isActive: false, updatedAt: new Date() }) - .where(eq(devices.id, deviceId)); - } - - async updateLastSeen(deviceId: string): Promise { - await this.db - .update(devices) - .set({ lastSeenAt: new Date(), updatedAt: new Date() }) - .where(eq(devices.id, deviceId)); - } - - async deactivateByToken(pushToken: string): Promise { - await this.db - .update(devices) - .set({ isActive: false, updatedAt: new Date() }) - .where(eq(devices.pushToken, pushToken)); - - this.logger.debug(`Deactivated device with token ${pushToken.substring(0, 20)}...`); - } -} diff --git a/services/mana-notify/src/health/health.controller.ts b/services/mana-notify/src/health/health.controller.ts deleted file mode 100644 index 4bd456345..000000000 --- a/services/mana-notify/src/health/health.controller.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { Controller, Get, Inject } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { DATABASE_CONNECTION } from '../db/database.module'; -import { sql } from 'drizzle-orm'; - -interface HealthStatus { - status: 'healthy' | 'unhealthy'; - version: string; - timestamp: string; - services: { - database: boolean; - redis: boolean; - }; -} - -@Controller() -export class HealthController { - constructor( - private readonly configService: ConfigService, - @Inject(DATABASE_CONNECTION) private readonly db: any - ) {} - - @Get('/health') - async getHealth(): Promise { - const dbHealthy = await this.checkDatabase(); - - return { - status: dbHealthy ? 'healthy' : 'unhealthy', - version: '1.0.0', - timestamp: new Date().toISOString(), - services: { - database: dbHealthy, - redis: true, // BullMQ manages Redis connection - }, - }; - } - - private async checkDatabase(): Promise { - try { - await this.db.execute(sql`SELECT 1`); - return true; - } catch { - return false; - } - } -} diff --git a/services/mana-notify/src/health/health.module.ts b/services/mana-notify/src/health/health.module.ts deleted file mode 100644 index a61d8b044..000000000 --- a/services/mana-notify/src/health/health.module.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { Module } from '@nestjs/common'; -import { HealthController } from './health.controller'; - -@Module({ - controllers: [HealthController], -}) -export class HealthModule {} diff --git a/services/mana-notify/src/main.ts b/services/mana-notify/src/main.ts deleted file mode 100644 index c1b056de4..000000000 --- a/services/mana-notify/src/main.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { NestFactory } from '@nestjs/core'; -import { ValidationPipe, Logger } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; -import { AppModule } from './app.module'; -import { HttpExceptionFilter } from './common/filters/http-exception.filter'; - -async function bootstrap() { - const logger = new Logger('Bootstrap'); - - const app = await NestFactory.create(AppModule); - - const configService = app.get(ConfigService); - const port = configService.get('port', 3040); - - // Global prefix - app.setGlobalPrefix('api/v1'); - - // CORS - app.enableCors({ - origin: configService.get('cors.origins', ['http://localhost:*']), - credentials: true, - }); - - // Global pipes - app.useGlobalPipes( - new ValidationPipe({ - whitelist: true, - transform: true, - forbidNonWhitelisted: true, - }) - ); - - // Global filters - app.useGlobalFilters(new HttpExceptionFilter()); - - await app.listen(port); - logger.log(`Mana Notify Service running on port ${port}`); - logger.log(`Health check: http://localhost:${port}/health`); - logger.log(`Metrics: http://localhost:${port}/metrics`); -} - -bootstrap(); diff --git a/services/mana-notify/src/metrics/metrics.controller.ts b/services/mana-notify/src/metrics/metrics.controller.ts deleted file mode 100644 index 5679d2eab..000000000 --- a/services/mana-notify/src/metrics/metrics.controller.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { Controller, Get, Header } from '@nestjs/common'; -import { MetricsService } from './metrics.service'; - -@Controller() -export class MetricsController { - constructor(private readonly metricsService: MetricsService) {} - - @Get('/metrics') - @Header('Content-Type', 'text/plain') - async getMetrics(): Promise { - return this.metricsService.getMetrics(); - } -} diff --git a/services/mana-notify/src/metrics/metrics.module.ts b/services/mana-notify/src/metrics/metrics.module.ts deleted file mode 100644 index 42029f4b8..000000000 --- a/services/mana-notify/src/metrics/metrics.module.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { Module } from '@nestjs/common'; -import { MetricsService } from './metrics.service'; -import { MetricsController } from './metrics.controller'; - -@Module({ - providers: [MetricsService], - controllers: [MetricsController], - exports: [MetricsService], -}) -export class MetricsModule {} diff --git a/services/mana-notify/src/metrics/metrics.service.ts b/services/mana-notify/src/metrics/metrics.service.ts deleted file mode 100644 index 42cc4ea5a..000000000 --- a/services/mana-notify/src/metrics/metrics.service.ts +++ /dev/null @@ -1,138 +0,0 @@ -import { Injectable, OnModuleInit } from '@nestjs/common'; -import { Counter, Histogram, Registry, collectDefaultMetrics } from 'prom-client'; - -@Injectable() -export class MetricsService implements OnModuleInit { - private readonly registry: Registry; - - // Notification counters - private readonly notificationsSent: Counter; - private readonly notificationsFailed: Counter; - - // Channel-specific counters - private readonly emailsSent: Counter; - private readonly pushNotificationsSent: Counter; - private readonly matrixMessagesSent: Counter; - private readonly webhooksSent: Counter; - - // Latency histograms - private readonly notificationLatency: Histogram; - private readonly emailLatency: Histogram; - private readonly pushLatency: Histogram; - - constructor() { - this.registry = new Registry(); - - // Total notifications - this.notificationsSent = new Counter({ - name: 'mana_notify_notifications_sent_total', - help: 'Total number of notifications sent successfully', - labelNames: ['channel', 'app_id'], - registers: [this.registry], - }); - - this.notificationsFailed = new Counter({ - name: 'mana_notify_notifications_failed_total', - help: 'Total number of notifications that failed to send', - labelNames: ['channel', 'app_id', 'error_type'], - registers: [this.registry], - }); - - // Channel-specific - this.emailsSent = new Counter({ - name: 'mana_notify_emails_sent_total', - help: 'Total number of emails sent', - labelNames: ['template', 'status'], - registers: [this.registry], - }); - - this.pushNotificationsSent = new Counter({ - name: 'mana_notify_push_sent_total', - help: 'Total number of push notifications sent', - labelNames: ['platform', 'status'], - registers: [this.registry], - }); - - this.matrixMessagesSent = new Counter({ - name: 'mana_notify_matrix_sent_total', - help: 'Total number of Matrix messages sent', - labelNames: ['status'], - registers: [this.registry], - }); - - this.webhooksSent = new Counter({ - name: 'mana_notify_webhooks_sent_total', - help: 'Total number of webhooks sent', - labelNames: ['status'], - registers: [this.registry], - }); - - // Latency - this.notificationLatency = new Histogram({ - name: 'mana_notify_notification_latency_seconds', - help: 'Notification processing latency in seconds', - labelNames: ['channel'], - buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5], - registers: [this.registry], - }); - - this.emailLatency = new Histogram({ - name: 'mana_notify_email_latency_seconds', - help: 'Email sending latency in seconds', - buckets: [0.1, 0.5, 1, 2, 5, 10], - registers: [this.registry], - }); - - this.pushLatency = new Histogram({ - name: 'mana_notify_push_latency_seconds', - help: 'Push notification sending latency in seconds', - buckets: [0.01, 0.05, 0.1, 0.5, 1], - registers: [this.registry], - }); - } - - onModuleInit() { - collectDefaultMetrics({ register: this.registry }); - } - - // Recording methods - recordNotificationSent(channel: string, appId: string) { - this.notificationsSent.inc({ channel, app_id: appId }); - } - - recordNotificationFailed(channel: string, appId: string, errorType: string) { - this.notificationsFailed.inc({ channel, app_id: appId, error_type: errorType }); - } - - recordEmailSent(template: string, success: boolean) { - this.emailsSent.inc({ template, status: success ? 'success' : 'failure' }); - } - - recordPushSent(platform: string, success: boolean) { - this.pushNotificationsSent.inc({ platform, status: success ? 'success' : 'failure' }); - } - - recordMatrixSent(success: boolean) { - this.matrixMessagesSent.inc({ status: success ? 'success' : 'failure' }); - } - - recordWebhookSent(success: boolean) { - this.webhooksSent.inc({ status: success ? 'success' : 'failure' }); - } - - recordNotificationLatency(channel: string, durationSeconds: number) { - this.notificationLatency.observe({ channel }, durationSeconds); - } - - recordEmailLatency(durationSeconds: number) { - this.emailLatency.observe(durationSeconds); - } - - recordPushLatency(durationSeconds: number) { - this.pushLatency.observe(durationSeconds); - } - - async getMetrics(): Promise { - return this.registry.metrics(); - } -} diff --git a/services/mana-notify/src/notifications/dto/index.ts b/services/mana-notify/src/notifications/dto/index.ts deleted file mode 100644 index d59790ed2..000000000 --- a/services/mana-notify/src/notifications/dto/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './send-notification.dto'; diff --git a/services/mana-notify/src/notifications/dto/send-notification.dto.ts b/services/mana-notify/src/notifications/dto/send-notification.dto.ts deleted file mode 100644 index e1534386d..000000000 --- a/services/mana-notify/src/notifications/dto/send-notification.dto.ts +++ /dev/null @@ -1,161 +0,0 @@ -import { - IsString, - IsOptional, - IsEnum, - IsObject, - IsArray, - IsDateString, - ValidateNested, - IsBoolean, -} from 'class-validator'; -import { Type } from 'class-transformer'; - -export enum NotificationChannel { - EMAIL = 'email', - PUSH = 'push', - MATRIX = 'matrix', - WEBHOOK = 'webhook', -} - -export enum NotificationPriority { - LOW = 'low', - NORMAL = 'normal', - HIGH = 'high', - CRITICAL = 'critical', -} - -export class EmailData { - @IsString() - from?: string; - - @IsString() - @IsOptional() - replyTo?: string; -} - -export class PushData { - @IsString() - @IsOptional() - sound?: 'default' | null; - - @IsOptional() - badge?: number; - - @IsString() - @IsOptional() - channelId?: string; -} - -export class WebhookData { - @IsString() - @IsOptional() - method?: 'POST' | 'PUT'; - - @IsObject() - @IsOptional() - headers?: Record; - - @IsOptional() - timeout?: number; -} - -export class MatrixData { - @IsString() - @IsOptional() - msgtype?: 'text' | 'notice'; - - @IsString() - @IsOptional() - formattedBody?: string; -} - -export class SendNotificationDto { - @IsEnum(NotificationChannel) - channel!: NotificationChannel; - - @IsString() - appId!: string; - - @IsString() - @IsOptional() - userId?: string; - - @IsString() - @IsOptional() - recipient?: string; // Email address, push token, room ID, or webhook URL - - @IsArray() - @IsString({ each: true }) - @IsOptional() - recipients?: string[]; // For batch sending to multiple recipients - - @IsString() - @IsOptional() - template?: string; // Template slug - - @IsString() - @IsOptional() - subject?: string; // Override template subject - - @IsString() - @IsOptional() - body?: string; // Override template body or custom content - - @IsObject() - @IsOptional() - data?: Record; // Template variables or push data payload - - @IsEnum(NotificationPriority) - @IsOptional() - priority?: NotificationPriority; - - @IsString() - @IsOptional() - externalId?: string; // For idempotency - - // Channel-specific options - @ValidateNested() - @Type(() => EmailData) - @IsOptional() - emailOptions?: EmailData; - - @ValidateNested() - @Type(() => PushData) - @IsOptional() - pushOptions?: PushData; - - @ValidateNested() - @Type(() => WebhookData) - @IsOptional() - webhookOptions?: WebhookData; - - @ValidateNested() - @Type(() => MatrixData) - @IsOptional() - matrixOptions?: MatrixData; -} - -export class ScheduleNotificationDto extends SendNotificationDto { - @IsDateString() - scheduledFor!: string; -} - -export class BatchNotificationDto { - @IsArray() - @ValidateNested({ each: true }) - @Type(() => SendNotificationDto) - notifications!: SendNotificationDto[]; -} - -export class NotificationResponse { - id!: string; - status!: string; - channel!: string; - createdAt!: Date; -} - -export class BatchNotificationResponse { - results!: NotificationResponse[]; - succeeded!: number; - failed!: number; -} diff --git a/services/mana-notify/src/notifications/notifications.controller.ts b/services/mana-notify/src/notifications/notifications.controller.ts deleted file mode 100644 index 6564cc106..000000000 --- a/services/mana-notify/src/notifications/notifications.controller.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { - Controller, - Get, - Post, - Delete, - Body, - Param, - UseGuards, - HttpCode, - HttpStatus, -} from '@nestjs/common'; -import { NotificationsService } from './notifications.service'; -import { ServiceAuthGuard } from '../common/guards/service-auth.guard'; -import { - SendNotificationDto, - ScheduleNotificationDto, - BatchNotificationDto, - NotificationResponse, - BatchNotificationResponse, -} from './dto/send-notification.dto'; -import { Notification } from '../db/schema'; - -@Controller('notifications') -@UseGuards(ServiceAuthGuard) -export class NotificationsController { - constructor(private readonly notificationsService: NotificationsService) {} - - @Post('send') - async send(@Body() dto: SendNotificationDto): Promise<{ notification: NotificationResponse }> { - const notification = await this.notificationsService.send(dto); - return { notification }; - } - - @Post('schedule') - async schedule( - @Body() dto: ScheduleNotificationDto - ): Promise<{ notification: NotificationResponse }> { - const notification = await this.notificationsService.schedule(dto); - return { notification }; - } - - @Post('batch') - async batch(@Body() dto: BatchNotificationDto): Promise { - const results = await this.notificationsService.sendBatch(dto.notifications); - - const succeeded = results.filter((r) => r.status !== 'failed').length; - const failed = results.length - succeeded; - - return { - results, - succeeded, - failed, - }; - } - - @Get(':id') - async getById(@Param('id') id: string): Promise<{ notification: Notification | null }> { - const notification = await this.notificationsService.getById(id); - return { notification }; - } - - @Delete(':id') - @HttpCode(HttpStatus.OK) - async cancel(@Param('id') id: string): Promise<{ notification: Notification }> { - const notification = await this.notificationsService.cancel(id); - return { notification }; - } -} diff --git a/services/mana-notify/src/notifications/notifications.module.ts b/services/mana-notify/src/notifications/notifications.module.ts deleted file mode 100644 index ef8a62ba1..000000000 --- a/services/mana-notify/src/notifications/notifications.module.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { Module } from '@nestjs/common'; -import { NotificationsService } from './notifications.service'; -import { NotificationsController } from './notifications.controller'; -import { TemplatesModule } from '../templates/templates.module'; -import { QueueModule } from '../queue/queue.module'; -import { DevicesModule } from '../devices/devices.module'; -import { PreferencesModule } from '../preferences/preferences.module'; - -@Module({ - imports: [TemplatesModule, QueueModule, DevicesModule, PreferencesModule], - providers: [NotificationsService], - controllers: [NotificationsController], - exports: [NotificationsService], -}) -export class NotificationsModule {} diff --git a/services/mana-notify/src/notifications/notifications.service.ts b/services/mana-notify/src/notifications/notifications.service.ts deleted file mode 100644 index e450a1abd..000000000 --- a/services/mana-notify/src/notifications/notifications.service.ts +++ /dev/null @@ -1,403 +0,0 @@ -import { Injectable, Logger, Inject, BadRequestException, NotFoundException } from '@nestjs/common'; -import { InjectQueue } from '@nestjs/bullmq'; -import { Queue } from 'bullmq'; -import { eq, and, desc } from 'drizzle-orm'; -import { DATABASE_CONNECTION } from '../db/database.module'; -import { notifications, type Notification, type NewNotification } from '../db/schema'; -import { EMAIL_QUEUE, PUSH_QUEUE, MATRIX_QUEUE, WEBHOOK_QUEUE } from '../queue/queue.module'; -import { TemplatesService } from '../templates/templates.service'; -import { DevicesService } from '../devices/devices.service'; -import { PreferencesService } from '../preferences/preferences.service'; -import { - SendNotificationDto, - ScheduleNotificationDto, - NotificationResponse, - NotificationChannel, -} from './dto/send-notification.dto'; -import { EmailJob } from '../queue/processors/email.processor'; -import { PushJob } from '../queue/processors/push.processor'; -import { MatrixJob } from '../queue/processors/matrix.processor'; -import { WebhookJob } from '../queue/processors/webhook.processor'; - -@Injectable() -export class NotificationsService { - private readonly logger = new Logger(NotificationsService.name); - - constructor( - @Inject(DATABASE_CONNECTION) private readonly db: any, - @InjectQueue(EMAIL_QUEUE) private readonly emailQueue: Queue, - @InjectQueue(PUSH_QUEUE) private readonly pushQueue: Queue, - @InjectQueue(MATRIX_QUEUE) private readonly matrixQueue: Queue, - @InjectQueue(WEBHOOK_QUEUE) private readonly webhookQueue: Queue, - private readonly templatesService: TemplatesService, - private readonly devicesService: DevicesService, - private readonly preferencesService: PreferencesService - ) {} - - async send(dto: SendNotificationDto): Promise { - // Check for idempotency - if (dto.externalId) { - const existing = await this.findByExternalId(dto.externalId); - if (existing) { - return this.toResponse(existing); - } - } - - // Check user preferences if userId is provided - if (dto.userId) { - const allowed = await this.checkPreferences(dto.userId, dto.channel, dto.appId); - if (!allowed) { - this.logger.debug(`Notification blocked by user preferences: ${dto.userId}`); - // Still create the notification but mark as cancelled - const notification = await this.createNotification({ - ...this.dtoToNotification(dto), - status: 'cancelled', - errorMessage: 'Blocked by user preferences', - }); - return this.toResponse(notification); - } - } - - // Render template if specified - let subject = dto.subject; - let body = dto.body; - - if (dto.template) { - const rendered = await this.templatesService.renderBySlug(dto.template, dto.data || {}); - if (rendered) { - subject = subject || rendered.subject; - body = body || rendered.body; - } else { - this.logger.warn(`Template not found: ${dto.template}`); - } - } - - if (!body && dto.channel !== NotificationChannel.WEBHOOK) { - throw new BadRequestException('Either template or body must be provided'); - } - - // Create notification record - const notification = await this.createNotification({ - ...this.dtoToNotification(dto), - subject, - body, - }); - - // Queue the notification based on channel - await this.queueNotification(notification, dto); - - return this.toResponse(notification); - } - - async schedule(dto: ScheduleNotificationDto): Promise { - const scheduledFor = new Date(dto.scheduledFor); - - if (scheduledFor <= new Date()) { - throw new BadRequestException('scheduledFor must be in the future'); - } - - // Render template if specified - let subject = dto.subject; - let body = dto.body; - - if (dto.template) { - const rendered = await this.templatesService.renderBySlug(dto.template, dto.data || {}); - if (rendered) { - subject = subject || rendered.subject; - body = body || rendered.body; - } - } - - // Create notification record with scheduled status - const notification = await this.createNotification({ - ...this.dtoToNotification(dto), - subject, - body, - scheduledFor, - status: 'pending', - }); - - // Queue with delay - const delay = scheduledFor.getTime() - Date.now(); - await this.queueNotification(notification, dto, delay); - - return this.toResponse(notification); - } - - async sendBatch(dtos: SendNotificationDto[]): Promise { - const results: NotificationResponse[] = []; - - for (const dto of dtos) { - try { - const result = await this.send(dto); - results.push(result); - } catch (error) { - this.logger.error(`Batch notification failed: ${error}`); - // Create a failed notification record - const notification = await this.createNotification({ - ...this.dtoToNotification(dto), - status: 'failed', - errorMessage: error instanceof Error ? error.message : 'Unknown error', - }); - results.push(this.toResponse(notification)); - } - } - - return results; - } - - async getById(id: string): Promise { - const [notification] = await this.db - .select() - .from(notifications) - .where(eq(notifications.id, id)) - .limit(1); - - return notification || null; - } - - async cancel(id: string): Promise { - const notification = await this.getById(id); - if (!notification) { - throw new NotFoundException(`Notification ${id} not found`); - } - - if (notification.status !== 'pending') { - throw new BadRequestException('Can only cancel pending notifications'); - } - - const [updated] = await this.db - .update(notifications) - .set({ status: 'cancelled', updatedAt: new Date() }) - .where(eq(notifications.id, id)) - .returning(); - - return updated; - } - - async listByUser(userId: string, limit: number = 50): Promise { - return this.db - .select() - .from(notifications) - .where(eq(notifications.userId, userId)) - .orderBy(desc(notifications.createdAt)) - .limit(limit); - } - - private async createNotification(data: NewNotification): Promise { - const [notification] = await this.db.insert(notifications).values(data).returning(); - return notification; - } - - private async findByExternalId(externalId: string): Promise { - const [notification] = await this.db - .select() - .from(notifications) - .where(eq(notifications.externalId, externalId)) - .limit(1); - - return notification || null; - } - - private async checkPreferences( - userId: string, - channel: NotificationChannel, - appId: string - ): Promise { - const prefs = await this.preferencesService.getByUserId(userId); - if (!prefs) { - return true; // No preferences = allow all - } - - // Check global channel settings - if (channel === NotificationChannel.EMAIL && !prefs.emailEnabled) { - return false; - } - if (channel === NotificationChannel.PUSH && !prefs.pushEnabled) { - return false; - } - - // Check quiet hours - if (prefs.quietHoursEnabled && this.isInQuietHours(prefs)) { - return false; - } - - return true; - } - - private isInQuietHours(prefs: { - quietHoursStart?: string | null; - quietHoursEnd?: string | null; - timezone?: string; - }): boolean { - if (!prefs.quietHoursStart || !prefs.quietHoursEnd) { - return false; - } - - const now = new Date(); - const [startHour, startMin] = prefs.quietHoursStart.split(':').map(Number); - const [endHour, endMin] = prefs.quietHoursEnd.split(':').map(Number); - - const currentHour = now.getHours(); - const currentMin = now.getMinutes(); - const currentTime = currentHour * 60 + currentMin; - const startTime = startHour * 60 + startMin; - const endTime = endHour * 60 + endMin; - - if (startTime <= endTime) { - return currentTime >= startTime && currentTime < endTime; - } else { - // Quiet hours span midnight - return currentTime >= startTime || currentTime < endTime; - } - } - - private dtoToNotification(dto: SendNotificationDto): NewNotification { - return { - userId: dto.userId, - appId: dto.appId, - channel: dto.channel, - templateId: dto.template, - subject: dto.subject, - body: dto.body, - data: dto.data, - recipient: dto.recipient, - externalId: dto.externalId, - priority: dto.priority || 'normal', - status: 'pending', - }; - } - - private async queueNotification( - notification: Notification, - dto: SendNotificationDto, - delay?: number - ): Promise { - const jobOptions = delay ? { delay } : undefined; - - switch (dto.channel) { - case NotificationChannel.EMAIL: - await this.queueEmail(notification, dto, jobOptions); - break; - case NotificationChannel.PUSH: - await this.queuePush(notification, dto, jobOptions); - break; - case NotificationChannel.MATRIX: - await this.queueMatrix(notification, dto, jobOptions); - break; - case NotificationChannel.WEBHOOK: - await this.queueWebhook(notification, dto, jobOptions); - break; - } - } - - private async queueEmail( - notification: Notification, - dto: SendNotificationDto, - jobOptions?: { delay: number } - ): Promise { - if (!dto.recipient) { - throw new BadRequestException('Email recipient is required'); - } - - const job: EmailJob = { - notificationId: notification.id, - to: dto.recipient, - subject: notification.subject || '', - html: notification.body || '', - from: dto.emailOptions?.from, - template: dto.template, - appId: dto.appId, - }; - - await this.emailQueue.add('send', job, jobOptions); - } - - private async queuePush( - notification: Notification, - dto: SendNotificationDto, - jobOptions?: { delay: number } - ): Promise { - let tokens: string[] = []; - - if (dto.recipients?.length) { - tokens = dto.recipients; - } else if (dto.recipient) { - tokens = [dto.recipient]; - } else if (dto.userId) { - // Get all device tokens for user - const devices = await this.devicesService.getActiveDevicesByUser(dto.userId); - tokens = devices.map((d) => d.pushToken); - } - - if (tokens.length === 0) { - throw new BadRequestException('No push tokens found'); - } - - const job: PushJob = { - notificationId: notification.id, - tokens, - title: notification.subject || '', - body: notification.body || '', - data: dto.data, - sound: dto.pushOptions?.sound, - badge: dto.pushOptions?.badge, - platform: 'mixed', - appId: dto.appId, - }; - - await this.pushQueue.add('send', job, jobOptions); - } - - private async queueMatrix( - notification: Notification, - dto: SendNotificationDto, - jobOptions?: { delay: number } - ): Promise { - if (!dto.recipient) { - throw new BadRequestException('Matrix room ID is required'); - } - - const job: MatrixJob = { - notificationId: notification.id, - roomId: dto.recipient, - body: notification.body || '', - formattedBody: dto.matrixOptions?.formattedBody, - msgtype: dto.matrixOptions?.msgtype, - appId: dto.appId, - }; - - await this.matrixQueue.add('send', job, jobOptions); - } - - private async queueWebhook( - notification: Notification, - dto: SendNotificationDto, - jobOptions?: { delay: number } - ): Promise { - if (!dto.recipient) { - throw new BadRequestException('Webhook URL is required'); - } - - const job: WebhookJob = { - notificationId: notification.id, - url: dto.recipient, - method: dto.webhookOptions?.method || 'POST', - headers: dto.webhookOptions?.headers, - body: dto.data || {}, - timeout: dto.webhookOptions?.timeout, - appId: dto.appId, - }; - - await this.webhookQueue.add('send', job, jobOptions); - } - - private toResponse(notification: Notification): NotificationResponse { - return { - id: notification.id, - status: notification.status, - channel: notification.channel, - createdAt: notification.createdAt, - }; - } -} diff --git a/services/mana-notify/src/preferences/preferences.controller.ts b/services/mana-notify/src/preferences/preferences.controller.ts deleted file mode 100644 index fc7f33073..000000000 --- a/services/mana-notify/src/preferences/preferences.controller.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { Controller, Get, Put, Body, UseGuards, Req } from '@nestjs/common'; -import { PreferencesService, UpdatePreferencesDto } from './preferences.service'; -import { JwtAuthGuard, AuthenticatedRequest } from '../common/guards/jwt-auth.guard'; -import { Preference } from '../db/schema'; -import { IsBoolean, IsOptional, IsString, IsObject, Matches } from 'class-validator'; - -class UpdatePreferencesRequestDto { - @IsBoolean() - @IsOptional() - emailEnabled?: boolean; - - @IsBoolean() - @IsOptional() - pushEnabled?: boolean; - - @IsBoolean() - @IsOptional() - quietHoursEnabled?: boolean; - - @IsString() - @IsOptional() - @Matches(/^([01]?[0-9]|2[0-3]):[0-5][0-9]$/, { - message: 'quietHoursStart must be in HH:mm format', - }) - quietHoursStart?: string; - - @IsString() - @IsOptional() - @Matches(/^([01]?[0-9]|2[0-3]):[0-5][0-9]$/, { - message: 'quietHoursEnd must be in HH:mm format', - }) - quietHoursEnd?: string; - - @IsString() - @IsOptional() - timezone?: string; - - @IsObject() - @IsOptional() - categoryPreferences?: Record>; -} - -@Controller('preferences') -@UseGuards(JwtAuthGuard) -export class PreferencesController { - constructor(private readonly preferencesService: PreferencesService) {} - - @Get() - async getPreferences(@Req() req: AuthenticatedRequest): Promise<{ preferences: Preference }> { - const prefs = await this.preferencesService.getOrCreate(req.user.userId); - return { preferences: prefs }; - } - - @Put() - async updatePreferences( - @Req() req: AuthenticatedRequest, - @Body() dto: UpdatePreferencesRequestDto - ): Promise<{ preferences: Preference }> { - const prefs = await this.preferencesService.update(req.user.userId, dto); - return { preferences: prefs }; - } -} diff --git a/services/mana-notify/src/preferences/preferences.module.ts b/services/mana-notify/src/preferences/preferences.module.ts deleted file mode 100644 index 21cc60686..000000000 --- a/services/mana-notify/src/preferences/preferences.module.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { Module } from '@nestjs/common'; -import { PreferencesService } from './preferences.service'; -import { PreferencesController } from './preferences.controller'; - -@Module({ - providers: [PreferencesService], - controllers: [PreferencesController], - exports: [PreferencesService], -}) -export class PreferencesModule {} diff --git a/services/mana-notify/src/preferences/preferences.service.ts b/services/mana-notify/src/preferences/preferences.service.ts deleted file mode 100644 index 7d95a3661..000000000 --- a/services/mana-notify/src/preferences/preferences.service.ts +++ /dev/null @@ -1,123 +0,0 @@ -import { Injectable, Logger, Inject } from '@nestjs/common'; -import { eq } from 'drizzle-orm'; -import { DATABASE_CONNECTION } from '../db/database.module'; -import { preferences, type Preference, type NewPreference } from '../db/schema'; - -export interface UpdatePreferencesDto { - emailEnabled?: boolean; - pushEnabled?: boolean; - quietHoursEnabled?: boolean; - quietHoursStart?: string; - quietHoursEnd?: string; - timezone?: string; - categoryPreferences?: Record>; -} - -@Injectable() -export class PreferencesService { - private readonly logger = new Logger(PreferencesService.name); - - constructor(@Inject(DATABASE_CONNECTION) private readonly db: any) {} - - async getByUserId(userId: string): Promise { - const [pref] = await this.db - .select() - .from(preferences) - .where(eq(preferences.userId, userId)) - .limit(1); - - return pref || null; - } - - async getOrCreate(userId: string): Promise { - const existingPref = await this.getByUserId(userId); - - if (existingPref) { - return existingPref; - } - - const [newPref] = await this.db - .insert(preferences) - .values({ - userId, - emailEnabled: false, - pushEnabled: true, - quietHoursEnabled: false, - timezone: 'Europe/Berlin', - }) - .returning(); - - this.logger.log(`Created default preferences for user ${userId}`); - return newPref; - } - - async update(userId: string, dto: UpdatePreferencesDto): Promise { - // First ensure preferences exist - await this.getOrCreate(userId); - - const updateData: Partial = {}; - - if (dto.emailEnabled !== undefined) { - updateData.emailEnabled = dto.emailEnabled; - } - if (dto.pushEnabled !== undefined) { - updateData.pushEnabled = dto.pushEnabled; - } - if (dto.quietHoursEnabled !== undefined) { - updateData.quietHoursEnabled = dto.quietHoursEnabled; - } - if (dto.quietHoursStart !== undefined) { - updateData.quietHoursStart = dto.quietHoursStart; - } - if (dto.quietHoursEnd !== undefined) { - updateData.quietHoursEnd = dto.quietHoursEnd; - } - if (dto.timezone !== undefined) { - updateData.timezone = dto.timezone; - } - if (dto.categoryPreferences !== undefined) { - updateData.categoryPreferences = dto.categoryPreferences; - } - - updateData.updatedAt = new Date(); - - const [updated] = await this.db - .update(preferences) - .set(updateData) - .where(eq(preferences.userId, userId)) - .returning(); - - this.logger.log(`Updated preferences for user ${userId}`); - return updated; - } - - async updateCategoryPreference( - userId: string, - appId: string, - category: string, - enabled: boolean - ): Promise { - const pref = await this.getOrCreate(userId); - - const categoryPrefs = pref.categoryPreferences || {}; - if (!categoryPrefs[appId]) { - categoryPrefs[appId] = {}; - } - categoryPrefs[appId][category] = enabled; - - return this.update(userId, { categoryPreferences: categoryPrefs }); - } - - isCategoryEnabled(pref: Preference, appId: string, category: string): boolean { - if (!pref.categoryPreferences) { - return true; // Default to enabled - } - - const appPrefs = pref.categoryPreferences[appId]; - if (!appPrefs) { - return true; - } - - return appPrefs[category] !== false; - } -} diff --git a/services/mana-notify/src/queue/processors/email.processor.ts b/services/mana-notify/src/queue/processors/email.processor.ts deleted file mode 100644 index ac50d17dd..000000000 --- a/services/mana-notify/src/queue/processors/email.processor.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { Processor, WorkerHost, OnWorkerEvent } from '@nestjs/bullmq'; -import { Logger, Inject } from '@nestjs/common'; -import { Job } from 'bullmq'; -import { eq } from 'drizzle-orm'; -import { EMAIL_QUEUE } from '../queue-names'; -import { EmailService } from '../../channels/email/email.service'; -import { MetricsService } from '../../metrics/metrics.service'; -import { DATABASE_CONNECTION } from '../../db/database.module'; -import { notifications, deliveryLogs, type NewDeliveryLog } from '../../db/schema'; - -export interface EmailJob { - notificationId: string; - to: string; - subject: string; - html: string; - text?: string; - from?: string; - template?: string; - appId: string; -} - -@Processor(EMAIL_QUEUE, { - concurrency: 5, -}) -export class EmailProcessor extends WorkerHost { - private readonly logger = new Logger(EmailProcessor.name); - - constructor( - private readonly emailService: EmailService, - private readonly metricsService: MetricsService, - @Inject(DATABASE_CONNECTION) private readonly db: any - ) { - super(); - } - - async process(job: Job): Promise { - const { notificationId, to, subject, html, text, from, template, appId } = job.data; - const startTime = Date.now(); - - this.logger.debug(`Processing email job ${job.id} to ${to}`); - - // Update notification status to processing - await this.updateNotificationStatus(notificationId, 'processing'); - - const result = await this.emailService.sendEmail({ - to, - subject, - html, - text, - from, - }); - - const durationMs = Date.now() - startTime; - - // Log the delivery attempt - await this.logDelivery({ - notificationId, - attemptNumber: job.attemptsMade + 1, - channel: 'email', - success: result.success, - errorMessage: result.error, - providerId: result.messageId, - durationMs, - }); - - // Record metrics - this.metricsService.recordEmailSent(template || 'custom', result.success); - this.metricsService.recordEmailLatency(durationMs / 1000); - - if (result.success) { - this.metricsService.recordNotificationSent('email', appId); - await this.updateNotificationStatus(notificationId, 'delivered', result.messageId); - this.logger.log(`Email sent successfully to ${to} in ${durationMs}ms`); - } else { - this.metricsService.recordNotificationFailed('email', appId, 'send_error'); - // Only mark as failed if no more retries - if (job.attemptsMade >= (job.opts.attempts || 3) - 1) { - await this.updateNotificationStatus(notificationId, 'failed', undefined, result.error); - } - throw new Error(result.error || 'Failed to send email'); - } - } - - @OnWorkerEvent('failed') - onFailed(job: Job, error: Error) { - this.logger.error(`Email job ${job.id} failed: ${error.message}`); - } - - private async updateNotificationStatus( - notificationId: string, - status: string, - providerId?: string, - errorMessage?: string - ): Promise { - try { - const updateData: Record = { - status, - updatedAt: new Date(), - }; - - if (status === 'delivered') { - updateData.deliveredAt = new Date(); - } - - if (errorMessage) { - updateData.errorMessage = errorMessage; - } - - await this.db - .update(notifications) - .set(updateData) - .where(eq(notifications.id, notificationId)); - } catch (error) { - this.logger.error(`Failed to update notification status: ${error}`); - } - } - - private async logDelivery(log: Omit): Promise { - try { - await this.db.insert(deliveryLogs).values(log); - } catch (error) { - this.logger.error(`Failed to log delivery: ${error}`); - } - } -} diff --git a/services/mana-notify/src/queue/processors/matrix.processor.ts b/services/mana-notify/src/queue/processors/matrix.processor.ts deleted file mode 100644 index aaf081ef9..000000000 --- a/services/mana-notify/src/queue/processors/matrix.processor.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { Processor, WorkerHost, OnWorkerEvent } from '@nestjs/bullmq'; -import { Logger, Inject } from '@nestjs/common'; -import { Job } from 'bullmq'; -import { eq } from 'drizzle-orm'; -import { MATRIX_QUEUE } from '../queue-names'; -import { MatrixService } from '../../channels/matrix/matrix.service'; -import { MetricsService } from '../../metrics/metrics.service'; -import { DATABASE_CONNECTION } from '../../db/database.module'; -import { notifications, deliveryLogs, type NewDeliveryLog } from '../../db/schema'; - -export interface MatrixJob { - notificationId: string; - roomId: string; - body: string; - formattedBody?: string; - msgtype?: 'text' | 'notice'; - appId: string; -} - -@Processor(MATRIX_QUEUE, { - concurrency: 5, -}) -export class MatrixProcessor extends WorkerHost { - private readonly logger = new Logger(MatrixProcessor.name); - - constructor( - private readonly matrixService: MatrixService, - private readonly metricsService: MetricsService, - @Inject(DATABASE_CONNECTION) private readonly db: any - ) { - super(); - } - - async process(job: Job): Promise { - const { notificationId, roomId, body, formattedBody, msgtype, appId } = job.data; - const startTime = Date.now(); - - this.logger.debug(`Processing Matrix job ${job.id} to room ${roomId}`); - - // Update notification status to processing - await this.updateNotificationStatus(notificationId, 'processing'); - - const result = await this.matrixService.sendMessage({ - roomId, - body, - formattedBody, - msgtype, - }); - - const durationMs = Date.now() - startTime; - - // Log the delivery attempt - await this.logDelivery({ - notificationId, - attemptNumber: job.attemptsMade + 1, - channel: 'matrix', - success: result.success, - errorMessage: result.error, - providerId: result.eventId, - durationMs, - }); - - this.metricsService.recordMatrixSent(result.success); - this.metricsService.recordNotificationLatency('matrix', durationMs / 1000); - - if (result.success) { - this.metricsService.recordNotificationSent('matrix', appId); - await this.updateNotificationStatus(notificationId, 'delivered', result.eventId); - this.logger.log(`Matrix message sent to ${roomId} in ${durationMs}ms`); - } else { - this.metricsService.recordNotificationFailed('matrix', appId, 'send_error'); - // Only mark as failed if no more retries - if (job.attemptsMade >= (job.opts.attempts || 3) - 1) { - await this.updateNotificationStatus(notificationId, 'failed', undefined, result.error); - } - throw new Error(result.error || 'Failed to send Matrix message'); - } - } - - @OnWorkerEvent('failed') - onFailed(job: Job, error: Error) { - this.logger.error(`Matrix job ${job.id} failed: ${error.message}`); - } - - private async updateNotificationStatus( - notificationId: string, - status: string, - providerId?: string, - errorMessage?: string - ): Promise { - try { - const updateData: Record = { - status, - updatedAt: new Date(), - }; - - if (status === 'delivered') { - updateData.deliveredAt = new Date(); - } - - if (errorMessage) { - updateData.errorMessage = errorMessage; - } - - await this.db - .update(notifications) - .set(updateData) - .where(eq(notifications.id, notificationId)); - } catch (error) { - this.logger.error(`Failed to update notification status: ${error}`); - } - } - - private async logDelivery(log: Omit): Promise { - try { - await this.db.insert(deliveryLogs).values(log); - } catch (error) { - this.logger.error(`Failed to log delivery: ${error}`); - } - } -} diff --git a/services/mana-notify/src/queue/processors/push.processor.ts b/services/mana-notify/src/queue/processors/push.processor.ts deleted file mode 100644 index 9ccf2975c..000000000 --- a/services/mana-notify/src/queue/processors/push.processor.ts +++ /dev/null @@ -1,154 +0,0 @@ -import { Processor, WorkerHost, OnWorkerEvent } from '@nestjs/bullmq'; -import { Logger, Inject } from '@nestjs/common'; -import { Job } from 'bullmq'; -import { eq } from 'drizzle-orm'; -import { PUSH_QUEUE } from '../queue-names'; -import { PushService } from '../../channels/push/push.service'; -import { MetricsService } from '../../metrics/metrics.service'; -import { DATABASE_CONNECTION } from '../../db/database.module'; -import { notifications, deliveryLogs, type NewDeliveryLog } from '../../db/schema'; - -export interface PushJob { - notificationId: string; - tokens: string[]; - title: string; - body: string; - data?: Record; - sound?: 'default' | null; - badge?: number; - platform: string; - appId: string; -} - -@Processor(PUSH_QUEUE, { - concurrency: 10, -}) -export class PushProcessor extends WorkerHost { - private readonly logger = new Logger(PushProcessor.name); - - constructor( - private readonly pushService: PushService, - private readonly metricsService: MetricsService, - @Inject(DATABASE_CONNECTION) private readonly db: any - ) { - super(); - } - - async process(job: Job): Promise { - const { notificationId, tokens, title, body, data, sound, badge, platform, appId } = job.data; - const startTime = Date.now(); - - this.logger.debug(`Processing push job ${job.id} to ${tokens.length} tokens`); - - // Update notification status to processing - await this.updateNotificationStatus(notificationId, 'processing'); - - const results = await this.pushService.sendToTokens(tokens, { - title, - body, - data, - sound, - badge, - }); - - const durationMs = Date.now() - startTime; - - // Count successes and failures - let successCount = 0; - let failCount = 0; - const ticketIds: string[] = []; - - for (const [token, result] of results) { - if (result.success) { - successCount++; - if (result.ticketId) { - ticketIds.push(result.ticketId); - } - } else { - failCount++; - } - - // Record per-token metrics - this.metricsService.recordPushSent(platform, result.success); - } - - // Log the delivery attempt - await this.logDelivery({ - notificationId, - attemptNumber: job.attemptsMade + 1, - channel: 'push', - success: successCount > 0, - errorMessage: failCount > 0 ? `${failCount}/${tokens.length} tokens failed` : undefined, - providerId: ticketIds.join(','), - durationMs, - }); - - this.metricsService.recordPushLatency(durationMs / 1000); - - if (successCount > 0) { - this.metricsService.recordNotificationSent('push', appId); - await this.updateNotificationStatus( - notificationId, - failCount === 0 ? 'delivered' : 'delivered', // Partial success still counts as delivered - ticketIds.join(',') - ); - this.logger.log( - `Push notification sent: ${successCount}/${tokens.length} successful in ${durationMs}ms` - ); - } else { - this.metricsService.recordNotificationFailed('push', appId, 'send_error'); - // Only mark as failed if no more retries - if (job.attemptsMade >= (job.opts.attempts || 3) - 1) { - await this.updateNotificationStatus( - notificationId, - 'failed', - undefined, - 'All tokens failed' - ); - } - throw new Error('All push tokens failed'); - } - } - - @OnWorkerEvent('failed') - onFailed(job: Job, error: Error) { - this.logger.error(`Push job ${job.id} failed: ${error.message}`); - } - - private async updateNotificationStatus( - notificationId: string, - status: string, - providerId?: string, - errorMessage?: string - ): Promise { - try { - const updateData: Record = { - status, - updatedAt: new Date(), - }; - - if (status === 'delivered') { - updateData.deliveredAt = new Date(); - } - - if (errorMessage) { - updateData.errorMessage = errorMessage; - } - - await this.db - .update(notifications) - .set(updateData) - .where(eq(notifications.id, notificationId)); - } catch (error) { - this.logger.error(`Failed to update notification status: ${error}`); - } - } - - private async logDelivery(log: Omit): Promise { - try { - await this.db.insert(deliveryLogs).values(log); - } catch (error) { - this.logger.error(`Failed to log delivery: ${error}`); - } - } -} diff --git a/services/mana-notify/src/queue/processors/webhook.processor.ts b/services/mana-notify/src/queue/processors/webhook.processor.ts deleted file mode 100644 index 7b1a6ded0..000000000 --- a/services/mana-notify/src/queue/processors/webhook.processor.ts +++ /dev/null @@ -1,123 +0,0 @@ -import { Processor, WorkerHost, OnWorkerEvent } from '@nestjs/bullmq'; -import { Logger, Inject } from '@nestjs/common'; -import { Job } from 'bullmq'; -import { eq } from 'drizzle-orm'; -import { WEBHOOK_QUEUE } from '../queue-names'; -import { WebhookService } from '../../channels/webhook/webhook.service'; -import { MetricsService } from '../../metrics/metrics.service'; -import { DATABASE_CONNECTION } from '../../db/database.module'; -import { notifications, deliveryLogs, type NewDeliveryLog } from '../../db/schema'; - -export interface WebhookJob { - notificationId: string; - url: string; - method?: 'POST' | 'PUT'; - headers?: Record; - body: Record; - timeout?: number; - appId: string; -} - -@Processor(WEBHOOK_QUEUE, { - concurrency: 10, -}) -export class WebhookProcessor extends WorkerHost { - private readonly logger = new Logger(WebhookProcessor.name); - - constructor( - private readonly webhookService: WebhookService, - private readonly metricsService: MetricsService, - @Inject(DATABASE_CONNECTION) private readonly db: any - ) { - super(); - } - - async process(job: Job): Promise { - const { notificationId, url, method, headers, body, timeout, appId } = job.data; - const startTime = Date.now(); - - this.logger.debug(`Processing webhook job ${job.id} to ${url}`); - - // Update notification status to processing - await this.updateNotificationStatus(notificationId, 'processing'); - - const result = await this.webhookService.send({ - url, - method, - headers, - body, - timeout, - }); - - const durationMs = Date.now() - startTime; - - // Log the delivery attempt - await this.logDelivery({ - notificationId, - attemptNumber: job.attemptsMade + 1, - channel: 'webhook', - success: result.success, - statusCode: result.statusCode, - errorMessage: result.error, - durationMs: result.durationMs, - }); - - this.metricsService.recordWebhookSent(result.success); - this.metricsService.recordNotificationLatency('webhook', durationMs / 1000); - - if (result.success) { - this.metricsService.recordNotificationSent('webhook', appId); - await this.updateNotificationStatus(notificationId, 'delivered'); - this.logger.log(`Webhook sent to ${url} in ${durationMs}ms`); - } else { - this.metricsService.recordNotificationFailed('webhook', appId, 'send_error'); - // Only mark as failed if no more retries - if (job.attemptsMade >= (job.opts.attempts || 5) - 1) { - await this.updateNotificationStatus(notificationId, 'failed', undefined, result.error); - } - throw new Error(result.error || 'Failed to send webhook'); - } - } - - @OnWorkerEvent('failed') - onFailed(job: Job, error: Error) { - this.logger.error(`Webhook job ${job.id} failed: ${error.message}`); - } - - private async updateNotificationStatus( - notificationId: string, - status: string, - providerId?: string, - errorMessage?: string - ): Promise { - try { - const updateData: Record = { - status, - updatedAt: new Date(), - }; - - if (status === 'delivered') { - updateData.deliveredAt = new Date(); - } - - if (errorMessage) { - updateData.errorMessage = errorMessage; - } - - await this.db - .update(notifications) - .set(updateData) - .where(eq(notifications.id, notificationId)); - } catch (error) { - this.logger.error(`Failed to update notification status: ${error}`); - } - } - - private async logDelivery(log: Omit): Promise { - try { - await this.db.insert(deliveryLogs).values(log); - } catch (error) { - this.logger.error(`Failed to log delivery: ${error}`); - } - } -} diff --git a/services/mana-notify/src/queue/queue-names.ts b/services/mana-notify/src/queue/queue-names.ts deleted file mode 100644 index 8c54ab6ae..000000000 --- a/services/mana-notify/src/queue/queue-names.ts +++ /dev/null @@ -1,5 +0,0 @@ -// Queue names - separate file to avoid circular imports with processors -export const EMAIL_QUEUE = 'email'; -export const PUSH_QUEUE = 'push'; -export const MATRIX_QUEUE = 'matrix'; -export const WEBHOOK_QUEUE = 'webhook'; diff --git a/services/mana-notify/src/queue/queue.module.ts b/services/mana-notify/src/queue/queue.module.ts deleted file mode 100644 index c2d0fb9cc..000000000 --- a/services/mana-notify/src/queue/queue.module.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { Module } from '@nestjs/common'; -import { BullModule } from '@nestjs/bullmq'; -import { EmailProcessor } from './processors/email.processor'; -import { PushProcessor } from './processors/push.processor'; -import { MatrixProcessor } from './processors/matrix.processor'; -import { WebhookProcessor } from './processors/webhook.processor'; -import { ChannelsModule } from '../channels/channels.module'; -import { MetricsModule } from '../metrics/metrics.module'; -import { EMAIL_QUEUE, PUSH_QUEUE, MATRIX_QUEUE, WEBHOOK_QUEUE } from './queue-names'; - -// Re-export for convenience -export { EMAIL_QUEUE, PUSH_QUEUE, MATRIX_QUEUE, WEBHOOK_QUEUE } from './queue-names'; - -@Module({ - imports: [ - BullModule.registerQueue( - { - name: EMAIL_QUEUE, - defaultJobOptions: { - attempts: 3, - backoff: { - type: 'exponential', - delay: 5000, - }, - removeOnComplete: 100, - removeOnFail: 1000, - }, - }, - { - name: PUSH_QUEUE, - defaultJobOptions: { - attempts: 3, - backoff: { - type: 'exponential', - delay: 1000, - }, - removeOnComplete: 100, - removeOnFail: 1000, - }, - }, - { - name: MATRIX_QUEUE, - defaultJobOptions: { - attempts: 3, - backoff: { - type: 'exponential', - delay: 2000, - }, - removeOnComplete: 100, - removeOnFail: 500, - }, - }, - { - name: WEBHOOK_QUEUE, - defaultJobOptions: { - attempts: 5, - backoff: { - type: 'exponential', - delay: 3000, - }, - removeOnComplete: 100, - removeOnFail: 1000, - }, - } - ), - ChannelsModule, - MetricsModule, - ], - providers: [EmailProcessor, PushProcessor, MatrixProcessor, WebhookProcessor], - exports: [BullModule], -}) -export class QueueModule {} diff --git a/services/mana-notify/src/templates/defaults/password-reset.hbs b/services/mana-notify/src/templates/defaults/password-reset.hbs deleted file mode 100644 index 18627c841..000000000 --- a/services/mana-notify/src/templates/defaults/password-reset.hbs +++ /dev/null @@ -1,30 +0,0 @@ - - - - - - - -
-

ManaCore

-
- -

Hallo {{userName}},

- -

Du hast eine Anfrage zum Zurücksetzen deines Passworts gestellt. Klicke auf den Button unten, um ein neues Passwort zu erstellen:

- - - -

Dieser Link ist 1 Stunde gültig. Falls du diese Anfrage nicht gestellt hast, kannst du diese E-Mail ignorieren.

- -
- -

- Diese E-Mail wurde automatisch von ManaCore gesendet.
- Falls der Button nicht funktioniert, kopiere diesen Link in deinen Browser:
- {{resetUrl}} -

- - diff --git a/services/mana-notify/src/templates/defaults/reminder.hbs b/services/mana-notify/src/templates/defaults/reminder.hbs deleted file mode 100644 index bce0e0531..000000000 --- a/services/mana-notify/src/templates/defaults/reminder.hbs +++ /dev/null @@ -1,37 +0,0 @@ - - - - - - - -
-

ManaCore

-
- -
-

{{eventTitle}}

- -

- Wann: {{eventTime}} -

- - {{#if eventLocation}} -

- Wo: {{eventLocation}} -

- {{/if}} -
- - - -
- -

- Diese Erinnerung wurde automatisch von ManaCore gesendet.
- Du kannst Erinnerungen in den Kalender-Einstellungen verwalten. -

- - diff --git a/services/mana-notify/src/templates/defaults/verification.hbs b/services/mana-notify/src/templates/defaults/verification.hbs deleted file mode 100644 index ef5a3fc5e..000000000 --- a/services/mana-notify/src/templates/defaults/verification.hbs +++ /dev/null @@ -1,30 +0,0 @@ - - - - - - - -
-

ManaCore

-
- -

Hallo {{userName}},

- -

Willkommen bei ManaCore! Bitte bestätige deine E-Mail-Adresse, um deinen Account zu aktivieren:

- - - -

Dieser Link ist 24 Stunden gültig. Falls du dich nicht bei ManaCore registriert hast, kannst du diese E-Mail ignorieren.

- -
- -

- Diese E-Mail wurde automatisch von ManaCore gesendet.
- Falls der Button nicht funktioniert, kopiere diesen Link in deinen Browser:
- {{verificationUrl}} -

- - diff --git a/services/mana-notify/src/templates/defaults/welcome.hbs b/services/mana-notify/src/templates/defaults/welcome.hbs deleted file mode 100644 index 39ced4471..000000000 --- a/services/mana-notify/src/templates/defaults/welcome.hbs +++ /dev/null @@ -1,30 +0,0 @@ - - - - - - - -
-

ManaCore

-
- -

Hallo {{userName}},

- -

Willkommen bei ManaCore! Dein Account wurde erfolgreich erstellt.

- -

Du kannst dich jetzt mit deiner E-Mail-Adresse und deinem Passwort anmelden und alle Features nutzen:

- - - -

Bei Fragen oder Problemen kannst du uns jederzeit kontaktieren.

- -
- -

- Diese E-Mail wurde automatisch von ManaCore gesendet. -

- - diff --git a/services/mana-notify/src/templates/templates.controller.ts b/services/mana-notify/src/templates/templates.controller.ts deleted file mode 100644 index d15f78d1c..000000000 --- a/services/mana-notify/src/templates/templates.controller.ts +++ /dev/null @@ -1,117 +0,0 @@ -import { - Controller, - Get, - Post, - Put, - Delete, - Body, - Param, - Query, - UseGuards, - HttpCode, - HttpStatus, -} from '@nestjs/common'; -import { TemplatesService, RenderedTemplate } from './templates.service'; -import { ServiceAuthGuard } from '../common/guards/service-auth.guard'; -import { Template } from '../db/schema'; - -class CreateTemplateDto { - slug!: string; - channel!: 'email' | 'push' | 'matrix' | 'webhook'; - subject?: string; - bodyTemplate!: string; - locale?: string; - appId?: string; - variables?: Record; -} - -class UpdateTemplateDto { - subject?: string; - bodyTemplate?: string; - isActive?: boolean; - variables?: Record; -} - -class PreviewTemplateDto { - subject?: string; - bodyTemplate!: string; - data!: Record; -} - -@Controller('templates') -@UseGuards(ServiceAuthGuard) -export class TemplatesController { - constructor(private readonly templatesService: TemplatesService) {} - - @Get() - async listTemplates(@Query('appId') appId?: string): Promise<{ templates: Template[] }> { - const templatesList = await this.templatesService.listTemplates(appId); - return { templates: templatesList }; - } - - @Get(':slug') - async getTemplate( - @Param('slug') slug: string, - @Query('locale') locale: string = 'de-DE' - ): Promise<{ template: Template | null }> { - const template = await this.templatesService.getTemplate(slug, locale); - return { template }; - } - - @Post() - async createTemplate(@Body() dto: CreateTemplateDto): Promise<{ template: Template }> { - const template = await this.templatesService.createTemplate({ - slug: dto.slug, - channel: dto.channel, - subject: dto.subject, - bodyTemplate: dto.bodyTemplate, - locale: dto.locale || 'de-DE', - appId: dto.appId, - variables: dto.variables, - isActive: true, - isSystem: false, - }); - return { template }; - } - - @Put(':slug') - async updateTemplate( - @Param('slug') slug: string, - @Query('locale') locale: string = 'de-DE', - @Body() dto: UpdateTemplateDto - ): Promise<{ template: Template }> { - const template = await this.templatesService.updateTemplate(slug, locale, dto); - return { template }; - } - - @Delete(':slug') - @HttpCode(HttpStatus.NO_CONTENT) - async deleteTemplate( - @Param('slug') slug: string, - @Query('locale') locale: string = 'de-DE' - ): Promise { - await this.templatesService.deleteTemplate(slug, locale); - } - - @Post(':slug/preview') - async previewTemplate( - @Param('slug') slug: string, - @Query('locale') locale: string = 'de-DE', - @Body() dto: { data: Record } - ): Promise<{ preview: RenderedTemplate | null }> { - const preview = await this.templatesService.renderBySlug(slug, dto.data, locale); - return { preview }; - } - - @Post('preview') - async previewCustomTemplate( - @Body() dto: PreviewTemplateDto - ): Promise<{ preview: RenderedTemplate }> { - const preview = this.templatesService.previewTemplate( - dto.subject || null, - dto.bodyTemplate, - dto.data - ); - return { preview }; - } -} diff --git a/services/mana-notify/src/templates/templates.module.ts b/services/mana-notify/src/templates/templates.module.ts deleted file mode 100644 index 78f341c7f..000000000 --- a/services/mana-notify/src/templates/templates.module.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { Module } from '@nestjs/common'; -import { TemplatesService } from './templates.service'; -import { TemplatesController } from './templates.controller'; - -@Module({ - providers: [TemplatesService], - controllers: [TemplatesController], - exports: [TemplatesService], -}) -export class TemplatesModule {} diff --git a/services/mana-notify/src/templates/templates.service.ts b/services/mana-notify/src/templates/templates.service.ts deleted file mode 100644 index 739be91a9..000000000 --- a/services/mana-notify/src/templates/templates.service.ts +++ /dev/null @@ -1,234 +0,0 @@ -import { Injectable, Logger, Inject, OnModuleInit, NotFoundException } from '@nestjs/common'; -import { eq, and } from 'drizzle-orm'; -import * as Handlebars from 'handlebars'; -import * as fs from 'fs'; -import * as path from 'path'; -import { DATABASE_CONNECTION } from '../db/database.module'; -import { templates, type Template, type NewTemplate } from '../db/schema'; - -export interface RenderedTemplate { - subject: string; - body: string; -} - -interface DefaultTemplate { - slug: string; - channel: 'email' | 'push' | 'matrix' | 'webhook'; - subject?: string; - bodyFile: string; - variables: Record; -} - -@Injectable() -export class TemplatesService implements OnModuleInit { - private readonly logger = new Logger(TemplatesService.name); - private readonly defaultTemplates: DefaultTemplate[] = [ - { - slug: 'auth-password-reset', - channel: 'email', - subject: 'Passwort zurücksetzen - ManaCore', - bodyFile: 'password-reset.hbs', - variables: { resetUrl: 'URL to reset password', userName: 'User display name' }, - }, - { - slug: 'auth-verification', - channel: 'email', - subject: 'E-Mail bestätigen - ManaCore', - bodyFile: 'verification.hbs', - variables: { verificationUrl: 'URL to verify email', userName: 'User display name' }, - }, - { - slug: 'auth-welcome', - channel: 'email', - subject: 'Willkommen bei ManaCore!', - bodyFile: 'welcome.hbs', - variables: { userName: 'User display name', loginUrl: 'URL to login' }, - }, - { - slug: 'calendar-reminder', - channel: 'email', - subject: 'Erinnerung: {{eventTitle}}', - bodyFile: 'reminder.hbs', - variables: { - eventTitle: 'Event title', - eventTime: 'Event start time', - eventLocation: 'Event location', - eventUrl: 'URL to view event', - }, - }, - ]; - - constructor(@Inject(DATABASE_CONNECTION) private readonly db: any) {} - - async onModuleInit() { - await this.seedDefaultTemplates(); - } - - private async seedDefaultTemplates(): Promise { - this.logger.log('Checking default templates...'); - - for (const defaultTemplate of this.defaultTemplates) { - const existing = await this.db - .select() - .from(templates) - .where(and(eq(templates.slug, defaultTemplate.slug), eq(templates.locale, 'de-DE'))) - .limit(1); - - if (existing.length === 0) { - this.logger.log(`Creating default template: ${defaultTemplate.slug}`); - - const bodyTemplate = this.loadDefaultTemplate(defaultTemplate.bodyFile); - - await this.db.insert(templates).values({ - slug: defaultTemplate.slug, - channel: defaultTemplate.channel, - subject: defaultTemplate.subject, - bodyTemplate, - locale: 'de-DE', - isActive: true, - isSystem: true, - variables: defaultTemplate.variables, - }); - } - } - - this.logger.log('Default templates initialized'); - } - - private loadDefaultTemplate(filename: string): string { - // Try multiple paths for flexibility - const paths = [ - path.join(__dirname, 'defaults', filename), - path.join(process.cwd(), 'src', 'templates', 'defaults', filename), - path.join(process.cwd(), 'dist', 'templates', 'defaults', filename), - ]; - - for (const templatePath of paths) { - try { - if (fs.existsSync(templatePath)) { - return fs.readFileSync(templatePath, 'utf-8'); - } - } catch { - // Continue to next path - } - } - - this.logger.warn(`Default template not found: ${filename}, using placeholder`); - return `

Template content for ${filename}

`; - } - - async getTemplate(slug: string, locale: string = 'de-DE'): Promise