feat(search): implement mana-search microservice

Central search microservice for all ManaCore apps featuring:

- NestJS API on port 3021
- SearXNG meta-search engine integration (40+ search engines)
- Redis caching layer for search results and extracted content
- Content extraction with markdown conversion
- Prometheus metrics for monitoring

API Endpoints:
- POST /api/v1/search - Web search with categories/engines
- POST /api/v1/extract - Content extraction from URLs
- POST /api/v1/extract/bulk - Bulk extraction
- GET /health - Health check
- GET /metrics - Prometheus metrics

Search categories: general, news, science, it, images, videos
Supported engines: Google, Bing, DuckDuckGo, Wikipedia, arXiv,
GitHub, StackOverflow, and many more.

https://claude.ai/code/session_01Rk3YVJCU3nM8uvVPghRz6r
This commit is contained in:
Claude 2026-01-28 20:41:59 +00:00
parent 590529c547
commit bd72b4d6d5
No known key found for this signature in database
35 changed files with 2219 additions and 0 deletions

View file

@ -0,0 +1,9 @@
import { Module, Global } from '@nestjs/common';
import { CacheService } from './cache.service';
@Global()
@Module({
providers: [CacheService],
exports: [CacheService],
})
export class CacheModule {}

View file

@ -0,0 +1,150 @@
import { Injectable, Logger, OnModuleInit, OnModuleDestroy } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import Redis from 'ioredis';
import { MetricsService } from '../metrics/metrics.service';
@Injectable()
export class CacheService implements OnModuleInit, OnModuleDestroy {
private readonly logger = new Logger(CacheService.name);
private client: Redis | null = null;
private readonly keyPrefix: string;
private stats = {
hits: 0,
misses: 0,
};
constructor(
private readonly configService: ConfigService,
private readonly metricsService: MetricsService,
) {
this.keyPrefix = this.configService.get<string>('redis.keyPrefix', 'mana-search:');
}
async onModuleInit() {
const host = this.configService.get<string>('redis.host', 'localhost');
const port = this.configService.get<number>('redis.port', 6379);
const password = this.configService.get<string>('redis.password');
try {
this.client = new Redis({
host,
port,
password,
retryStrategy: (times) => {
if (times > 3) {
this.logger.warn('Redis connection failed, running without cache');
return null; // Stop retrying
}
return Math.min(times * 200, 2000);
},
maxRetriesPerRequest: 1,
});
this.client.on('error', (err) => {
this.logger.error(`Redis error: ${err.message}`);
});
this.client.on('connect', () => {
this.logger.log(`Connected to Redis at ${host}:${port}`);
});
// Test connection
await this.client.ping();
} catch (error) {
this.logger.warn(`Could not connect to Redis: ${error}. Running without cache.`);
this.client = null;
}
}
async onModuleDestroy() {
if (this.client) {
await this.client.quit();
}
}
private buildKey(key: string): string {
return `${this.keyPrefix}${key}`;
}
async get<T>(key: string): Promise<T | null> {
if (!this.client) return null;
try {
const data = await this.client.get(this.buildKey(key));
if (data) {
this.stats.hits++;
this.metricsService.recordCacheHit();
return JSON.parse(data);
}
this.stats.misses++;
this.metricsService.recordCacheMiss();
return null;
} catch (error) {
this.logger.error(`Cache get error: ${error}`);
return null;
}
}
async set(key: string, value: unknown, ttlSeconds: number): Promise<void> {
if (!this.client) return;
try {
await this.client.setex(this.buildKey(key), ttlSeconds, JSON.stringify(value));
} catch (error) {
this.logger.error(`Cache set error: ${error}`);
}
}
async delete(key: string): Promise<void> {
if (!this.client) return;
try {
await this.client.del(this.buildKey(key));
} catch (error) {
this.logger.error(`Cache delete error: ${error}`);
}
}
async clear(): Promise<number> {
if (!this.client) return 0;
try {
const keys = await this.client.keys(`${this.keyPrefix}*`);
if (keys.length > 0) {
await this.client.del(...keys);
}
return keys.length;
} catch (error) {
this.logger.error(`Cache clear error: ${error}`);
return 0;
}
}
getStats() {
const total = this.stats.hits + this.stats.misses;
return {
hits: this.stats.hits,
misses: this.stats.misses,
hitRate: total > 0 ? this.stats.hits / total : 0,
};
}
async healthCheck(): Promise<{ status: string; latency: number }> {
if (!this.client) {
return { status: 'disabled', latency: 0 };
}
const start = Date.now();
try {
await this.client.ping();
return { status: 'ok', latency: Date.now() - start };
} catch {
return { status: 'error', latency: Date.now() - start };
}
}
isConnected(): boolean {
return this.client !== null && this.client.status === 'ready';
}
}