mirror of
https://github.com/Memo-2023/mana-monorepo.git
synced 2026-05-15 22:39:40 +02:00
NestJS-based web crawler service for structured content extraction. Features: - Depth-controlled crawling with URL pattern filtering - robots.txt compliance - HTML/PDF/Markdown content extraction - BullMQ job queue for async processing - Redis caching layer - Prometheus metrics
47 lines
1.4 KiB
TypeScript
47 lines
1.4 KiB
TypeScript
import { Module } from '@nestjs/common';
|
|
import { ConfigModule } from '@nestjs/config';
|
|
import { BullModule } from '@nestjs/bullmq';
|
|
import { BullBoardModule } from '@bull-board/nestjs';
|
|
import { ExpressAdapter } from '@bull-board/express';
|
|
import configuration from './config/configuration';
|
|
import { DatabaseModule } from './db/database.module';
|
|
import { HealthModule } from './health/health.module';
|
|
import { MetricsModule } from './metrics/metrics.module';
|
|
import { CacheModule } from './cache/cache.module';
|
|
import { CrawlerModule } from './crawler/crawler.module';
|
|
import { QueueModule } from './queue/queue.module';
|
|
import { ProcessorModule } from './queue/processor.module';
|
|
import { ParserModule } from './parser/parser.module';
|
|
import { RobotsModule } from './robots/robots.module';
|
|
|
|
@Module({
|
|
imports: [
|
|
ConfigModule.forRoot({
|
|
isGlobal: true,
|
|
load: [configuration],
|
|
}),
|
|
BullModule.forRootAsync({
|
|
useFactory: () => ({
|
|
connection: {
|
|
host: process.env.REDIS_HOST || 'localhost',
|
|
port: parseInt(process.env.REDIS_PORT || '6379', 10),
|
|
password: process.env.REDIS_PASSWORD || undefined,
|
|
},
|
|
}),
|
|
}),
|
|
BullBoardModule.forRoot({
|
|
route: '/queue/dashboard',
|
|
adapter: ExpressAdapter,
|
|
}),
|
|
DatabaseModule,
|
|
HealthModule,
|
|
MetricsModule,
|
|
CacheModule,
|
|
RobotsModule,
|
|
ParserModule,
|
|
QueueModule,
|
|
ProcessorModule,
|
|
CrawlerModule,
|
|
],
|
|
})
|
|
export class AppModule {}
|