mirror of
https://github.com/Memo-2023/mana-monorepo.git
synced 2026-05-17 09:19:39 +02:00
4a3295d1d0
NestJS-based web crawler service for structured content extraction. Features: - Depth-controlled crawling with URL pattern filtering - robots.txt compliance - HTML/PDF/Markdown content extraction - BullMQ job queue for async processing - Redis caching layer - Prometheus metrics
10 lines
268 B
TypeScript
10 lines
268 B
TypeScript
import { Module } from '@nestjs/common';
|
|
import { RobotsService } from './robots.service';
|
|
import { CacheModule } from '../cache/cache.module';
|
|
|
|
@Module({
|
|
imports: [CacheModule],
|
|
providers: [RobotsService],
|
|
exports: [RobotsService],
|
|
})
|
|
export class RobotsModule {}
|