mirror of
https://github.com/Memo-2023/mana-monorepo.git
synced 2026-05-22 23:06:41 +02:00
feat(matrix-mana-bot): add voice input support (Phase 1)
- Add VoiceModule and VoiceService for STT integration
- Override handleAudioMessage to process voice notes
- Transcribe audio via mana-stt (Whisper)
- Route transcribed text through CommandRouter
- Add voice configuration and environment variables
- Update help text and documentation
Voice flow:
1. User sends voice note
2. Bot downloads and transcribes audio
3. Shows transcription: 🎤 *"text"*
4. Routes as normal text command
5. Returns text response
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
parent
f04c27fe26
commit
db07b5613d
9 changed files with 833 additions and 13 deletions
|
|
@ -3,9 +3,10 @@ import { MatrixService } from './matrix.service';
|
|||
import { CommandRouterService } from './command-router.service';
|
||||
import { HandlersModule } from '../handlers/handlers.module';
|
||||
import { OrchestrationModule } from '../orchestration/orchestration.module';
|
||||
import { VoiceModule } from '../voice/voice.module';
|
||||
|
||||
@Module({
|
||||
imports: [forwardRef(() => HandlersModule), forwardRef(() => OrchestrationModule)],
|
||||
imports: [forwardRef(() => HandlersModule), forwardRef(() => OrchestrationModule), VoiceModule],
|
||||
providers: [MatrixService, CommandRouterService],
|
||||
exports: [MatrixService, CommandRouterService],
|
||||
})
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ export interface CommandContext {
|
|||
userId: string;
|
||||
message: string;
|
||||
event: any;
|
||||
isVoice?: boolean; // True if message came from voice input
|
||||
}
|
||||
|
||||
interface CommandRoute {
|
||||
|
|
@ -23,7 +24,10 @@ interface CommandRoute {
|
|||
const KEYWORD_COMMANDS: { keywords: string[]; command: string }[] = [
|
||||
{ keywords: ['hilfe', 'help', 'was kannst du', 'befehle'], command: '!help' },
|
||||
{ keywords: ['modelle', 'models', 'welche modelle'], command: '!models' },
|
||||
{ keywords: ['meine aufgaben', 'zeige aufgaben', 'todo liste', 'was muss ich'], command: '!list' },
|
||||
{
|
||||
keywords: ['meine aufgaben', 'zeige aufgaben', 'todo liste', 'was muss ich'],
|
||||
command: '!list',
|
||||
},
|
||||
{ keywords: ['heute', 'was steht heute an'], command: '!today' },
|
||||
{ keywords: ['termine', 'kalender', 'meine termine'], command: '!cal' },
|
||||
{ keywords: ['timer', 'stoppuhr'], command: '!timers' },
|
||||
|
|
@ -97,7 +101,7 @@ export class CommandRouterService {
|
|||
{
|
||||
patterns: ['!today', '!heute'],
|
||||
handler: (ctx) => this.todoHandler.today(ctx),
|
||||
description: 'Today\'s todos',
|
||||
description: "Today's todos",
|
||||
},
|
||||
{
|
||||
patterns: ['!inbox'],
|
||||
|
|
@ -124,7 +128,7 @@ export class CommandRouterService {
|
|||
{
|
||||
patterns: ['!cal', '!termine'],
|
||||
handler: (ctx) => this.calendarHandler.today(ctx),
|
||||
description: 'Today\'s events',
|
||||
description: "Today's events",
|
||||
},
|
||||
{
|
||||
patterns: ['!week', '!woche'],
|
||||
|
|
|
|||
|
|
@ -1,28 +1,32 @@
|
|||
import { Injectable, Inject, forwardRef } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import {
|
||||
BaseMatrixService,
|
||||
MatrixBotConfig,
|
||||
MatrixRoomEvent,
|
||||
} from '@manacore/matrix-bot-common';
|
||||
import { BaseMatrixService, MatrixBotConfig, MatrixRoomEvent } from '@manacore/matrix-bot-common';
|
||||
import { CommandRouterService, CommandContext } from './command-router.service';
|
||||
import { VoiceService } from '../voice/voice.service';
|
||||
import { HELP_TEXT, WELCOME_TEXT, BOT_INTRODUCTION } from '../config/configuration';
|
||||
|
||||
@Injectable()
|
||||
export class MatrixService extends BaseMatrixService {
|
||||
private voiceEnabled: boolean;
|
||||
|
||||
constructor(
|
||||
configService: ConfigService,
|
||||
@Inject(forwardRef(() => CommandRouterService))
|
||||
private commandRouter: CommandRouterService
|
||||
private commandRouter: CommandRouterService,
|
||||
@Inject(forwardRef(() => VoiceService))
|
||||
private voiceService: VoiceService
|
||||
) {
|
||||
super(configService);
|
||||
this.voiceEnabled = configService.get('voice.enabled') !== false;
|
||||
}
|
||||
|
||||
protected getConfig(): MatrixBotConfig {
|
||||
return {
|
||||
homeserverUrl: this.configService.get<string>('matrix.homeserverUrl') || 'http://localhost:8008',
|
||||
homeserverUrl:
|
||||
this.configService.get<string>('matrix.homeserverUrl') || 'http://localhost:8008',
|
||||
accessToken: this.configService.get<string>('matrix.accessToken') || '',
|
||||
storagePath: this.configService.get<string>('matrix.storagePath') || './data/mana-bot-storage.json',
|
||||
storagePath:
|
||||
this.configService.get<string>('matrix.storagePath') || './data/mana-bot-storage.json',
|
||||
allowedRooms: this.configService.get<string[]>('matrix.allowedRooms') || [],
|
||||
};
|
||||
}
|
||||
|
|
@ -99,6 +103,81 @@ export class MatrixService extends BaseMatrixService {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle voice note messages - transcribe and process as text
|
||||
*/
|
||||
protected async handleAudioMessage(
|
||||
roomId: string,
|
||||
event: MatrixRoomEvent,
|
||||
sender: string
|
||||
): Promise<void> {
|
||||
if (!this.voiceEnabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
const audioUrl = event.content?.url;
|
||||
if (!audioUrl) {
|
||||
this.logger.warn('Audio message without URL');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// Set typing indicator
|
||||
await this.client.setTyping(roomId, true, 60000);
|
||||
|
||||
// Download audio from Matrix
|
||||
this.logger.debug(`Downloading audio from ${audioUrl}`);
|
||||
const audioBuffer = await this.downloadMedia(audioUrl);
|
||||
|
||||
// Transcribe audio
|
||||
this.logger.debug(`Transcribing ${audioBuffer.length} bytes`);
|
||||
const transcription = await this.voiceService.transcribe(audioBuffer);
|
||||
|
||||
if (!transcription.text || transcription.text.trim() === '') {
|
||||
await this.client.setTyping(roomId, false);
|
||||
await this.sendReply(
|
||||
roomId,
|
||||
event,
|
||||
'🎤 Ich konnte leider nichts verstehen. Bitte versuche es noch einmal.'
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const message = transcription.text.trim();
|
||||
this.logger.log(`Transcribed from ${sender}: "${message}"`);
|
||||
|
||||
// Show what was understood
|
||||
await this.sendReply(roomId, event, `🎤 *"${message}"*`);
|
||||
|
||||
// Create context and route
|
||||
const ctx: CommandContext = {
|
||||
roomId,
|
||||
userId: sender,
|
||||
message,
|
||||
event,
|
||||
isVoice: true, // Flag for voice input
|
||||
};
|
||||
|
||||
// Route the transcribed message
|
||||
const response = await this.commandRouter.route(ctx);
|
||||
|
||||
// Stop typing
|
||||
await this.client.setTyping(roomId, false);
|
||||
|
||||
if (response) {
|
||||
await this.sendReply(roomId, event, response);
|
||||
}
|
||||
} catch (error) {
|
||||
await this.client.setTyping(roomId, false);
|
||||
this.logger.error(`Error handling voice message:`, error);
|
||||
await this.sendReply(
|
||||
roomId,
|
||||
event,
|
||||
'❌ Spracherkennung fehlgeschlagen. Bitte versuche es noch einmal.'
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async sendWelcomeMessage(roomId: string, userId: string) {
|
||||
try {
|
||||
await this.sendMessage(roomId, WELCOME_TEXT);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue