feat(matrix-mana-bot): add voice input support (Phase 1)

- Add VoiceModule and VoiceService for STT integration
- Override handleAudioMessage to process voice notes
- Transcribe audio via mana-stt (Whisper)
- Route transcribed text through CommandRouter
- Add voice configuration and environment variables
- Update help text and documentation

Voice flow:
1. User sends voice note
2. Bot downloads and transcribes audio
3. Shows transcription: 🎤 *"text"*
4. Routes as normal text command
5. Returns text response

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Till-JS 2026-02-01 02:59:13 +01:00
parent f04c27fe26
commit db07b5613d
9 changed files with 833 additions and 13 deletions

View file

@ -3,9 +3,10 @@ import { MatrixService } from './matrix.service';
import { CommandRouterService } from './command-router.service';
import { HandlersModule } from '../handlers/handlers.module';
import { OrchestrationModule } from '../orchestration/orchestration.module';
import { VoiceModule } from '../voice/voice.module';
@Module({
imports: [forwardRef(() => HandlersModule), forwardRef(() => OrchestrationModule)],
imports: [forwardRef(() => HandlersModule), forwardRef(() => OrchestrationModule), VoiceModule],
providers: [MatrixService, CommandRouterService],
exports: [MatrixService, CommandRouterService],
})

View file

@ -11,6 +11,7 @@ export interface CommandContext {
userId: string;
message: string;
event: any;
isVoice?: boolean; // True if message came from voice input
}
interface CommandRoute {
@ -23,7 +24,10 @@ interface CommandRoute {
const KEYWORD_COMMANDS: { keywords: string[]; command: string }[] = [
{ keywords: ['hilfe', 'help', 'was kannst du', 'befehle'], command: '!help' },
{ keywords: ['modelle', 'models', 'welche modelle'], command: '!models' },
{ keywords: ['meine aufgaben', 'zeige aufgaben', 'todo liste', 'was muss ich'], command: '!list' },
{
keywords: ['meine aufgaben', 'zeige aufgaben', 'todo liste', 'was muss ich'],
command: '!list',
},
{ keywords: ['heute', 'was steht heute an'], command: '!today' },
{ keywords: ['termine', 'kalender', 'meine termine'], command: '!cal' },
{ keywords: ['timer', 'stoppuhr'], command: '!timers' },
@ -97,7 +101,7 @@ export class CommandRouterService {
{
patterns: ['!today', '!heute'],
handler: (ctx) => this.todoHandler.today(ctx),
description: 'Today\'s todos',
description: "Today's todos",
},
{
patterns: ['!inbox'],
@ -124,7 +128,7 @@ export class CommandRouterService {
{
patterns: ['!cal', '!termine'],
handler: (ctx) => this.calendarHandler.today(ctx),
description: 'Today\'s events',
description: "Today's events",
},
{
patterns: ['!week', '!woche'],

View file

@ -1,28 +1,32 @@
import { Injectable, Inject, forwardRef } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import {
BaseMatrixService,
MatrixBotConfig,
MatrixRoomEvent,
} from '@manacore/matrix-bot-common';
import { BaseMatrixService, MatrixBotConfig, MatrixRoomEvent } from '@manacore/matrix-bot-common';
import { CommandRouterService, CommandContext } from './command-router.service';
import { VoiceService } from '../voice/voice.service';
import { HELP_TEXT, WELCOME_TEXT, BOT_INTRODUCTION } from '../config/configuration';
@Injectable()
export class MatrixService extends BaseMatrixService {
private voiceEnabled: boolean;
constructor(
configService: ConfigService,
@Inject(forwardRef(() => CommandRouterService))
private commandRouter: CommandRouterService
private commandRouter: CommandRouterService,
@Inject(forwardRef(() => VoiceService))
private voiceService: VoiceService
) {
super(configService);
this.voiceEnabled = configService.get('voice.enabled') !== false;
}
protected getConfig(): MatrixBotConfig {
return {
homeserverUrl: this.configService.get<string>('matrix.homeserverUrl') || 'http://localhost:8008',
homeserverUrl:
this.configService.get<string>('matrix.homeserverUrl') || 'http://localhost:8008',
accessToken: this.configService.get<string>('matrix.accessToken') || '',
storagePath: this.configService.get<string>('matrix.storagePath') || './data/mana-bot-storage.json',
storagePath:
this.configService.get<string>('matrix.storagePath') || './data/mana-bot-storage.json',
allowedRooms: this.configService.get<string[]>('matrix.allowedRooms') || [],
};
}
@ -99,6 +103,81 @@ export class MatrixService extends BaseMatrixService {
}
}
/**
* Handle voice note messages - transcribe and process as text
*/
protected async handleAudioMessage(
roomId: string,
event: MatrixRoomEvent,
sender: string
): Promise<void> {
if (!this.voiceEnabled) {
return;
}
const audioUrl = event.content?.url;
if (!audioUrl) {
this.logger.warn('Audio message without URL');
return;
}
try {
// Set typing indicator
await this.client.setTyping(roomId, true, 60000);
// Download audio from Matrix
this.logger.debug(`Downloading audio from ${audioUrl}`);
const audioBuffer = await this.downloadMedia(audioUrl);
// Transcribe audio
this.logger.debug(`Transcribing ${audioBuffer.length} bytes`);
const transcription = await this.voiceService.transcribe(audioBuffer);
if (!transcription.text || transcription.text.trim() === '') {
await this.client.setTyping(roomId, false);
await this.sendReply(
roomId,
event,
'🎤 Ich konnte leider nichts verstehen. Bitte versuche es noch einmal.'
);
return;
}
const message = transcription.text.trim();
this.logger.log(`Transcribed from ${sender}: "${message}"`);
// Show what was understood
await this.sendReply(roomId, event, `🎤 *"${message}"*`);
// Create context and route
const ctx: CommandContext = {
roomId,
userId: sender,
message,
event,
isVoice: true, // Flag for voice input
};
// Route the transcribed message
const response = await this.commandRouter.route(ctx);
// Stop typing
await this.client.setTyping(roomId, false);
if (response) {
await this.sendReply(roomId, event, response);
}
} catch (error) {
await this.client.setTyping(roomId, false);
this.logger.error(`Error handling voice message:`, error);
await this.sendReply(
roomId,
event,
'❌ Spracherkennung fehlgeschlagen. Bitte versuche es noch einmal.'
);
}
}
private async sendWelcomeMessage(roomId: string, userId: string) {
try {
await this.sendMessage(roomId, WELCOME_TEXT);