diff --git a/packages/matrix-bot-common/src/base/base-matrix.service.ts b/packages/matrix-bot-common/src/base/base-matrix.service.ts index 77a3bb3e4..dc53133ec 100644 --- a/packages/matrix-bot-common/src/base/base-matrix.service.ts +++ b/packages/matrix-bot-common/src/base/base-matrix.service.ts @@ -7,7 +7,13 @@ import { } from 'matrix-bot-sdk'; import * as path from 'path'; import * as fs from 'fs'; -import { type MatrixBotConfig, type MatrixRoomEvent, isTextMessage, isAudioMessage } from './types'; +import { + type MatrixBotConfig, + type MatrixRoomEvent, + isTextMessage, + isAudioMessage, + isImageMessage, +} from './types'; import { markdownToHtml } from '../markdown/markdown-formatter'; /** @@ -84,6 +90,17 @@ export abstract class BaseMatrixService implements OnModuleInit, OnModuleDestroy // Default: no-op, override in subclass for voice support } + /** + * Handle an image message (optional override) + */ + protected async handleImageMessage( + _roomId: string, + _event: MatrixRoomEvent, + _sender: string + ): Promise { + // Default: no-op, override in subclass for image support + } + /** * Get welcome/introduction message (optional override) */ @@ -201,6 +218,8 @@ export abstract class BaseMatrixService implements OnModuleInit, OnModuleDestroy await this.handleTextMessage(roomId, event, message, event.sender); } else if (isAudioMessage(event)) { await this.handleAudioMessage(roomId, event, event.sender); + } else if (isImageMessage(event)) { + await this.handleImageMessage(roomId, event, event.sender); } } catch (error) { this.logger.error(`Error handling message: ${error}`); diff --git a/services/matrix-onboarding-bot/src/bot/matrix.service.ts b/services/matrix-onboarding-bot/src/bot/matrix.service.ts index ae47d9a3e..c7aaf7a43 100644 --- a/services/matrix-onboarding-bot/src/bot/matrix.service.ts +++ b/services/matrix-onboarding-bot/src/bot/matrix.service.ts @@ -203,9 +203,9 @@ export class MatrixService extends BaseMatrixService { const field = parts[0].toLowerCase(); const value = parts.slice(1).join(' '); - let fieldKey: 'fullName' | 'interests' | 'locale' | null = null; + let fieldKey: 'displayName' | 'interests' | 'locale' | null = null; if (field === 'name' || field === 'namen') { - fieldKey = 'fullName'; + fieldKey = 'displayName'; } else if (field === 'interests' || field === 'interessen') { fieldKey = 'interests'; } else if (field === 'language' || field === 'sprache' || field === 'lang') { diff --git a/services/matrix-planta-bot/CLAUDE.md b/services/matrix-planta-bot/CLAUDE.md index a9bb75230..e3e72bd2c 100644 --- a/services/matrix-planta-bot/CLAUDE.md +++ b/services/matrix-planta-bot/CLAUDE.md @@ -2,7 +2,7 @@ ## Overview -Matrix Planta Bot provides plant care management via Matrix chat. It integrates with the Planta backend for plant CRUD operations, watering schedules, watering history, and care settings. +Matrix Planta Bot provides plant care management via Matrix chat. It integrates with the Planta backend for plant CRUD operations, watering schedules, watering history, care settings, and **AI-powered plant identification** via image analysis. ## Tech Stack @@ -67,6 +67,18 @@ services/matrix-planta-bot/ | `!loeschen [nr]` | delete, entfernen | Remove plant | | `!edit [nr] [feld] [wert]` | bearbeiten | Edit plant field | +### AI Plant Identification + +| Action | Description | +|--------|-------------| +| Send image | Automatically analyzes plant with Gemini Vision AI | + +When you send an image to the bot, it will: +1. Upload the image to the Planta backend +2. Analyze it using Google Gemini Vision +3. Return identification (scientific name, common names, confidence) +4. Show health assessment and care tips + ### Watering | Command | Aliases | Description | @@ -96,6 +108,18 @@ services/matrix-planta-bot/ # Login !login max@example.com mypassword +# Send a plant image -> Bot responds with: +# 🌿 Pflanze erkannt! +# Monstera deliciosa (Fensterblatt) +# βœ… Konfidenz: 92% +# +# Gesundheit: πŸ’š Gesund +# +# πŸ“‹ Pflegetipps: +# β€’ β˜€οΈ Helles Licht - Heller Standort mit indirektem Sonnenlicht +# β€’ πŸ’§ Alle 7 Tage giessen +# β€’ 🌱 Blaetter regelmaessig mit Wasser besprΓΌhen + # Add a new plant !neu Monstera Deliciosa @@ -182,6 +206,8 @@ curl http://localhost:3322/health | `/api/watering/:plantId/water` | POST | Log watering | | `/api/watering/:plantId` | PUT | Update watering schedule | | `/api/watering/:plantId/history` | GET | Get watering history | +| `/api/photos/upload` | POST | Upload plant photo (multipart) | +| `/api/analysis/identify` | POST | Analyze photo with Gemini Vision AI | ## Number-Based Reference System diff --git a/services/matrix-planta-bot/src/bot/matrix.service.ts b/services/matrix-planta-bot/src/bot/matrix.service.ts index 7d691034b..8729865a0 100644 --- a/services/matrix-planta-bot/src/bot/matrix.service.ts +++ b/services/matrix-planta-bot/src/bot/matrix.service.ts @@ -8,7 +8,7 @@ import { KeywordCommandDetector, COMMON_KEYWORDS, } from '@manacore/matrix-bot-common'; -import { PlantaService, Plant } from '../planta/planta.service'; +import { PlantaService, Plant, PlantAnalysis } from '../planta/planta.service'; import { SessionService, TranscriptionService, @@ -86,6 +86,130 @@ export class MatrixService extends BaseMatrixService { } } + protected override async handleImageMessage( + roomId: string, + event: MatrixRoomEvent, + sender: string + ): Promise { + try { + const mxcUrl = event.content.url; + if (!mxcUrl) return; + + // Check auth + const token = await this.sessionService.getToken(sender); + if (!token) { + await this.sendMessage( + roomId, + '

🌱 Melde dich an, um Pflanzen zu analysieren: !login email passwort

' + ); + return; + } + + // Send processing message + await this.sendMessage(roomId, '

πŸ” Analysiere Pflanzenbild...

'); + + // Download image + const imageBuffer = await this.downloadMedia(mxcUrl); + const mimeType = (event.content.info?.mimetype as string) || 'image/jpeg'; + const filename = event.content.body || 'plant.jpg'; + + // Upload and analyze + const result = await this.plantaService.uploadAndAnalyze( + token, + imageBuffer, + mimeType, + filename + ); + + if (result.error || !result.data) { + await this.sendMessage(roomId, `

❌ ${result.error || 'Analyse fehlgeschlagen'}

`); + return; + } + + // Format and send result + const html = this.formatAnalysisResult(result.data); + await this.sendMessage(roomId, html); + } catch (error) { + this.logger.error(`Image analysis error: ${error}`); + await this.sendMessage(roomId, '

❌ Fehler bei der Bildanalyse.

'); + } + } + + private formatAnalysisResult(analysis: PlantAnalysis): string { + const confidence = analysis.confidence || 0; + const confidenceEmoji = confidence >= 80 ? 'βœ…' : confidence >= 50 ? 'πŸ€”' : '❓'; + + let html = '

🌿 Pflanze erkannt!

'; + + // Identification + const scientificName = analysis.scientificName || analysis.identifiedSpecies || 'Unbekannt'; + const commonNames = analysis.commonNames?.join(', ') || ''; + + html += `

${scientificName}`; + if (commonNames) { + html += ` (${commonNames})`; + } + html += `
${confidenceEmoji} Konfidenz: ${confidence}%

`; + + // Health + if (analysis.healthAssessment) { + const healthEmoji = this.getHealthStatusEmoji(analysis.healthAssessment); + html += `

Gesundheit: ${healthEmoji} ${this.translateHealthStatus(analysis.healthAssessment)}`; + if (analysis.healthDetails) { + html += `
${analysis.healthDetails}`; + } + html += '

'; + + if (analysis.issues && analysis.issues.length > 0) { + html += '

Probleme:

'; + } + } + + // Care tips + html += '

πŸ“‹ Pflegetipps:

'; + + // Call to action + html += '

Pflanze hinzufuegen mit: !neu Pflanzenname

'; + + return html; + } + + private getHealthStatusEmoji(status: string): string { + const emojiMap: Record = { + healthy: 'πŸ’š', + minor_issues: 'πŸ’›', + needs_care: '🧑', + critical: '❀️', + }; + return emojiMap[status] || 'πŸ’š'; + } + + private translateHealthStatus(status: string): string { + const statusMap: Record = { + healthy: 'Gesund', + minor_issues: 'Kleinere Probleme', + needs_care: 'Braucht Pflege', + critical: 'Kritisch', + }; + return statusMap[status] || status; + } + protected getConfig(): MatrixBotConfig { return { homeserverUrl: diff --git a/services/matrix-planta-bot/src/planta/planta.service.ts b/services/matrix-planta-bot/src/planta/planta.service.ts index ce866f2ae..2a42fc602 100644 --- a/services/matrix-planta-bot/src/planta/planta.service.ts +++ b/services/matrix-planta-bot/src/planta/planta.service.ts @@ -42,6 +42,30 @@ export interface UpcomingWatering { isOverdue: boolean; } +export interface PlantPhoto { + id: string; + plantId?: string; + storagePath: string; + publicUrl?: string; + isPrimary: boolean; + isAnalyzed: boolean; +} + +export interface PlantAnalysis { + id: string; + photoId: string; + identifiedSpecies?: string; + scientificName?: string; + commonNames?: string[]; + confidence?: number; + healthAssessment?: string; + healthDetails?: string; + issues?: string[]; + wateringAdvice?: string; + lightAdvice?: string; + generalTips?: string[]; +} + @Injectable() export class PlantaService { private readonly logger = new Logger(PlantaService.name); @@ -49,7 +73,8 @@ export class PlantaService { private apiPrefix: string; constructor(private configService: ConfigService) { - this.backendUrl = this.configService.get('planta.backendUrl') || 'http://localhost:3022'; + this.backendUrl = + this.configService.get('planta.backendUrl') || 'http://localhost:3022'; this.apiPrefix = this.configService.get('planta.apiPrefix') || '/api'; } @@ -118,7 +143,9 @@ export class PlantaService { } // Watering operations - async getUpcomingWaterings(token: string): Promise<{ data?: UpcomingWatering[]; error?: string }> { + async getUpcomingWaterings( + token: string + ): Promise<{ data?: UpcomingWatering[]; error?: string }> { return this.request(token, '/watering/upcoming'); } @@ -159,4 +186,100 @@ export class PlantaService { return false; } } + + /** + * Upload a photo for analysis + */ + async uploadPhoto( + token: string, + imageBuffer: Buffer, + mimeType: string, + filename: string, + plantId?: string + ): Promise<{ data?: PlantPhoto; error?: string }> { + try { + const formData = new FormData(); + // Convert Buffer to Blob - use type assertion to bypass strict TypeScript check + const blob = new Blob([imageBuffer as unknown as BlobPart], { type: mimeType }); + formData.append('file', blob, filename); + + let url = `${this.backendUrl}${this.apiPrefix}/photos/upload`; + if (plantId) { + url += `?plantId=${plantId}`; + } + + const response = await fetch(url, { + method: 'POST', + headers: { + Authorization: `Bearer ${token}`, + }, + body: formData, + }); + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})); + return { error: errorData.message || `Fehler: ${response.status}` }; + } + + const data = await response.json(); + return { data }; + } catch (error) { + this.logger.error('Photo upload failed:', error); + return { error: 'Foto-Upload fehlgeschlagen' }; + } + } + + /** + * Analyze a photo with AI + */ + async analyzePhoto( + token: string, + photoId: string, + plantId?: string + ): Promise<{ data?: PlantAnalysis; error?: string }> { + try { + const body: Record = { photoId }; + if (plantId) body.plantId = plantId; + + const response = await fetch(`${this.backendUrl}${this.apiPrefix}/analysis/identify`, { + method: 'POST', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(body), + }); + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})); + return { error: errorData.message || `Fehler: ${response.status}` }; + } + + const data = await response.json(); + return { data }; + } catch (error) { + this.logger.error('Photo analysis failed:', error); + return { error: 'Analyse fehlgeschlagen' }; + } + } + + /** + * Upload and analyze a photo in one step + */ + async uploadAndAnalyze( + token: string, + imageBuffer: Buffer, + mimeType: string, + filename: string, + plantId?: string + ): Promise<{ data?: PlantAnalysis; error?: string }> { + // Step 1: Upload + const uploadResult = await this.uploadPhoto(token, imageBuffer, mimeType, filename, plantId); + if (uploadResult.error || !uploadResult.data) { + return { error: uploadResult.error || 'Upload fehlgeschlagen' }; + } + + // Step 2: Analyze + return this.analyzePhoto(token, uploadResult.data.id, plantId); + } }