✨ feat(planta-bot): add AI plant identification via image upload

- Add handleImageMessage() to BaseMatrixService for image support
- Implement photo upload and Gemini Vision analysis in PlantaService
- Add image handler in MatrixService that downloads, uploads, and analyzes
- Format analysis results with health status, care tips, and confidence
- Update CLAUDE.md documentation with new feature
- Fix type mismatch in onboarding-bot (fullName → displayName)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

packages/matrix-bot-common/src/base/base-matrix.service.ts

@@ -7,7 +7,13 @@ import {
 } from 'matrix-bot-sdk';
 import * as path from 'path';
 import * as fs from 'fs';
-import { type MatrixBotConfig, type MatrixRoomEvent, isTextMessage, isAudioMessage } from './types';
+import {
+	type MatrixBotConfig,
+	type MatrixRoomEvent,
+	isTextMessage,
+	isAudioMessage,
+	isImageMessage,
+} from './types';
 import { markdownToHtml } from '../markdown/markdown-formatter';
 
 /**
@@ -84,6 +90,17 @@ export abstract class BaseMatrixService implements OnModuleInit, OnModuleDestroy
 		// Default: no-op, override in subclass for voice support
 	}
 
+	/**
+	 * Handle an image message (optional override)
+	 */
+	protected async handleImageMessage(
+		_roomId: string,
+		_event: MatrixRoomEvent,
+		_sender: string
+	): Promise<void> {
+		// Default: no-op, override in subclass for image support
+	}
+
 	/**
 	 * Get welcome/introduction message (optional override)
 	 */
@@ -201,6 +218,8 @@ export abstract class BaseMatrixService implements OnModuleInit, OnModuleDestroy
 				await this.handleTextMessage(roomId, event, message, event.sender);
 			} else if (isAudioMessage(event)) {
 				await this.handleAudioMessage(roomId, event, event.sender);
+			} else if (isImageMessage(event)) {
+				await this.handleImageMessage(roomId, event, event.sender);
 			}
 		} catch (error) {
 			this.logger.error(`Error handling message: ${error}`);

services/matrix-onboarding-bot/src/bot/matrix.service.ts

@@ -203,9 +203,9 @@ export class MatrixService extends BaseMatrixService {
 		const field = parts[0].toLowerCase();
 		const value = parts.slice(1).join(' ');
 
-		let fieldKey: 'fullName' | 'interests' | 'locale' | null = null;
+		let fieldKey: 'displayName' | 'interests' | 'locale' | null = null;
 		if (field === 'name' || field === 'namen') {
-			fieldKey = 'fullName';
+			fieldKey = 'displayName';
 		} else if (field === 'interests' || field === 'interessen') {
 			fieldKey = 'interests';
 		} else if (field === 'language' || field === 'sprache' || field === 'lang') {

services/matrix-planta-bot/CLAUDE.md

@@ -2,7 +2,7 @@
 
 ## Overview
 
-Matrix Planta Bot provides plant care management via Matrix chat. It integrates with the Planta backend for plant CRUD operations, watering schedules, watering history, and care settings.
+Matrix Planta Bot provides plant care management via Matrix chat. It integrates with the Planta backend for plant CRUD operations, watering schedules, watering history, care settings, and **AI-powered plant identification** via image analysis.
 
 ## Tech Stack
 
@@ -67,6 +67,18 @@ services/matrix-planta-bot/
 | `!loeschen [nr]` | delete, entfernen | Remove plant |
 | `!edit [nr] [feld] [wert]` | bearbeiten | Edit plant field |
 
+### AI Plant Identification
+
+| Action | Description |
+|--------|-------------|
+| Send image | Automatically analyzes plant with Gemini Vision AI |
+
+When you send an image to the bot, it will:
+1. Upload the image to the Planta backend
+2. Analyze it using Google Gemini Vision
+3. Return identification (scientific name, common names, confidence)
+4. Show health assessment and care tips
+
 ### Watering
 
 | Command | Aliases | Description |
@@ -96,6 +108,18 @@ services/matrix-planta-bot/
 # Login
 !login max@example.com mypassword
 
+# Send a plant image -> Bot responds with:
+# 🌿 Pflanze erkannt!
+# Monstera deliciosa (Fensterblatt)
+# ✅ Konfidenz: 92%
+#
+# Gesundheit: 💚 Gesund
+#
+# 📋 Pflegetipps:
+# • ☀️ Helles Licht - Heller Standort mit indirektem Sonnenlicht
+# • 💧 Alle 7 Tage giessen
+# • 🌱 Blaetter regelmaessig mit Wasser besprühen
+
 # Add a new plant
 !neu Monstera Deliciosa
 
@@ -182,6 +206,8 @@ curl http://localhost:3322/health
 | `/api/watering/:plantId/water` | POST | Log watering |
 | `/api/watering/:plantId` | PUT | Update watering schedule |
 | `/api/watering/:plantId/history` | GET | Get watering history |
+| `/api/photos/upload` | POST | Upload plant photo (multipart) |
+| `/api/analysis/identify` | POST | Analyze photo with Gemini Vision AI |
 
 ## Number-Based Reference System
 

services/matrix-planta-bot/src/bot/matrix.service.ts

@@ -8,7 +8,7 @@ import {
 	KeywordCommandDetector,
 	COMMON_KEYWORDS,
 } from '@manacore/matrix-bot-common';
-import { PlantaService, Plant } from '../planta/planta.service';
+import { PlantaService, Plant, PlantAnalysis } from '../planta/planta.service';
 import {
 	SessionService,
 	TranscriptionService,
@@ -86,6 +86,130 @@ export class MatrixService extends BaseMatrixService {
 		}
 	}
 
+	protected override async handleImageMessage(
+		roomId: string,
+		event: MatrixRoomEvent,
+		sender: string
+	): Promise<void> {
+		try {
+			const mxcUrl = event.content.url;
+			if (!mxcUrl) return;
+
+			// Check auth
+			const token = await this.sessionService.getToken(sender);
+			if (!token) {
+				await this.sendMessage(
+					roomId,
+					'<p>🌱 Melde dich an, um Pflanzen zu analysieren: <code>!login email passwort</code></p>'
+				);
+				return;
+			}
+
+			// Send processing message
+			await this.sendMessage(roomId, '<p>🔍 Analysiere Pflanzenbild...</p>');
+
+			// Download image
+			const imageBuffer = await this.downloadMedia(mxcUrl);
+			const mimeType = (event.content.info?.mimetype as string) || 'image/jpeg';
+			const filename = event.content.body || 'plant.jpg';
+
+			// Upload and analyze
+			const result = await this.plantaService.uploadAndAnalyze(
+				token,
+				imageBuffer,
+				mimeType,
+				filename
+			);
+
+			if (result.error || !result.data) {
+				await this.sendMessage(roomId, `<p>❌ ${result.error || 'Analyse fehlgeschlagen'}</p>`);
+				return;
+			}
+
+			// Format and send result
+			const html = this.formatAnalysisResult(result.data);
+			await this.sendMessage(roomId, html);
+		} catch (error) {
+			this.logger.error(`Image analysis error: ${error}`);
+			await this.sendMessage(roomId, '<p>❌ Fehler bei der Bildanalyse.</p>');
+		}
+	}
+
+	private formatAnalysisResult(analysis: PlantAnalysis): string {
+		const confidence = analysis.confidence || 0;
+		const confidenceEmoji = confidence >= 80 ? '✅' : confidence >= 50 ? '🤔' : '❓';
+
+		let html = '<h3>🌿 Pflanze erkannt!</h3>';
+
+		// Identification
+		const scientificName = analysis.scientificName || analysis.identifiedSpecies || 'Unbekannt';
+		const commonNames = analysis.commonNames?.join(', ') || '';
+
+		html += `<p><strong>${scientificName}</strong>`;
+		if (commonNames) {
+			html += ` <em>(${commonNames})</em>`;
+		}
+		html += `<br/>${confidenceEmoji} Konfidenz: ${confidence}%</p>`;
+
+		// Health
+		if (analysis.healthAssessment) {
+			const healthEmoji = this.getHealthStatusEmoji(analysis.healthAssessment);
+			html += `<p><strong>Gesundheit:</strong> ${healthEmoji} ${this.translateHealthStatus(analysis.healthAssessment)}`;
+			if (analysis.healthDetails) {
+				html += `<br/><em>${analysis.healthDetails}</em>`;
+			}
+			html += '</p>';
+
+			if (analysis.issues && analysis.issues.length > 0) {
+				html += '<p><strong>Probleme:</strong></p><ul>';
+				for (const issue of analysis.issues) {
+					html += `<li>⚠️ ${issue}</li>`;
+				}
+				html += '</ul>';
+			}
+		}
+
+		// Care tips
+		html += '<p><strong>📋 Pflegetipps:</strong></p><ul>';
+		if (analysis.lightAdvice) {
+			html += `<li>☀️ ${analysis.lightAdvice}</li>`;
+		}
+		if (analysis.wateringAdvice) {
+			html += `<li>💧 ${analysis.wateringAdvice}</li>`;
+		}
+		if (analysis.generalTips && analysis.generalTips.length > 0) {
+			for (const tip of analysis.generalTips.slice(0, 3)) {
+				html += `<li>🌱 ${tip}</li>`;
+			}
+		}
+		html += '</ul>';
+
+		// Call to action
+		html += '<p><em>Pflanze hinzufuegen mit: <code>!neu Pflanzenname</code></em></p>';
+
+		return html;
+	}
+
+	private getHealthStatusEmoji(status: string): string {
+		const emojiMap: Record<string, string> = {
+			healthy: '💚',
+			minor_issues: '💛',
+			needs_care: '🧡',
+			critical: '❤️',
+		};
+		return emojiMap[status] || '💚';
+	}
+
+	private translateHealthStatus(status: string): string {
+		const statusMap: Record<string, string> = {
+			healthy: 'Gesund',
+			minor_issues: 'Kleinere Probleme',
+			needs_care: 'Braucht Pflege',
+			critical: 'Kritisch',
+		};
+		return statusMap[status] || status;
+	}
+
 	protected getConfig(): MatrixBotConfig {
 		return {
 			homeserverUrl:

services/matrix-planta-bot/src/planta/planta.service.ts

@@ -42,6 +42,30 @@ export interface UpcomingWatering {
 	isOverdue: boolean;
 }
 
+export interface PlantPhoto {
+	id: string;
+	plantId?: string;
+	storagePath: string;
+	publicUrl?: string;
+	isPrimary: boolean;
+	isAnalyzed: boolean;
+}
+
+export interface PlantAnalysis {
+	id: string;
+	photoId: string;
+	identifiedSpecies?: string;
+	scientificName?: string;
+	commonNames?: string[];
+	confidence?: number;
+	healthAssessment?: string;
+	healthDetails?: string;
+	issues?: string[];
+	wateringAdvice?: string;
+	lightAdvice?: string;
+	generalTips?: string[];
+}
+
 @Injectable()
 export class PlantaService {
 	private readonly logger = new Logger(PlantaService.name);
@@ -49,7 +73,8 @@ export class PlantaService {
 	private apiPrefix: string;
 
 	constructor(private configService: ConfigService) {
-		this.backendUrl = this.configService.get<string>('planta.backendUrl') || 'http://localhost:3022';
+		this.backendUrl =
+			this.configService.get<string>('planta.backendUrl') || 'http://localhost:3022';
 		this.apiPrefix = this.configService.get<string>('planta.apiPrefix') || '/api';
 	}
 
@@ -118,7 +143,9 @@ export class PlantaService {
 	}
 
 	// Watering operations
-	async getUpcomingWaterings(token: string): Promise<{ data?: UpcomingWatering[]; error?: string }> {
+	async getUpcomingWaterings(
+		token: string
+	): Promise<{ data?: UpcomingWatering[]; error?: string }> {
 		return this.request<UpcomingWatering[]>(token, '/watering/upcoming');
 	}
 
@@ -159,4 +186,100 @@ export class PlantaService {
 			return false;
 		}
 	}
+
+	/**
+	 * Upload a photo for analysis
+	 */
+	async uploadPhoto(
+		token: string,
+		imageBuffer: Buffer,
+		mimeType: string,
+		filename: string,
+		plantId?: string
+	): Promise<{ data?: PlantPhoto; error?: string }> {
+		try {
+			const formData = new FormData();
+			// Convert Buffer to Blob - use type assertion to bypass strict TypeScript check
+			const blob = new Blob([imageBuffer as unknown as BlobPart], { type: mimeType });
+			formData.append('file', blob, filename);
+
+			let url = `${this.backendUrl}${this.apiPrefix}/photos/upload`;
+			if (plantId) {
+				url += `?plantId=${plantId}`;
+			}
+
+			const response = await fetch(url, {
+				method: 'POST',
+				headers: {
+					Authorization: `Bearer ${token}`,
+				},
+				body: formData,
+			});
+
+			if (!response.ok) {
+				const errorData = await response.json().catch(() => ({}));
+				return { error: errorData.message || `Fehler: ${response.status}` };
+			}
+
+			const data = await response.json();
+			return { data };
+		} catch (error) {
+			this.logger.error('Photo upload failed:', error);
+			return { error: 'Foto-Upload fehlgeschlagen' };
+		}
+	}
+
+	/**
+	 * Analyze a photo with AI
+	 */
+	async analyzePhoto(
+		token: string,
+		photoId: string,
+		plantId?: string
+	): Promise<{ data?: PlantAnalysis; error?: string }> {
+		try {
+			const body: Record<string, string> = { photoId };
+			if (plantId) body.plantId = plantId;
+
+			const response = await fetch(`${this.backendUrl}${this.apiPrefix}/analysis/identify`, {
+				method: 'POST',
+				headers: {
+					Authorization: `Bearer ${token}`,
+					'Content-Type': 'application/json',
+				},
+				body: JSON.stringify(body),
+			});
+
+			if (!response.ok) {
+				const errorData = await response.json().catch(() => ({}));
+				return { error: errorData.message || `Fehler: ${response.status}` };
+			}
+
+			const data = await response.json();
+			return { data };
+		} catch (error) {
+			this.logger.error('Photo analysis failed:', error);
+			return { error: 'Analyse fehlgeschlagen' };
+		}
+	}
+
+	/**
+	 * Upload and analyze a photo in one step
+	 */
+	async uploadAndAnalyze(
+		token: string,
+		imageBuffer: Buffer,
+		mimeType: string,
+		filename: string,
+		plantId?: string
+	): Promise<{ data?: PlantAnalysis; error?: string }> {
+		// Step 1: Upload
+		const uploadResult = await this.uploadPhoto(token, imageBuffer, mimeType, filename, plantId);
+		if (uploadResult.error || !uploadResult.data) {
+			return { error: uploadResult.error || 'Upload fehlgeschlagen' };
+		}
+
+		// Step 2: Analyze
+		return this.analyzePhoto(token, uploadResult.data.id, plantId);
+	}
 }