mirror of
https://github.com/Memo-2023/mana-monorepo.git
synced 2026-05-14 19:01:08 +02:00
a55aae6cb5
Three Mac Mini infrastructure follow-ups bundled:
1. docker-compose.macmini.yml — drop ghost backend env vars from
the mana-app-web service (todo, calendar, contacts, chat, storage,
cards, music, nutriphi `PUBLIC_*_API_URL{,_CLIENT}` plus the memoro
server URLs). The matching consumers were removed in the earlier
ghost-API cleanup commits, so these env entries had been wiring
nothing into the running container for several deploys. Force-
recreating mana-app-web after pulling this commit will pick up
the slimmer env automatically.
2. docker-compose.macmini.yml — bump `mana-mon-blackbox` mem_limit
from 32m to 128m. blackbox-exporter v0.25 sits north of 32m
under load and was OOM-restart-looping every ~90 seconds, which
in turn made `status.mana.how` and the prometheus probe metrics
stale (since the scraper was missing every other window).
3. docker/prometheus/prometheus.yml — split `blackbox-gpu` into two
jobs:
- `blackbox-gpu` now probes `/health` via the http_health
module, because the GPU services (whisper STT, FLUX image
gen, Coqui TTS) return 401/404 on `/` by design (auth or
API-only). The previous http_2xx-on-`/` probe was reporting
all four as down even though they answered `/health` with
200, which inflated the down count on status.mana.how.
- `blackbox-gpu-root` keeps the http_2xx-on-`/` probe for
Ollama, which has no `/health` endpoint but does answer
2xx on its root.
Both jobs share the same blackbox-exporter relabel rewrite so
the targets are routed through the exporter container, not
scraped directly by VictoriaMetrics.
Verified post-fix: status.mana.how reports 41/42 services up (only
`gpu-video` remains down — LTX Video Gen is intentionally not
deployed yet on the Windows GPU box).
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
354 lines
9.8 KiB
YAML
354 lines
9.8 KiB
YAML
# Mana Prometheus Configuration
|
|
# Scrapes metrics from all services
|
|
|
|
global:
|
|
scrape_interval: 15s
|
|
evaluation_interval: 15s
|
|
|
|
# Load alerting rules
|
|
rule_files:
|
|
- /etc/prometheus/alerts.yml
|
|
|
|
# Alertmanager configuration
|
|
alerting:
|
|
alertmanagers:
|
|
- static_configs:
|
|
- targets: ['alertmanager:9093']
|
|
|
|
scrape_configs:
|
|
# Prometheus self-monitoring
|
|
- job_name: 'prometheus'
|
|
static_configs:
|
|
- targets: ['localhost:9090']
|
|
|
|
# Host system metrics via node-exporter
|
|
- job_name: 'node'
|
|
static_configs:
|
|
- targets: ['node-exporter:9100']
|
|
relabel_configs:
|
|
- source_labels: [__address__]
|
|
target_label: instance
|
|
replacement: 'mac-mini'
|
|
|
|
# Docker container metrics via cAdvisor
|
|
- job_name: 'cadvisor'
|
|
static_configs:
|
|
- targets: ['cadvisor:8080']
|
|
|
|
# PostgreSQL metrics
|
|
- job_name: 'postgres'
|
|
static_configs:
|
|
- targets: ['postgres-exporter:9187']
|
|
|
|
# Redis metrics
|
|
- job_name: 'redis'
|
|
static_configs:
|
|
- targets: ['redis-exporter:9121']
|
|
|
|
# ============================================
|
|
# Core Services (Hono/Bun + Go)
|
|
# ============================================
|
|
|
|
# Auth Service
|
|
- job_name: 'mana-auth'
|
|
static_configs:
|
|
- targets: ['mana-auth:3001']
|
|
metrics_path: '/metrics'
|
|
scrape_interval: 30s
|
|
|
|
# Credits Service
|
|
- job_name: 'mana-credits'
|
|
static_configs:
|
|
- targets: ['mana-credits:3002']
|
|
metrics_path: '/metrics'
|
|
scrape_interval: 30s
|
|
|
|
# User Service
|
|
- job_name: 'mana-user'
|
|
static_configs:
|
|
- targets: ['mana-user:3062']
|
|
metrics_path: '/metrics'
|
|
scrape_interval: 30s
|
|
|
|
# Subscriptions Service
|
|
- job_name: 'mana-subscriptions'
|
|
static_configs:
|
|
- targets: ['mana-subscriptions:3063']
|
|
metrics_path: '/metrics'
|
|
scrape_interval: 30s
|
|
|
|
# Analytics Service
|
|
- job_name: 'mana-analytics'
|
|
static_configs:
|
|
- targets: ['mana-analytics:3064']
|
|
metrics_path: '/metrics'
|
|
scrape_interval: 30s
|
|
|
|
# ULoad Server
|
|
- job_name: 'uload-server'
|
|
static_configs:
|
|
- targets: ['mana-app-uload-server:3070']
|
|
metrics_path: '/metrics'
|
|
scrape_interval: 30s
|
|
|
|
# Memoro Server
|
|
- job_name: 'memoro-server'
|
|
static_configs:
|
|
- targets: ['mana-app-memoro-server:3015']
|
|
metrics_path: '/metrics'
|
|
scrape_interval: 30s
|
|
|
|
# NOTE: Individual app backends (chat, todo, calendar, contacts, storage,
|
|
# nutriphi, music, planta, picture) have been REMOVED — all migrated to
|
|
# local-first architecture. Only uload-server and memoro-server remain.
|
|
|
|
# Mana LLM Gateway (Ollama + Google Fallback)
|
|
- job_name: 'mana-llm'
|
|
static_configs:
|
|
- targets: ['mana-llm:3020']
|
|
metrics_path: '/metrics'
|
|
scrape_interval: 15s
|
|
|
|
# Mana Search Service
|
|
- job_name: 'mana-search'
|
|
static_configs:
|
|
- targets: ['mana-search:3012']
|
|
metrics_path: '/metrics'
|
|
scrape_interval: 30s
|
|
|
|
# Mana Media Service
|
|
- job_name: 'mana-media'
|
|
static_configs:
|
|
- targets: ['mana-media:3011']
|
|
metrics_path: '/metrics'
|
|
scrape_interval: 30s
|
|
|
|
# Matrix Synapse
|
|
- job_name: 'synapse'
|
|
static_configs:
|
|
- targets: ['synapse:9002']
|
|
metrics_path: '/_synapse/metrics'
|
|
scrape_interval: 30s
|
|
|
|
# ============================================
|
|
# GPU Server (Windows PC, LAN: 192.168.178.11)
|
|
# ============================================
|
|
|
|
# GPU: LLM Gateway
|
|
- job_name: 'gpu-llm'
|
|
static_configs:
|
|
- targets: ['192.168.178.11:3025']
|
|
labels:
|
|
instance: 'gpu-server'
|
|
metrics_path: '/metrics'
|
|
scrape_interval: 15s
|
|
|
|
# GPU: Speech-to-Text (WhisperX)
|
|
- job_name: 'gpu-stt'
|
|
static_configs:
|
|
- targets: ['192.168.178.11:3020']
|
|
labels:
|
|
instance: 'gpu-server'
|
|
metrics_path: '/health'
|
|
scrape_interval: 30s
|
|
|
|
# GPU: Text-to-Speech
|
|
- job_name: 'gpu-tts'
|
|
static_configs:
|
|
- targets: ['192.168.178.11:3022']
|
|
labels:
|
|
instance: 'gpu-server'
|
|
metrics_path: '/health'
|
|
scrape_interval: 30s
|
|
|
|
# GPU: Image Generation (FLUX.2)
|
|
- job_name: 'gpu-image-gen'
|
|
static_configs:
|
|
- targets: ['192.168.178.11:3023']
|
|
labels:
|
|
instance: 'gpu-server'
|
|
metrics_path: '/health'
|
|
scrape_interval: 30s
|
|
|
|
# GPU: Video Generation (LTX-Video)
|
|
- job_name: 'gpu-video-gen'
|
|
static_configs:
|
|
- targets: ['192.168.178.11:3026']
|
|
labels:
|
|
instance: 'gpu-server'
|
|
metrics_path: '/health'
|
|
scrape_interval: 30s
|
|
|
|
# ============================================
|
|
# Go Infrastructure Services
|
|
# ============================================
|
|
|
|
# API Gateway (Go)
|
|
- job_name: 'mana-api-gateway'
|
|
static_configs:
|
|
- targets: ['mana-api-gateway:3016']
|
|
metrics_path: '/metrics'
|
|
scrape_interval: 15s
|
|
|
|
# Matrix Bot (Go) — consolidated 21 bots
|
|
- job_name: 'mana-matrix-bot'
|
|
static_configs:
|
|
- targets: ['mana-matrix-bot:4000']
|
|
metrics_path: '/metrics'
|
|
scrape_interval: 30s
|
|
|
|
# Sync Server (Go) — local-first data sync
|
|
- job_name: 'mana-sync'
|
|
static_configs:
|
|
- targets: ['mana-core-sync:3051']
|
|
metrics_path: '/metrics'
|
|
scrape_interval: 30s
|
|
|
|
# Notification Service (Go) — email, push, matrix, webhook
|
|
- job_name: 'mana-notify'
|
|
static_configs:
|
|
- targets: ['mana-core-notify:3013']
|
|
metrics_path: '/metrics'
|
|
scrape_interval: 30s
|
|
|
|
# Crawler Service (Go)
|
|
- job_name: 'mana-crawler'
|
|
static_configs:
|
|
- targets: ['mana-crawler:3014']
|
|
metrics_path: '/metrics'
|
|
scrape_interval: 30s
|
|
|
|
# ============================================
|
|
# Blackbox Exporter — HTTP Uptime Probes
|
|
# ============================================
|
|
|
|
# Web Apps (Unified Mana app at mana.how + standalone games)
|
|
- job_name: 'blackbox-web'
|
|
metrics_path: /probe
|
|
params:
|
|
module: [http_2xx]
|
|
static_configs:
|
|
- targets:
|
|
# Unified Mana app (all modules as routes)
|
|
- https://mana.how
|
|
- https://mana.how/chat
|
|
- https://mana.how/todo
|
|
- https://mana.how/calendar
|
|
- https://mana.how/contacts
|
|
- https://mana.how/times
|
|
- https://mana.how/photos
|
|
- https://mana.how/picture
|
|
- https://mana.how/storage
|
|
- https://mana.how/presi
|
|
- https://mana.how/nutriphi
|
|
- https://mana.how/planta
|
|
- https://mana.how/calc
|
|
- https://mana.how/zitare
|
|
- https://mana.how/cards
|
|
- https://mana.how/skilltree
|
|
- https://mana.how/music
|
|
- https://mana.how/citycorners
|
|
- https://mana.how/memoro
|
|
- https://mana.how/moodlit
|
|
- https://mana.how/context
|
|
- https://mana.how/questions
|
|
- https://mana.how/uload
|
|
- https://mana.how/notes
|
|
- https://mana.how/habits
|
|
- https://mana.how/guides
|
|
- https://mana.how/inventar
|
|
- https://mana.how/playground
|
|
# Standalone games (separate containers)
|
|
- https://whopxl.mana.how
|
|
- https://arcade.mana.how
|
|
relabel_configs:
|
|
- source_labels: [__address__]
|
|
target_label: __param_target
|
|
- source_labels: [__param_target]
|
|
target_label: instance
|
|
- target_label: __address__
|
|
replacement: blackbox-exporter:9115
|
|
|
|
# API Health Endpoints (only services with running containers)
|
|
- job_name: 'blackbox-api'
|
|
metrics_path: /probe
|
|
params:
|
|
module: [http_health]
|
|
static_configs:
|
|
- targets:
|
|
- https://auth.mana.how/health
|
|
- https://api.mana.how/health
|
|
relabel_configs:
|
|
- source_labels: [__address__]
|
|
target_label: __param_target
|
|
- source_labels: [__param_target]
|
|
target_label: instance
|
|
- target_label: __address__
|
|
replacement: blackbox-exporter:9115
|
|
|
|
# Infrastructure & Monitoring Tools
|
|
- job_name: 'blackbox-infra'
|
|
metrics_path: /probe
|
|
params:
|
|
module: [http_2xx]
|
|
static_configs:
|
|
- targets:
|
|
- https://git.mana.how
|
|
- https://grafana.mana.how
|
|
- https://stats.mana.how
|
|
- https://glitchtip.mana.how
|
|
- https://matrix.mana.how
|
|
- https://element.mana.how
|
|
relabel_configs:
|
|
- source_labels: [__address__]
|
|
target_label: __param_target
|
|
- source_labels: [__param_target]
|
|
target_label: instance
|
|
- target_label: __address__
|
|
replacement: blackbox-exporter:9115
|
|
|
|
# GPU Server Services — probe /health, not /
|
|
# The GPU services (whisper STT, TTS, FLUX image gen) only return 2xx
|
|
# on /health; their root path returns 401/403/404 by design (auth or
|
|
# API-only). Ollama is the exception — its / returns 200, but it has
|
|
# no /health endpoint, so we keep it on / via a separate target.
|
|
- job_name: 'blackbox-gpu'
|
|
metrics_path: /probe
|
|
params:
|
|
module: [http_health]
|
|
static_configs:
|
|
- targets:
|
|
- https://gpu-stt.mana.how/health
|
|
- https://gpu-tts.mana.how/health
|
|
- https://gpu-img.mana.how/health
|
|
- https://gpu-video.mana.how/health
|
|
relabel_configs:
|
|
- source_labels: [__address__]
|
|
target_label: __param_target
|
|
- source_labels: [__param_target]
|
|
target_label: instance
|
|
- target_label: __address__
|
|
replacement: blackbox-exporter:9115
|
|
|
|
- job_name: 'blackbox-gpu-root'
|
|
metrics_path: /probe
|
|
params:
|
|
module: [http_2xx]
|
|
static_configs:
|
|
- targets:
|
|
- https://gpu-ollama.mana.how
|
|
relabel_configs:
|
|
- source_labels: [__address__]
|
|
target_label: __param_target
|
|
- source_labels: [__param_target]
|
|
target_label: instance
|
|
- target_label: __address__
|
|
replacement: blackbox-exporter:9115
|
|
|
|
# ============================================
|
|
# Pushgateway (deploy metrics, batch jobs)
|
|
# ============================================
|
|
- job_name: 'pushgateway'
|
|
honor_labels: true
|
|
static_configs:
|
|
- targets: ['pushgateway:9091']
|