feat(infra): smart build memory check and baseline monitoring script

build-app.sh now checks available RAM before builds and only stops
monitoring containers when free memory is below 3 GB threshold.
New memory-baseline.sh script measures per-container and per-category
RAM usage for capacity planning.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Till JS 2026-03-29 15:07:20 +02:00
parent 2b448419ba
commit f5cd77b2b0
10 changed files with 550 additions and 14 deletions

View file

@ -3,8 +3,10 @@ FROM sveltekit-base:local AS builder
ARG PUBLIC_BACKEND_URL=http://mana-auth
ARG PUBLIC_MANA_CORE_AUTH_URL=http://mana-auth:3001
ARG MIDDLEWARE_URL=http://mana-auth:3001
ENV PUBLIC_BACKEND_URL=$PUBLIC_BACKEND_URL
ENV PUBLIC_MANA_CORE_AUTH_URL=$PUBLIC_MANA_CORE_AUTH_URL
ENV MIDDLEWARE_URL=$MIDDLEWARE_URL
COPY apps/manacore/apps/web ./apps/manacore/apps/web

View file

@ -3,8 +3,10 @@ FROM sveltekit-base:local AS builder
ARG PUBLIC_BACKEND_URL=http://manadeck-server
ARG PUBLIC_MANA_CORE_AUTH_URL=http://mana-auth:3001
ARG PUBLIC_API_URL=http://manadeck-server
ENV PUBLIC_BACKEND_URL=$PUBLIC_BACKEND_URL
ENV PUBLIC_MANA_CORE_AUTH_URL=$PUBLIC_MANA_CORE_AUTH_URL
ENV PUBLIC_API_URL=$PUBLIC_API_URL
COPY apps/manadeck/apps/web ./apps/manadeck/apps/web

View file

@ -61,6 +61,7 @@
"butterchurn": "^2.6.7",
"butterchurn-presets": "^2.4.7",
"pixi.js": "^8.17.1",
"svelte-i18n": "^4.0.1",
"wavesurfer.js": "^7.8.0"
},
"type": "module"

View file

@ -1,10 +1,12 @@
# syntax=docker/dockerfile:1
FROM sveltekit-base:local AS builder
ARG PUBLIC_BACKEND_URL=http://mana-auth
ARG PUBLIC_BACKEND_URL=http://mana-llm:3025
ARG PUBLIC_MANA_CORE_AUTH_URL=http://mana-auth:3001
ARG PUBLIC_MANA_LLM_URL=http://mana-llm:3025
ENV PUBLIC_BACKEND_URL=$PUBLIC_BACKEND_URL
ENV PUBLIC_MANA_CORE_AUTH_URL=$PUBLIC_MANA_CORE_AUTH_URL
ENV PUBLIC_MANA_LLM_URL=$PUBLIC_MANA_LLM_URL
COPY apps/playground/apps/web ./apps/playground/apps/web

View file

@ -0,0 +1,54 @@
#!/bin/bash
# Generate random token
gen_token() {
openssl rand -hex 32
}
# Bot configurations: name, sender_localpart
declare -a BOTS=(
"mana:mana-bot"
"ollama:ollama-bot"
"stats:stats-bot"
"projectdoc:projectdoc-bot"
"todo:todo-bot"
"calendar:calendar-bot"
"nutriphi:nutriphi-bot"
"zitare:zitare-bot"
"clock:clock-bot"
"tts:tts-bot"
)
echo "# Generated AS tokens for .env file:" > as-tokens.env
echo "" >> as-tokens.env
for bot_config in "${BOTS[@]}"; do
IFS=":" read -r name sender <<< "$bot_config"
as_token=$(gen_token)
hs_token=$(gen_token)
cat > "${name}-bot.yaml" << EOF
id: ${name}-bot
hs_token: ${hs_token}
as_token: ${as_token}
url: null
sender_localpart: ${sender}
namespaces:
users:
- exclusive: true
regex: '@${sender}:mana\.how'
rooms: []
aliases: []
rate_limited: false
EOF
# Convert name to uppercase for env var
env_name=$(echo "${name}" | tr '[:lower:]' '[:upper:]' | tr '-' '_')
echo "MATRIX_${env_name}_BOT_AS_TOKEN=${as_token}" >> as-tokens.env
echo "Created ${name}-bot.yaml with AS token"
done
echo ""
echo "Done! Add the tokens from as-tokens.env to your .env file"

View file

@ -0,0 +1,222 @@
# ManaCore Matrix Synapse Configuration
# Documentation: https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html
server_name: "mana.how"
pid_file: /data/homeserver.pid
public_baseurl: https://matrix.mana.how/
# ============================================
# Listeners
# ============================================
listeners:
- port: 8008
tls: false
type: http
x_forwarded: true
resources:
- names: [client, federation]
compress: false
# ============================================
# Database (PostgreSQL)
# ============================================
database:
name: psycopg2
txn_limit: 10000
args:
user: synapse
password: "synapse-secure-password"
database: matrix
host: postgres
port: 5432
cp_min: 5
cp_max: 10
# ============================================
# Logging
# ============================================
log_config: "/config/log.config.yaml"
# ============================================
# Media Storage
# ============================================
media_store_path: /data/media_store
max_upload_size: 50M
url_preview_enabled: true
url_preview_ip_range_blacklist:
- '127.0.0.0/8'
- '10.0.0.0/8'
- '172.16.0.0/12'
- '192.168.0.0/16'
- '100.64.0.0/10'
- '192.0.0.0/24'
- '169.254.0.0/16'
- '198.18.0.0/15'
- '192.0.2.0/24'
- '198.51.100.0/24'
- '203.0.113.0/24'
- '224.0.0.0/4'
- '::1/128'
- 'fe80::/10'
- 'fc00::/7'
- '2001:db8::/32'
- 'ff00::/8'
- 'fec0::/10'
# ============================================
# Registration & Authentication
# ============================================
enable_registration: false
enable_registration_without_verification: false
# Password config (disabled - all users authenticate via OIDC/SSO)
password_config:
enabled: false
localdb_enabled: false
pepper: "${SYNAPSE_PASSWORD_PEPPER:-change-me-pepper}"
# Session lifetime (must be >= refresh_token_lifetime)
# Set to 10 years for bot tokens to avoid frequent expiration
session_lifetime: 87600h
refresh_token_lifetime: 87600h
# ============================================
# Rate Limiting
# ============================================
rc_message:
per_second: 5
burst_count: 20
rc_registration:
per_second: 0.5
burst_count: 5
rc_login:
address:
per_second: 0.5
burst_count: 5
account:
per_second: 0.5
burst_count: 5
failed_attempts:
per_second: 0.5
burst_count: 5
# ============================================
# Federation
# ============================================
# Allow federation with other Matrix servers
federation_domain_whitelist: []
trusted_key_servers:
- server_name: "matrix.org"
# ============================================
# DSGVO / Data Retention
# ============================================
retention:
enabled: true
default_policy:
min_lifetime: 1d
max_lifetime: 365d
allowed_lifetime_min: 1d
allowed_lifetime_max: 365d
purge_jobs:
- longest_max_lifetime: 3d
interval: 12h
- shortest_max_lifetime: 365d
interval: 1d
# Forgotten room retention
forgotten_room_retention_period: 7d
# ============================================
# Security
# ============================================
signing_key_path: "/data/signing.key"
form_secret: "${SYNAPSE_FORM_SECRET:-change-me-form-secret}"
macaroon_secret_key: "${SYNAPSE_MACAROON_SECRET:-change-me-macaroon-secret}"
registration_shared_secret: "${SYNAPSE_REGISTRATION_SECRET:-change-me-registration-secret}"
# ============================================
# Application Services (for Bots)
# Currently disabled - using long-lived user tokens instead
# TODO: Migrate bots to AS for truly permanent tokens
# ============================================
app_service_config_files: []
# ============================================
# Metrics & Telemetry
# ============================================
report_stats: false
enable_metrics: true
metrics_port: 9002
# ============================================
# Caching
# ============================================
caches:
global_factor: 0.5
per_cache_factors: {}
expire_caches: true
cache_entry_ttl: 30m
# ============================================
# Background Tasks
# ============================================
run_background_tasks_on: synapse
# ============================================
# Email (optional, for password reset)
# ============================================
# email:
# smtp_host: smtp-relay.brevo.com
# smtp_port: 587
# smtp_user: "${SMTP_USER}"
# smtp_pass: "${SMTP_PASSWORD}"
# require_transport_security: true
# notif_from: "ManaCore Matrix <noreply@mana.how>"
# ============================================
# OIDC / SSO Configuration (Mana Core Auth)
# ============================================
# Enable SSO via Mana Core Auth OIDC Provider
oidc_providers:
- idp_id: manacore
idp_name: "Mana Core"
idp_brand: "org.matrix.custom"
discover: true
issuer: "https://auth.mana.how"
client_id: "matrix-synapse"
client_secret: "6dc67d2dbea5c19409d21cbaec5ba77265b0296796d4ebb015d70209c68f3fd5"
scopes: ["openid", "profile", "email"]
user_mapping_provider:
config:
subject_claim: "sub"
localpart_template: "{{ user.email.split('@')[0] }}"
display_name_template: "{{ user.name }}"
email_template: "{{ user.email }}"
allow_existing_users: true
enable_registration: true
# SSO UI Settings
sso:
client_whitelist:
- "https://element.mana.how"
- "https://matrix.mana.how"

View file

@ -0,0 +1,34 @@
# Synapse Logging Configuration
version: 1
formatters:
precise:
format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
handlers:
console:
class: logging.StreamHandler
formatter: precise
stream: 'ext://sys.stdout'
file:
class: logging.handlers.TimedRotatingFileHandler
formatter: precise
filename: /data/logs/homeserver.log
when: midnight
backupCount: 7
encoding: utf8
loggers:
synapse.storage.SQL:
level: WARNING
synapse.access.http.8008:
level: WARNING
root:
level: INFO
handlers: [console, file]
disable_existing_loggers: false

View file

@ -458,12 +458,13 @@ Die Base Images enthalten alle Shared Packages (`packages/`) vorinstalliert und
### Build-Script (`build-app.sh`)
Das Script löst das RAM-Problem beim Bauen: Der Mac Mini hat 16 GB, davon sind ~10 GB durch laufende Container belegt. Docker Builds (besonders Vite/SvelteKit) brauchen 4+ GB.
Das Script prüft vor dem Build den verfügbaren RAM und stoppt Monitoring-Container **nur wenn nötig** (< 3 GB frei). Alle Container haben explizite `mem_limit` Obergrenzen in der `docker-compose.macmini.yml`, sodass der tatsächliche Verbrauch typischerweise 50-70% der Limits beträgt und genug Headroom für Builds bleibt.
**Was es tut:**
1. Stoppt automatisch 13 Monitoring-Container (~2 GB RAM frei)
2. Baut die angegebenen Services
3. Startet Monitoring bei Exit automatisch wieder (auch bei Fehler/Ctrl+C via `trap`)
1. Prüft verfügbaren RAM in der Colima VM
2. Stoppt 13 Monitoring-Container nur wenn < 3 GB frei (vorher: immer)
3. Baut die angegebenen Services
4. Startet Monitoring bei Exit automatisch wieder (auch bei Fehler/Ctrl+C via `trap`)
```bash
# Einzelne App
@ -474,8 +475,43 @@ Das Script löst das RAM-Problem beim Bauen: Der Mac Mini hat 16 GB, davon sind
# Alle Web-Apps
./scripts/mac-mini/build-app.sh --all-web
# Monitoring immer stoppen (altes Verhalten)
./scripts/mac-mini/build-app.sh --force-free todo-web
```
### Memory Baseline
Misst den tatsächlichen RAM-Verbrauch aller Container, sortiert nach Kategorie:
```bash
# Einmalige Messung mit Zusammenfassung
./scripts/mac-mini/memory-baseline.sh
# Live-Monitoring (docker stats)
./scripts/mac-mini/memory-baseline.sh --watch
```
### Memory-Limits
Alle 63 Container haben explizite `mem_limit` in `docker-compose.macmini.yml`:
| Kategorie | Container | Budget |
|-----------|-----------|--------|
| Infrastructure | 6 | 1.712 MB |
| Forgejo | 2 | 768 MB |
| Core (Hono/Bun) | 5 | 704 MB |
| Go Services | 5 | 384 MB |
| Other Backend | 3 | 576 MB |
| Matrix | 4 | 784 MB |
| Web Apps | 20 | 2.560 MB |
| LLM | 2 | 384 MB |
| Monitoring | 14 | 1.792 MB |
| Games/Auto | 2 | 192 MB |
| **Total** | **63** | **9.856 MB (9,6 GiB)** |
Colima VM: 12 GiB → Headroom: ~2,4 GiB (Limits) / ~5-6 GiB (real)
### Backup
Die PostgreSQL-Datenbank sollte regelmäßig gesichert werden:
@ -510,7 +546,8 @@ docker image prune -a
| `restart.sh` | Startet alle Container neu |
| `stop.sh` | Stoppt alle Container |
| `deploy.sh` | Pullt neue Images und startet neu |
| `build-app.sh` | Baut einzelne Apps (stoppt Monitoring für RAM) |
| `build-app.sh` | Baut einzelne Apps (smart memory check, stoppt Monitoring nur wenn nötig) |
| `memory-baseline.sh` | Misst RAM-Verbrauch aller Container nach Kategorie |
## Hardware

View file

@ -1,12 +1,13 @@
#!/bin/bash
# Build and deploy specific app containers on the Mac Mini
# Automatically frees RAM by stopping monitoring before build
# Checks available memory and only stops monitoring if needed for builds.
#
# Usage:
# ./scripts/mac-mini/build-app.sh todo-web
# ./scripts/mac-mini/build-app.sh todo-web todo-backend
# ./scripts/mac-mini/build-app.sh --all-web # rebuild all web apps
# ./scripts/mac-mini/build-app.sh --base # rebuild base images only
# ./scripts/mac-mini/build-app.sh --force-free # always stop monitoring
set -e
@ -15,6 +16,9 @@ PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
COMPOSE_FILE="$PROJECT_ROOT/docker-compose.macmini.yml"
DOCKER="${DOCKER_CMD:-/usr/local/bin/docker}"
# Minimum free memory (in MB) needed for a Docker build
BUILD_MEM_THRESHOLD_MB=3000
# Monitoring containers (by container name — more reliable than compose service names)
MONITORING_CONTAINERS=(
mana-mon-grafana
@ -47,14 +51,56 @@ cleanup() {
# Always restart monitoring on exit (success, failure, or interrupt)
trap cleanup EXIT
stop_monitoring() {
echo "=== Stopping monitoring to free RAM ==="
get_available_memory_mb() {
# Get Colima VM total memory and current Docker usage
local vm_total_mb
vm_total_mb=$(colima list -j 2>/dev/null | python3 -c "import sys,json; d=json.load(sys.stdin); print(int(d[0].get('memory',0) / 1048576))" 2>/dev/null || echo "12288")
# Sum all container memory usage
local used_mb
used_mb=$($DOCKER stats --no-stream --format '{{.MemUsage}}' 2>/dev/null | \
awk '{
split($1, a, "/");
val = a[1];
gsub(/[[:space:]]/, "", val);
if (index(val, "GiB") > 0) { gsub(/GiB/, "", val); total += val * 1024; }
else if (index(val, "MiB") > 0) { gsub(/MiB/, "", val); total += val; }
else if (index(val, "KiB") > 0) { gsub(/KiB/, "", val); total += val / 1024; }
} END { printf "%.0f", total }')
echo $(( vm_total_mb - used_mb ))
}
maybe_stop_monitoring() {
local force="${1:-false}"
if [ "$force" = "true" ]; then
echo "=== Force-freeing RAM (--force-free) ==="
stop_monitoring_now
return
fi
echo "=== Checking available memory ==="
local avail_mb
avail_mb=$(get_available_memory_mb)
echo " Available: ${avail_mb} MB (need: ${BUILD_MEM_THRESHOLD_MB} MB)"
if [ "$avail_mb" -lt "$BUILD_MEM_THRESHOLD_MB" ]; then
echo " → Not enough — stopping monitoring to free RAM"
stop_monitoring_now
else
echo " → Sufficient — monitoring stays running ✓"
echo ""
# Still prune build cache
$DOCKER builder prune -f 2>/dev/null | tail -1 || true
fi
}
stop_monitoring_now() {
$DOCKER stop "${MONITORING_CONTAINERS[@]}" 2>/dev/null || true
MONITORING_STOPPED=true
# Also prune dangling build cache
$DOCKER builder prune -f 2>/dev/null | tail -1 || true
echo "RAM freed."
echo " RAM freed."
echo ""
}
@ -104,17 +150,30 @@ if [ $# -eq 0 ]; then
echo " $0 --base # Rebuild base images"
echo " $0 --all-web # Rebuild all web apps"
echo " $0 mana-matrix-bot # Build & restart consolidated Matrix bot (Go)"
echo " $0 --force-free todo-web # Force stop monitoring before build"
exit 1
fi
cd "$PROJECT_ROOT"
# Check for --force-free flag
FORCE_FREE=false
ARGS=()
for arg in "$@"; do
if [ "$arg" = "--force-free" ]; then
FORCE_FREE=true
else
ARGS+=("$arg")
fi
done
set -- "${ARGS[@]}"
# Pull latest code
echo "=== Pulling latest code ==="
git pull
# Free RAM
stop_monitoring
# Smart memory check — only stop monitoring if needed
maybe_stop_monitoring "$FORCE_FREE"
case "$1" in
--base)

View file

@ -0,0 +1,123 @@
#!/bin/bash
# Memory Baseline Script
# Run on the Mac Mini to measure actual container memory usage
# Usage: ./scripts/mac-mini/memory-baseline.sh [--watch]
set -euo pipefail
BOLD="\033[1m"
GREEN="\033[0;32m"
YELLOW="\033[0;33m"
RED="\033[0;31m"
CYAN="\033[0;36m"
RESET="\033[0m"
echo -e "${BOLD}╔══════════════════════════════════════════════════╗${RESET}"
echo -e "${BOLD}║ Mana Docker Memory Baseline Report ║${RESET}"
echo -e "${BOLD}╚══════════════════════════════════════════════════╝${RESET}"
echo ""
echo -e "${CYAN}Timestamp:${RESET} $(date '+%Y-%m-%d %H:%M:%S')"
echo ""
# Get Colima VM memory
COLIMA_MEM=$(colima list -j 2>/dev/null | python3 -c "import sys,json; d=json.load(sys.stdin); print(d[0].get('memory',0))" 2>/dev/null || echo "unknown")
if [ "$COLIMA_MEM" != "unknown" ] && [ "$COLIMA_MEM" -gt 0 ] 2>/dev/null; then
COLIMA_MEM_GB=$(echo "scale=1; $COLIMA_MEM / 1073741824" | bc 2>/dev/null || echo "$COLIMA_MEM")
echo -e "${CYAN}Colima VM Memory:${RESET} ${COLIMA_MEM_GB} GB"
else
echo -e "${CYAN}Colima VM Memory:${RESET} (run 'colima list' to check)"
fi
echo ""
# Docker stats snapshot (no-stream = one-shot)
echo -e "${BOLD}── Per-Container Memory Usage ──${RESET}"
echo ""
printf "${BOLD}%-40s %10s %10s %10s${RESET}\n" "CONTAINER" "MEM USAGE" "MEM LIMIT" "MEM %"
echo "────────────────────────────────────────────────────────────────────────"
# Collect stats and sort by memory usage (descending)
docker stats --no-stream --format "{{.Name}}\t{{.MemUsage}}\t{{.MemPerc}}" | \
sort -t$'\t' -k2 -h -r | \
while IFS=$'\t' read -r name usage perc; do
# Extract just the usage part (before " / ")
mem_used=$(echo "$usage" | cut -d'/' -f1 | xargs)
mem_limit=$(echo "$usage" | cut -d'/' -f2 | xargs)
perc_num=$(echo "$perc" | tr -d '%')
# Color based on percentage
if (( $(echo "$perc_num > 80" | bc -l 2>/dev/null || echo 0) )); then
COLOR=$RED
elif (( $(echo "$perc_num > 50" | bc -l 2>/dev/null || echo 0) )); then
COLOR=$YELLOW
else
COLOR=$GREEN
fi
printf "%-40s %10s %10s ${COLOR}%10s${RESET}\n" "$name" "$mem_used" "$mem_limit" "$perc"
done
echo ""
echo -e "${BOLD}── Category Summary ──${RESET}"
echo ""
# Category totals using docker stats
get_category_mem() {
local pattern="$1"
docker stats --no-stream --format "{{.Name}}\t{{.MemUsage}}" | \
grep "$pattern" | \
awk -F'\t' '{
split($2, a, "/");
gsub(/[[:space:]]/, "", a[1]);
val = a[1];
if (index(val, "GiB") > 0) { gsub(/GiB/, "", val); total += val * 1024; }
else if (index(val, "MiB") > 0) { gsub(/MiB/, "", val); total += val; }
else if (index(val, "KiB") > 0) { gsub(/KiB/, "", val); total += val / 1024; }
} END { printf "%.0f", total }'
}
infra=$(get_category_mem "mana-infra")
core=$(get_category_mem "mana-core\|mana-auth\|mana-credits\|mana-user\|mana-subscriptions\|mana-analytics\|mana-api-gateway\|mana-crawler\|mana-service")
matrix=$(get_category_mem "mana-matrix")
apps=$(get_category_mem "mana-app")
monitoring=$(get_category_mem "mana-mon")
games=$(get_category_mem "mana-game")
auto=$(get_category_mem "mana-auto")
total=$((infra + core + matrix + apps + monitoring + games + auto))
printf "%-25s %8s MiB\n" "Infrastructure:" "$infra"
printf "%-25s %8s MiB\n" "Core Services:" "$core"
printf "%-25s %8s MiB\n" "Matrix Stack:" "$matrix"
printf "%-25s %8s MiB\n" "Web Apps:" "$apps"
printf "%-25s %8s MiB\n" "Monitoring:" "$monitoring"
printf "%-25s %8s MiB\n" "Games:" "$games"
printf "%-25s %8s MiB\n" "Automation:" "$auto"
echo "─────────────────────────────────────"
printf "${BOLD}%-25s %8s MiB (%.1f GiB)${RESET}\n" "TOTAL:" "$total" "$(echo "scale=1; $total / 1024" | bc)"
echo ""
echo -e "${BOLD}── Container Count ──${RESET}"
echo ""
running=$(docker ps -q | wc -l | xargs)
stopped=$(docker ps -aq --filter "status=exited" | wc -l | xargs)
echo "Running: $running | Stopped: $stopped | Total: $((running + stopped))"
echo ""
echo -e "${BOLD}── Recommendations ──${RESET}"
echo ""
if [ "$total" -gt 10240 ]; then
echo -e "${RED}WARNING: Total usage exceeds 10 GiB — builds will struggle${RESET}"
elif [ "$total" -gt 8192 ]; then
echo -e "${YELLOW}NOTICE: Usage above 8 GiB — builds may need monitoring stopped${RESET}"
else
echo -e "${GREEN}OK: Usage under 8 GiB — sufficient headroom for builds${RESET}"
fi
# Watch mode
if [ "${1:-}" = "--watch" ]; then
echo ""
echo -e "${CYAN}Entering watch mode (Ctrl+C to exit)...${RESET}"
echo ""
docker stats --format "table {{.Name}}\t{{.MemUsage}}\t{{.MemPerc}}\t{{.CPUPerc}}"
fi