#!/bin/bash # ManaCore Container Health Enforcer # Ensures all containers are actually running, not just created # # This script detects containers that are stuck in "Created" or "Exited" # status and automatically starts them. # # Run via LaunchD every 5 minutes or after system startup. set -e # Ensure PATH includes docker export PATH="/usr/local/bin:/opt/homebrew/bin:$PATH" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" COMPOSE_FILE="$PROJECT_ROOT/docker-compose.macmini.yml" ENV_FILE="$PROJECT_ROOT/.env.macmini" LOG_FILE="/tmp/manacore-container-health.log" # Load notification config if exists if [ -f "$PROJECT_ROOT/.env.notifications" ]; then source "$PROJECT_ROOT/.env.notifications" fi log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE" } send_notification() { local message="$1" # Telegram if [ -n "$TELEGRAM_BOT_TOKEN" ] && [ -n "$TELEGRAM_CHAT_ID" ]; then curl -s -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \ -d "chat_id=${TELEGRAM_CHAT_ID}" \ -d "text=${message}" \ -d "parse_mode=HTML" \ >/dev/null 2>&1 || true fi # ntfy if [ -n "$NTFY_TOPIC" ]; then curl -s -d "$message" \ -H "Title: ManaCore Container Fixed" \ -H "Priority: default" \ -H "Tags: white_check_mark" \ "https://ntfy.sh/$NTFY_TOPIC" >/dev/null 2>&1 || true fi } # Check if docker is running if ! docker info >/dev/null 2>&1; then log "ERROR: Docker is not running" exit 1 fi # Get containers that are NOT running (Created, Exited, etc.) # Filter only mana-* containers from our compose file STUCK_CONTAINERS=$(docker ps -a --filter "status=created" --filter "status=exited" --format "{{.Names}}" | grep "^mana-" || true) if [ -z "$STUCK_CONTAINERS" ]; then log "OK: All containers are running" exit 0 fi log "WARNING: Found containers not running:" echo "$STUCK_CONTAINERS" | while read container; do STATUS=$(docker inspect "$container" --format '{{.State.Status}}' 2>/dev/null || echo "unknown") log " - $container (status: $STATUS)" done # Start the stuck containers using docker compose log "Starting stuck containers via docker compose..." cd "$PROJECT_ROOT" # Use docker compose up for the specific services # This ensures dependencies are respected for container in $STUCK_CONTAINERS; do # Extract service name from container name (remove mana-app- or mana-* prefix) # Container naming: mana-{category}-{service} or mana-app-{service}-{type} SERVICE_NAME="" case "$container" in mana-app-todo-web) SERVICE_NAME="todo-web" ;; mana-app-todo-backend) SERVICE_NAME="todo-backend" ;; mana-app-chat-web) SERVICE_NAME="chat-web" ;; mana-app-chat-backend) SERVICE_NAME="chat-backend" ;; mana-app-calendar-web) SERVICE_NAME="calendar-web" ;; mana-app-calendar-backend) SERVICE_NAME="calendar-backend" ;; mana-app-clock-web) SERVICE_NAME="clock-web" ;; mana-app-clock-backend) SERVICE_NAME="clock-backend" ;; mana-app-contacts-web) SERVICE_NAME="contacts-web" ;; mana-app-contacts-backend) SERVICE_NAME="contacts-backend" ;; mana-app-storage-web) SERVICE_NAME="storage-web" ;; mana-app-storage-backend) SERVICE_NAME="storage-backend" ;; mana-app-presi-web) SERVICE_NAME="presi-web" ;; mana-app-presi-backend) SERVICE_NAME="presi-backend" ;; mana-app-nutriphi-web) SERVICE_NAME="nutriphi-web" ;; mana-app-nutriphi-backend) SERVICE_NAME="nutriphi-backend" ;; mana-app-skilltree-web) SERVICE_NAME="skilltree-web" ;; mana-app-skilltree-backend) SERVICE_NAME="skilltree-backend" ;; mana-app-photos-web) SERVICE_NAME="photos-web" ;; mana-app-photos-backend) SERVICE_NAME="photos-backend" ;; mana-app-web) SERVICE_NAME="mana-web" ;; mana-core-auth) SERVICE_NAME="mana-auth" ;; mana-core-gateway) SERVICE_NAME="api-gateway" ;; mana-core-search) SERVICE_NAME="mana-search" ;; mana-core-searxng) SERVICE_NAME="searxng" ;; mana-core-media) SERVICE_NAME="mana-media" ;; mana-infra-postgres) SERVICE_NAME="postgres" ;; mana-infra-redis) SERVICE_NAME="redis" ;; mana-infra-minio) SERVICE_NAME="minio" ;; mana-matrix-synapse) SERVICE_NAME="synapse" ;; mana-matrix-element) SERVICE_NAME="element-web" ;; mana-matrix-web) SERVICE_NAME="matrix-web" ;; mana-matrix-bot-*) SERVICE_NAME="${container#mana-matrix-bot-}"; SERVICE_NAME="matrix-${SERVICE_NAME}-bot" ;; mana-mon-*) SERVICE_NAME="${container#mana-mon-}" ;; mana-auto-*) SERVICE_NAME="${container#mana-auto-}" ;; mana-service-*) SERVICE_NAME="${container#mana-service-}" ;; mana-app-llm-playground) SERVICE_NAME="llm-playground" ;; *) log " Unknown container pattern: $container, trying direct start" docker start "$container" 2>&1 || true continue ;; esac if [ -n "$SERVICE_NAME" ]; then log " Starting service: $SERVICE_NAME" docker compose -f "$COMPOSE_FILE" --env-file "$ENV_FILE" up -d "$SERVICE_NAME" 2>&1 || { log " WARNING: Failed to start $SERVICE_NAME via compose, trying direct start" docker start "$container" 2>&1 || true } fi done # Wait for containers to start sleep 10 # Verify containers are now running STILL_STUCK=$(docker ps -a --filter "status=created" --filter "status=exited" --format "{{.Names}}" | grep "^mana-" || true) if [ -z "$STILL_STUCK" ]; then FIXED_MSG="Auto-fixed stuck containers: $(echo $STUCK_CONTAINERS | tr '\n' ', ')" log "SUCCESS: $FIXED_MSG" send_notification "🔧 ManaCore Auto-Recovery\n\n$FIXED_MSG" else log "ERROR: Some containers still not running:" echo "$STILL_STUCK" | while read container; do log " - $container" done send_notification "⚠️ ManaCore Container Issue\n\nContainers still stuck: $(echo $STILL_STUCK | tr '\n' ', ')" exit 1 fi