mirror of
https://github.com/Memo-2023/mana-monorepo.git
synced 2026-05-14 22:21:10 +02:00
feat(infra): smart build memory check and baseline monitoring script
build-app.sh now checks available RAM before builds and only stops monitoring containers when free memory is below 3 GB threshold. New memory-baseline.sh script measures per-container and per-category RAM usage for capacity planning. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Till JS
parent
2b448419ba
commit
f5cd77b2b0
10 changed files with 550 additions and 14 deletions
|
|
@ -1,12 +1,13 @@
|
|||
#!/bin/bash
|
||||
# Build and deploy specific app containers on the Mac Mini
|
||||
# Automatically frees RAM by stopping monitoring before build
|
||||
# Checks available memory and only stops monitoring if needed for builds.
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/mac-mini/build-app.sh todo-web
|
||||
# ./scripts/mac-mini/build-app.sh todo-web todo-backend
|
||||
# ./scripts/mac-mini/build-app.sh --all-web # rebuild all web apps
|
||||
# ./scripts/mac-mini/build-app.sh --base # rebuild base images only
|
||||
# ./scripts/mac-mini/build-app.sh --force-free # always stop monitoring
|
||||
|
||||
set -e
|
||||
|
||||
|
|
@ -15,6 +16,9 @@ PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
|||
COMPOSE_FILE="$PROJECT_ROOT/docker-compose.macmini.yml"
|
||||
DOCKER="${DOCKER_CMD:-/usr/local/bin/docker}"
|
||||
|
||||
# Minimum free memory (in MB) needed for a Docker build
|
||||
BUILD_MEM_THRESHOLD_MB=3000
|
||||
|
||||
# Monitoring containers (by container name — more reliable than compose service names)
|
||||
MONITORING_CONTAINERS=(
|
||||
mana-mon-grafana
|
||||
|
|
@ -47,14 +51,56 @@ cleanup() {
|
|||
# Always restart monitoring on exit (success, failure, or interrupt)
|
||||
trap cleanup EXIT
|
||||
|
||||
stop_monitoring() {
|
||||
echo "=== Stopping monitoring to free RAM ==="
|
||||
get_available_memory_mb() {
|
||||
# Get Colima VM total memory and current Docker usage
|
||||
local vm_total_mb
|
||||
vm_total_mb=$(colima list -j 2>/dev/null | python3 -c "import sys,json; d=json.load(sys.stdin); print(int(d[0].get('memory',0) / 1048576))" 2>/dev/null || echo "12288")
|
||||
|
||||
# Sum all container memory usage
|
||||
local used_mb
|
||||
used_mb=$($DOCKER stats --no-stream --format '{{.MemUsage}}' 2>/dev/null | \
|
||||
awk '{
|
||||
split($1, a, "/");
|
||||
val = a[1];
|
||||
gsub(/[[:space:]]/, "", val);
|
||||
if (index(val, "GiB") > 0) { gsub(/GiB/, "", val); total += val * 1024; }
|
||||
else if (index(val, "MiB") > 0) { gsub(/MiB/, "", val); total += val; }
|
||||
else if (index(val, "KiB") > 0) { gsub(/KiB/, "", val); total += val / 1024; }
|
||||
} END { printf "%.0f", total }')
|
||||
|
||||
echo $(( vm_total_mb - used_mb ))
|
||||
}
|
||||
|
||||
maybe_stop_monitoring() {
|
||||
local force="${1:-false}"
|
||||
|
||||
if [ "$force" = "true" ]; then
|
||||
echo "=== Force-freeing RAM (--force-free) ==="
|
||||
stop_monitoring_now
|
||||
return
|
||||
fi
|
||||
|
||||
echo "=== Checking available memory ==="
|
||||
local avail_mb
|
||||
avail_mb=$(get_available_memory_mb)
|
||||
echo " Available: ${avail_mb} MB (need: ${BUILD_MEM_THRESHOLD_MB} MB)"
|
||||
|
||||
if [ "$avail_mb" -lt "$BUILD_MEM_THRESHOLD_MB" ]; then
|
||||
echo " → Not enough — stopping monitoring to free RAM"
|
||||
stop_monitoring_now
|
||||
else
|
||||
echo " → Sufficient — monitoring stays running ✓"
|
||||
echo ""
|
||||
# Still prune build cache
|
||||
$DOCKER builder prune -f 2>/dev/null | tail -1 || true
|
||||
fi
|
||||
}
|
||||
|
||||
stop_monitoring_now() {
|
||||
$DOCKER stop "${MONITORING_CONTAINERS[@]}" 2>/dev/null || true
|
||||
MONITORING_STOPPED=true
|
||||
|
||||
# Also prune dangling build cache
|
||||
$DOCKER builder prune -f 2>/dev/null | tail -1 || true
|
||||
echo "RAM freed."
|
||||
echo " RAM freed."
|
||||
echo ""
|
||||
}
|
||||
|
||||
|
|
@ -104,17 +150,30 @@ if [ $# -eq 0 ]; then
|
|||
echo " $0 --base # Rebuild base images"
|
||||
echo " $0 --all-web # Rebuild all web apps"
|
||||
echo " $0 mana-matrix-bot # Build & restart consolidated Matrix bot (Go)"
|
||||
echo " $0 --force-free todo-web # Force stop monitoring before build"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cd "$PROJECT_ROOT"
|
||||
|
||||
# Check for --force-free flag
|
||||
FORCE_FREE=false
|
||||
ARGS=()
|
||||
for arg in "$@"; do
|
||||
if [ "$arg" = "--force-free" ]; then
|
||||
FORCE_FREE=true
|
||||
else
|
||||
ARGS+=("$arg")
|
||||
fi
|
||||
done
|
||||
set -- "${ARGS[@]}"
|
||||
|
||||
# Pull latest code
|
||||
echo "=== Pulling latest code ==="
|
||||
git pull
|
||||
|
||||
# Free RAM
|
||||
stop_monitoring
|
||||
# Smart memory check — only stop monitoring if needed
|
||||
maybe_stop_monitoring "$FORCE_FREE"
|
||||
|
||||
case "$1" in
|
||||
--base)
|
||||
|
|
|
|||
123
scripts/mac-mini/memory-baseline.sh
Executable file
123
scripts/mac-mini/memory-baseline.sh
Executable file
|
|
@ -0,0 +1,123 @@
|
|||
#!/bin/bash
|
||||
# Memory Baseline Script
|
||||
# Run on the Mac Mini to measure actual container memory usage
|
||||
# Usage: ./scripts/mac-mini/memory-baseline.sh [--watch]
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
BOLD="\033[1m"
|
||||
GREEN="\033[0;32m"
|
||||
YELLOW="\033[0;33m"
|
||||
RED="\033[0;31m"
|
||||
CYAN="\033[0;36m"
|
||||
RESET="\033[0m"
|
||||
|
||||
echo -e "${BOLD}╔══════════════════════════════════════════════════╗${RESET}"
|
||||
echo -e "${BOLD}║ Mana Docker Memory Baseline Report ║${RESET}"
|
||||
echo -e "${BOLD}╚══════════════════════════════════════════════════╝${RESET}"
|
||||
echo ""
|
||||
echo -e "${CYAN}Timestamp:${RESET} $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo ""
|
||||
|
||||
# Get Colima VM memory
|
||||
COLIMA_MEM=$(colima list -j 2>/dev/null | python3 -c "import sys,json; d=json.load(sys.stdin); print(d[0].get('memory',0))" 2>/dev/null || echo "unknown")
|
||||
if [ "$COLIMA_MEM" != "unknown" ] && [ "$COLIMA_MEM" -gt 0 ] 2>/dev/null; then
|
||||
COLIMA_MEM_GB=$(echo "scale=1; $COLIMA_MEM / 1073741824" | bc 2>/dev/null || echo "$COLIMA_MEM")
|
||||
echo -e "${CYAN}Colima VM Memory:${RESET} ${COLIMA_MEM_GB} GB"
|
||||
else
|
||||
echo -e "${CYAN}Colima VM Memory:${RESET} (run 'colima list' to check)"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Docker stats snapshot (no-stream = one-shot)
|
||||
echo -e "${BOLD}── Per-Container Memory Usage ──${RESET}"
|
||||
echo ""
|
||||
printf "${BOLD}%-40s %10s %10s %10s${RESET}\n" "CONTAINER" "MEM USAGE" "MEM LIMIT" "MEM %"
|
||||
echo "────────────────────────────────────────────────────────────────────────"
|
||||
|
||||
# Collect stats and sort by memory usage (descending)
|
||||
docker stats --no-stream --format "{{.Name}}\t{{.MemUsage}}\t{{.MemPerc}}" | \
|
||||
sort -t$'\t' -k2 -h -r | \
|
||||
while IFS=$'\t' read -r name usage perc; do
|
||||
# Extract just the usage part (before " / ")
|
||||
mem_used=$(echo "$usage" | cut -d'/' -f1 | xargs)
|
||||
mem_limit=$(echo "$usage" | cut -d'/' -f2 | xargs)
|
||||
perc_num=$(echo "$perc" | tr -d '%')
|
||||
|
||||
# Color based on percentage
|
||||
if (( $(echo "$perc_num > 80" | bc -l 2>/dev/null || echo 0) )); then
|
||||
COLOR=$RED
|
||||
elif (( $(echo "$perc_num > 50" | bc -l 2>/dev/null || echo 0) )); then
|
||||
COLOR=$YELLOW
|
||||
else
|
||||
COLOR=$GREEN
|
||||
fi
|
||||
|
||||
printf "%-40s %10s %10s ${COLOR}%10s${RESET}\n" "$name" "$mem_used" "$mem_limit" "$perc"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo -e "${BOLD}── Category Summary ──${RESET}"
|
||||
echo ""
|
||||
|
||||
# Category totals using docker stats
|
||||
get_category_mem() {
|
||||
local pattern="$1"
|
||||
docker stats --no-stream --format "{{.Name}}\t{{.MemUsage}}" | \
|
||||
grep "$pattern" | \
|
||||
awk -F'\t' '{
|
||||
split($2, a, "/");
|
||||
gsub(/[[:space:]]/, "", a[1]);
|
||||
val = a[1];
|
||||
if (index(val, "GiB") > 0) { gsub(/GiB/, "", val); total += val * 1024; }
|
||||
else if (index(val, "MiB") > 0) { gsub(/MiB/, "", val); total += val; }
|
||||
else if (index(val, "KiB") > 0) { gsub(/KiB/, "", val); total += val / 1024; }
|
||||
} END { printf "%.0f", total }'
|
||||
}
|
||||
|
||||
infra=$(get_category_mem "mana-infra")
|
||||
core=$(get_category_mem "mana-core\|mana-auth\|mana-credits\|mana-user\|mana-subscriptions\|mana-analytics\|mana-api-gateway\|mana-crawler\|mana-service")
|
||||
matrix=$(get_category_mem "mana-matrix")
|
||||
apps=$(get_category_mem "mana-app")
|
||||
monitoring=$(get_category_mem "mana-mon")
|
||||
games=$(get_category_mem "mana-game")
|
||||
auto=$(get_category_mem "mana-auto")
|
||||
|
||||
total=$((infra + core + matrix + apps + monitoring + games + auto))
|
||||
|
||||
printf "%-25s %8s MiB\n" "Infrastructure:" "$infra"
|
||||
printf "%-25s %8s MiB\n" "Core Services:" "$core"
|
||||
printf "%-25s %8s MiB\n" "Matrix Stack:" "$matrix"
|
||||
printf "%-25s %8s MiB\n" "Web Apps:" "$apps"
|
||||
printf "%-25s %8s MiB\n" "Monitoring:" "$monitoring"
|
||||
printf "%-25s %8s MiB\n" "Games:" "$games"
|
||||
printf "%-25s %8s MiB\n" "Automation:" "$auto"
|
||||
echo "─────────────────────────────────────"
|
||||
printf "${BOLD}%-25s %8s MiB (%.1f GiB)${RESET}\n" "TOTAL:" "$total" "$(echo "scale=1; $total / 1024" | bc)"
|
||||
|
||||
echo ""
|
||||
echo -e "${BOLD}── Container Count ──${RESET}"
|
||||
echo ""
|
||||
running=$(docker ps -q | wc -l | xargs)
|
||||
stopped=$(docker ps -aq --filter "status=exited" | wc -l | xargs)
|
||||
echo "Running: $running | Stopped: $stopped | Total: $((running + stopped))"
|
||||
|
||||
echo ""
|
||||
echo -e "${BOLD}── Recommendations ──${RESET}"
|
||||
echo ""
|
||||
|
||||
if [ "$total" -gt 10240 ]; then
|
||||
echo -e "${RED}WARNING: Total usage exceeds 10 GiB — builds will struggle${RESET}"
|
||||
elif [ "$total" -gt 8192 ]; then
|
||||
echo -e "${YELLOW}NOTICE: Usage above 8 GiB — builds may need monitoring stopped${RESET}"
|
||||
else
|
||||
echo -e "${GREEN}OK: Usage under 8 GiB — sufficient headroom for builds${RESET}"
|
||||
fi
|
||||
|
||||
# Watch mode
|
||||
if [ "${1:-}" = "--watch" ]; then
|
||||
echo ""
|
||||
echo -e "${CYAN}Entering watch mode (Ctrl+C to exit)...${RESET}"
|
||||
echo ""
|
||||
docker stats --format "table {{.Name}}\t{{.MemUsage}}\t{{.MemPerc}}\t{{.CPUPerc}}"
|
||||
fi
|
||||
Loading…
Add table
Add a link
Reference in a new issue