From 4fce6a3edea0c827fb1771438cdb35d5b447cf4b Mon Sep 17 00:00:00 2001 From: Till JS Date: Wed, 8 Apr 2026 17:50:37 +0200 Subject: [PATCH] feat(env): persistent dev secrets via .env.secrets override MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Local dev secrets like MANA_STT_API_KEY had no persistent home — they lived only in the gitignored, generator-overwritten per-app .env files. Every `pnpm setup:env` wiped them, so devs had to re-paste keys after any env regeneration. Same recurring friction for MANA_LLM_API_KEY, MANA_AUTH_KEK, OAuth keys, etc. New layer: `.env.secrets` at the repo root. - Gitignored, optional, never required for the build to pass - Read by generate-env.mjs AFTER .env.development; non-empty values override the matching key, so the merged result drives every per-app .env the generator writes - Empty values fall through to the .env.development defaults — a freshly-copied .env.secrets.example is a no-op - One source of truth for all dev secrets, propagated to every app with one `pnpm setup:env` Files: - `.env.secrets.example` — committed template documenting all known secret keys (mana-stt, mana-llm, auth KEK, sync JWT, MinIO, third- party APIs). Devs `cp .env.secrets.example .env.secrets` and fill in. - `.gitignore` — ignores .env.secrets, allows .env.secrets.example - `scripts/generate-env.mjs` — loads .env.secrets if present, prints "Loaded N secrets from .env.secrets" so devs see the override taking effect - `scripts/setup-secrets.mjs` + `pnpm setup:secrets` — convenience script that SSHes to mana-server, greps the prod .env for the keys defined in .env.secrets.example, and writes them locally. Confirms before overwriting an existing .env.secrets unless --force is set; reports which keys couldn't be found on the remote so devs know what's left to fill manually - `docs/LOCAL_DEVELOPMENT.md` + `docs/ENVIRONMENT_VARIABLES.md` — walk-through and architecture diagram update Verified end-to-end: - `rm .env.secrets apps/mana/apps/web/.env && pnpm setup:env` → STT key empty (no regression for devs who haven't opted in) - `pnpm setup:secrets --force && pnpm setup:env` → STT key propagated, "Loaded 3 secrets from .env.secrets" in output - POST /api/v1/voice/transcribe with a real audio file → full transcript back via gpu-stt.mana.how, end-to-end working Co-Authored-By: Claude Opus 4.6 (1M context) --- .env.secrets.example | 87 ++++++++++++++++++ .gitignore | 4 +- docs/ENVIRONMENT_VARIABLES.md | 22 +++-- docs/LOCAL_DEVELOPMENT.md | 40 ++++++++- package.json | 1 + scripts/generate-env.mjs | 24 +++++ scripts/setup-secrets.mjs | 165 ++++++++++++++++++++++++++++++++++ 7 files changed, 335 insertions(+), 8 deletions(-) create mode 100644 .env.secrets.example create mode 100644 scripts/setup-secrets.mjs diff --git a/.env.secrets.example b/.env.secrets.example new file mode 100644 index 000000000..5b65fd04f --- /dev/null +++ b/.env.secrets.example @@ -0,0 +1,87 @@ +# ============================================================================= +# .env.secrets — Local secret overrides for development +# ============================================================================= +# +# Copy this file to `.env.secrets` (gitignored) and fill in real values. +# Anything you set here overrides the matching key in `.env.development` +# during `pnpm setup:env` and gets propagated into every per-app .env that +# the generator writes. This is the persistent place to put dev secrets — +# unlike per-app `.env` files, which are wiped and regenerated on every +# `pnpm setup:env`. +# +# How to populate (one-shot from the prod machine): +# +# pnpm setup:secrets +# +# That command SSHes to mana-server, greps the prod `.env` for the keys +# below, and writes them here. You can also paste values manually if you +# don't have SSH access — anything in this file overrides the defaults. +# +# IMPORTANT: +# - This file is gitignored. Never commit real values. +# - Only put SECRETS here. Non-secret config (URLs, ports, feature flags) +# belongs in `.env.development` so the whole team shares the same setup. +# - Empty values fall through to whatever `.env.development` defines. +# + +# ----------------------------------------------------------------------------- +# mana-stt — Speech-to-Text proxy on the Windows GPU box +# Used by /api/v1/voice/transcribe in the mana-web app. +# Source of truth: services/mana-stt/.env on the GPU box (API_KEYS=…) +# ----------------------------------------------------------------------------- +MANA_STT_API_KEY= + +# ----------------------------------------------------------------------------- +# mana-llm — LLM gateway. Only required when pointing at gpu-llm.mana.how +# (which enforces X-API-Key). The public llm.mana.how is open and needs +# no key — leave empty unless you've explicitly switched MANA_LLM_URL. +# ----------------------------------------------------------------------------- +MANA_LLM_API_KEY= + +# ----------------------------------------------------------------------------- +# mana-auth — Master encryption key used to wrap user vault keys. +# Production: rotated via the mana-auth deploy. Local dev can leave empty +# (the auth service falls back to a fixed dev KEK in NODE_ENV=development). +# ----------------------------------------------------------------------------- +MANA_AUTH_KEK= + +# ----------------------------------------------------------------------------- +# Better Auth — session signing secret. Local dev defaults to "dev-secret- +# change-me" so the auth service starts cleanly; only override if you need +# tokens to verify against the prod issuer. +# ----------------------------------------------------------------------------- +BETTER_AUTH_SECRET= + +# ----------------------------------------------------------------------------- +# Sync engine — JWT signing key shared between mana-auth and mana-sync. +# Local dev defaults to a fixed dev key in .env.development. +# ----------------------------------------------------------------------------- +MANA_SYNC_JWT_SECRET= + +# ----------------------------------------------------------------------------- +# Service-to-service auth — used by backends to call other Mana services +# without going through user JWTs. Required for some prod paths only. +# ----------------------------------------------------------------------------- +MANA_SERVICE_KEY= + +# ----------------------------------------------------------------------------- +# Object storage — MinIO credentials. Local dev uses minioadmin/minioadmin +# from `.env.development`; production uses real keys from this file. +# ----------------------------------------------------------------------------- +MINIO_ACCESS_KEY= +MINIO_SECRET_KEY= + +# ----------------------------------------------------------------------------- +# Third-party APIs — only set when you actually need them locally +# ----------------------------------------------------------------------------- +OPENROUTER_API_KEY= +GOOGLE_GENAI_API_KEY= +GOOGLE_API_KEY= +GROQ_API_KEY= +TOGETHER_API_KEY= + +# ----------------------------------------------------------------------------- +# Sentry / GlitchTip DSNs — leave empty in dev unless you actively want +# local errors to land in the shared error tracker +# ----------------------------------------------------------------------------- +GLITCHTIP_DSN_MANA_WEB= diff --git a/.gitignore b/.gitignore index 1101cf51a..6d4526ca0 100644 --- a/.gitignore +++ b/.gitignore @@ -28,9 +28,11 @@ ios/ .env.test.local .env.production.local .env*.local +.env.secrets -# BUT commit the central development env file +# BUT commit the central development env file + the secrets template !.env.development +!.env.secrets.example # IDE .idea/ diff --git a/docs/ENVIRONMENT_VARIABLES.md b/docs/ENVIRONMENT_VARIABLES.md index e46f41367..7a4843a2a 100644 --- a/docs/ENVIRONMENT_VARIABLES.md +++ b/docs/ENVIRONMENT_VARIABLES.md @@ -17,16 +17,26 @@ That's it! All app-specific `.env` files are generated from `.env.development`. ## How It Works ``` -.env.development # Central config (committed) +.env.development # Central config (committed, no secrets) + │ + ├── .env.secrets # Optional gitignored override (your API keys) + ▼ +scripts/generate-env.mjs # Merges + transforms variables │ ▼ -scripts/generate-env.mjs # Transforms variables - │ - ▼ -apps/**/apps/**/.env # Generated files (gitignored) +apps/**/apps/**/.env # Generated files (gitignored) ``` -The generator reads `.env.development` and creates app-specific `.env` files with the correct prefixes for each platform: +The generator reads `.env.development` first, then layers `.env.secrets` (if it exists) on +top — non-empty values in `.env.secrets` override the matching key in `.env.development`. +This is where personal dev secrets like `MANA_STT_API_KEY` live, so you don't have to +re-paste them into per-app `.env` files after every `pnpm setup:env`. + +To populate `.env.secrets` from the Mac Mini in one shot, run `pnpm setup:secrets` (see +[`docs/LOCAL_DEVELOPMENT.md`](LOCAL_DEVELOPMENT.md#personal-dev-secrets--envsecrets) for +the full walk-through). + +The generator then creates app-specific `.env` files with the correct prefixes for each platform: | Platform | Prefix | Example | |----------|--------|---------| diff --git a/docs/LOCAL_DEVELOPMENT.md b/docs/LOCAL_DEVELOPMENT.md index 99fbce5bf..e02aff375 100644 --- a/docs/LOCAL_DEVELOPMENT.md +++ b/docs/LOCAL_DEVELOPMENT.md @@ -101,12 +101,50 @@ pnpm docker:up # 2. Build mana-sync (first time only, or after Go changes) pnpm dev:sync:build -# 3. Generate environment files (runs automatically on pnpm install) +# 3. (Optional) Pull dev secrets from the Mac Mini into .env.secrets +pnpm setup:secrets + +# 4. Generate environment files (runs automatically on pnpm install) pnpm setup:env ``` For `dev:*:local`, only mana-sync needs to be built. No Docker required unless your server uses a database. +### Personal dev secrets — `.env.secrets` + +`.env.development` is committed and contains non-secret defaults. Real API keys (mana-stt, +gpu-llm, OpenRouter, etc.) live in a separate gitignored `.env.secrets` at the repo root. +The env generator merges this file on top of `.env.development`, so any key you set there +gets propagated into every per-app `.env` on `pnpm setup:env` — no more re-pasting keys +into individual app folders after every regeneration. + +**One-time setup:** + +```bash +# Pulls keys from ~/projects/mana-monorepo/.env on the Mac Mini via SSH +pnpm setup:secrets + +# Then propagate into per-app .env files +pnpm setup:env +``` + +`pnpm setup:secrets` reads `.env.secrets.example` to know which keys to look for, asks +before overwriting an existing `.env.secrets`, and reports which keys it couldn't find on +the remote (some service-specific keys live in their own per-service `.env` files on the +Mac Mini and need to be filled in manually). + +If you don't have SSH access to `mana-server`, copy the template and fill values manually: + +```bash +cp .env.secrets.example .env.secrets +$EDITOR .env.secrets +pnpm setup:env +``` + +Empty values in `.env.secrets` are no-ops — they fall through to the `.env.development` +defaults. So a freshly-copied template doesn't change anything until you start filling +keys in. + ## Database Setup Commands ### Individual Service Setup diff --git a/package.json b/package.json index 62985bbbd..828ec6243 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "audit:deps": "node scripts/audit-workspace-deps.mjs", "generate:dockerfiles": "node scripts/generate-dockerfiles.mjs", "setup:env": "node scripts/generate-env.mjs", + "setup:secrets": "node scripts/setup-secrets.mjs", "setup:db": "./scripts/setup-databases.sh", "setup:db:chat": "./scripts/setup-databases.sh chat", "setup:db:auth": "./scripts/setup-databases.sh auth", diff --git a/scripts/generate-env.mjs b/scripts/generate-env.mjs index e236ff0d0..95e0e8c70 100644 --- a/scripts/generate-env.mjs +++ b/scripts/generate-env.mjs @@ -16,6 +16,11 @@ import { fileURLToPath } from 'url'; const __dirname = dirname(fileURLToPath(import.meta.url)); const ROOT_DIR = join(__dirname, '..'); const ENV_FILE = join(ROOT_DIR, '.env.development'); +// Optional gitignored override for personal dev secrets. Keys defined +// here win over .env.development, so devs can keep API keys in one +// place instead of re-pasting them into per-app .env files after every +// `pnpm setup:env`. See .env.secrets.example for the template. +const SECRETS_FILE = join(ROOT_DIR, '.env.secrets'); // Parse a .env file into an object function parseEnvFile(content) { @@ -763,6 +768,25 @@ function main() { const sourceContent = readFileSync(ENV_FILE, 'utf-8'); const sourceEnv = parseEnvFile(sourceContent); + // Layer .env.secrets (gitignored) on top — only non-empty values + // override. An empty value in .env.secrets is treated as "use the + // .env.development default", so a freshly-copied .env.secrets.example + // (all keys present, all values empty) is a no-op. + let secretsLoaded = 0; + if (existsSync(SECRETS_FILE)) { + const secretsContent = readFileSync(SECRETS_FILE, 'utf-8'); + const secretsEnv = parseEnvFile(secretsContent); + for (const [key, value] of Object.entries(secretsEnv)) { + if (value !== '' && value !== undefined) { + sourceEnv[key] = value; + secretsLoaded++; + } + } + console.log( + `Loaded ${secretsLoaded} secret${secretsLoaded === 1 ? '' : 's'} from .env.secrets\n` + ); + } + let generated = 0; let skipped = 0; diff --git a/scripts/setup-secrets.mjs b/scripts/setup-secrets.mjs new file mode 100644 index 000000000..ad5a77baa --- /dev/null +++ b/scripts/setup-secrets.mjs @@ -0,0 +1,165 @@ +#!/usr/bin/env node + +/** + * setup-secrets.mjs — Pull dev secrets from the Mac Mini into .env.secrets + * + * SSHes to mana-server, reads ~/projects/mana-monorepo/.env, and writes + * the secret-shaped keys into a local .env.secrets file. Skips keys that + * are already populated locally so re-running is safe. + * + * Usage: pnpm setup:secrets + * + * Requires: + * - SSH access to `mana-server` (Cloudflare Tunnel) + * - .env.secrets.example as the canonical list of secret keys + * + * Refuses to overwrite an existing .env.secrets without --force, so a + * fat-fingered re-run can't blow away values you've manually edited. + */ + +import { readFileSync, writeFileSync, existsSync } from 'fs'; +import { execSync } from 'child_process'; +import { dirname, join } from 'path'; +import { fileURLToPath } from 'url'; +import { createInterface } from 'readline'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const ROOT_DIR = join(__dirname, '..'); +const TEMPLATE_FILE = join(ROOT_DIR, '.env.secrets.example'); +const TARGET_FILE = join(ROOT_DIR, '.env.secrets'); +const REMOTE_HOST = 'mana-server'; +const REMOTE_ENV_PATH = '~/projects/mana-monorepo/.env'; + +const FORCE = process.argv.includes('--force'); + +function parseEnvKeys(content) { + // Returns the ordered list of keys defined in a .env file (skips + // comments and blanks). We use this to drive what we ask the remote + // for, so a new key in .env.secrets.example is automatically picked + // up next time someone runs setup:secrets. + const keys = []; + for (const line of content.split('\n')) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith('#')) continue; + const m = trimmed.match(/^([A-Z_][A-Z0-9_]*)=/); + if (m) keys.push(m[1]); + } + return keys; +} + +function parseEnvFile(content) { + const env = {}; + for (const line of content.split('\n')) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith('#')) continue; + const m = trimmed.match(/^([^=]+)=(.*)$/); + if (m) { + let [, key, value] = m; + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + value = value.slice(1, -1); + } + env[key.trim()] = value; + } + } + return env; +} + +async function confirm(question) { + const rl = createInterface({ input: process.stdin, output: process.stdout }); + return new Promise((resolve) => { + rl.question(`${question} [y/N] `, (answer) => { + rl.close(); + resolve(/^y(es)?$/i.test(answer.trim())); + }); + }); +} + +async function main() { + if (!existsSync(TEMPLATE_FILE)) { + console.error(`Error: ${TEMPLATE_FILE} not found.`); + console.error('This script reads .env.secrets.example to know which keys to fetch.'); + process.exit(1); + } + + const templateContent = readFileSync(TEMPLATE_FILE, 'utf-8'); + const wantedKeys = parseEnvKeys(templateContent); + + if (existsSync(TARGET_FILE) && !FORCE) { + console.log(`.env.secrets already exists at ${TARGET_FILE}`); + const ok = await confirm('Overwrite with values pulled from mana-server?'); + if (!ok) { + console.log('Aborted. Pass --force to overwrite without prompting.'); + process.exit(0); + } + } + + console.log( + `Fetching ${wantedKeys.length} secret keys from ${REMOTE_HOST}:${REMOTE_ENV_PATH}...` + ); + + let remoteContent; + try { + remoteContent = execSync(`ssh ${REMOTE_HOST} 'cat ${REMOTE_ENV_PATH}'`, { + encoding: 'utf-8', + stdio: ['ignore', 'pipe', 'pipe'], + }); + } catch (e) { + console.error(`Failed to read remote env: ${e.message}`); + console.error('Check that `ssh mana-server` works from this machine.'); + process.exit(1); + } + + const remoteEnv = parseEnvFile(remoteContent); + + // Build the new .env.secrets by walking the template line-by-line so + // we preserve comments + ordering, but substitute real values for + // any KEY=… line whose key exists in the remote env. + const outputLines = []; + let filledCount = 0; + let missingCount = 0; + const missingKeys = []; + + for (const line of templateContent.split('\n')) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith('#')) { + outputLines.push(line); + continue; + } + const m = trimmed.match(/^([A-Z_][A-Z0-9_]*)=/); + if (!m) { + outputLines.push(line); + continue; + } + const key = m[1]; + if (key in remoteEnv && remoteEnv[key] !== '') { + const value = remoteEnv[key]; + const needsQuotes = value.includes(' ') || value.includes('#'); + outputLines.push(`${key}=${needsQuotes ? `"${value}"` : value}`); + filledCount++; + } else { + outputLines.push(`${key}=`); + missingCount++; + missingKeys.push(key); + } + } + + writeFileSync(TARGET_FILE, outputLines.join('\n')); + console.log(`\nWrote ${TARGET_FILE}`); + console.log(` ${filledCount} key${filledCount === 1 ? '' : 's'} populated from mana-server`); + if (missingCount > 0) { + console.log( + ` ${missingCount} key${missingCount === 1 ? '' : 's'} left empty (not present on mana-server):` + ); + for (const k of missingKeys) console.log(` - ${k}`); + console.log(' → fill these in manually if you need them locally.'); + } + console.log('\nNext step: run `pnpm setup:env` to propagate into per-app .env files.'); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +});