mirror of
https://github.com/Memo-2023/mana-monorepo.git
synced 2026-05-14 21:21:10 +02:00
feat(llm-aliases): M5 — migrate consumers to MANA_LLM aliases
Final milestone of docs/plans/llm-fallback-aliases.md. Every backend
caller now requests models via the `mana/<class>` alias system instead
of hardcoded `ollama/...` strings. mana-llm resolves aliases through
`services/mana-llm/aliases.yaml` with health-aware fallback (M3) and
emits resolved-model + fallback metrics (M4).
SSOT moved to `packages/shared-ai/src/llm-aliases.ts` so apps/api,
apps/mana/apps/web, and services/mana-ai all import the same
`MANA_LLM` constant via the existing `@mana/shared-ai` workspace
dependency. Three additional sites (memoro-server, mana-events,
mana-research) inline the alias string with a SSOT comment because
they don't pull @mana/shared-ai today.
Migrated 14 sites across 10 files:
- apps/api: writing(LONG_FORM), comic(STRUCTURED), context(FAST_TEXT),
food(VISION), plants(VISION), research orchestrator (3 tiers
collapsed to STRUCTURED+FAST_TEXT/LONG_FORM)
- apps/mana/apps/web: voice/parse-task + parse-habit (STRUCTURED)
- services/mana-ai: planner llm-client + tick.ts (REASONING)
- services/mana-events: website-extractor (STRUCTURED, inlined)
- services/mana-research: mana-llm client (FAST_TEXT, inlined)
- apps/memoro/apps/server: ai.ts (FAST_TEXT, inlined)
Legacy env-vars removed: WRITING_MODEL, COMIC_STORYBOARD_MODEL,
VISION_MODEL, MANA_LLM_DEFAULT_MODEL. The chain in aliases.yaml is
now the single tuning surface; SIGHUP reloads it without redeploys.
New `scripts/validate-llm-strings.mjs` regex-scans 2538 files for
hardcoded `<provider>/<model>` strings and fails the build if any
land outside the SSOT or the explicitly-allowed paths (image-gen
modules, model-inspector code, this validator itself, the registry).
Wired into `validate:all` next to the i18n + theme validators.
Verified: `pnpm validate:llm-strings` clean, `pnpm --filter @mana/api
type-check` clean, `pnpm --filter @mana/ai-service type-check`
clean. Web type-check has 2 pre-existing errors in
SettingsSidebar.svelte (i18n MessageFormatter type drift, last
touched in 988c17a67 — unrelated to this work).
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
8a49e3ffd5
commit
fea3adf5fe
19 changed files with 299 additions and 50 deletions
198
scripts/validate-llm-strings.mjs
Normal file
198
scripts/validate-llm-strings.mjs
Normal file
|
|
@ -0,0 +1,198 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Validate that no consumer code hardcodes provider/model strings.
|
||||
*
|
||||
* After M5 of `docs/plans/llm-fallback-aliases.md`, every backend caller
|
||||
* should request models via the `mana/<class>` aliases (resolved server-
|
||||
* side by mana-llm). Bare `provider/model` strings (`ollama/gemma3:4b`,
|
||||
* `groq/llama-3.3-70b-versatile`, …) are a sign someone bypassed the
|
||||
* registry — that path skips fallback, the health-cache, and the alias-
|
||||
* resolution metrics.
|
||||
*
|
||||
* The two legitimate places to keep concrete model strings are:
|
||||
*
|
||||
* - `services/mana-llm/aliases.yaml` — the registry itself
|
||||
* - `services/mana-llm/**` — provider adapters, tests, fixtures
|
||||
*
|
||||
* Anything else fails the check. Add the file to ALLOWED_PATHS below
|
||||
* with a comment if you have a justified reason.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/validate-llm-strings.mjs
|
||||
*/
|
||||
|
||||
import { readdirSync, readFileSync, statSync } from 'fs';
|
||||
import { dirname, join, relative, sep } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = join(__dirname, '..');
|
||||
|
||||
const RED = '\x1b[31m';
|
||||
const GREEN = '\x1b[32m';
|
||||
const YELLOW = '\x1b[33m';
|
||||
const DIM = '\x1b[2m';
|
||||
const BOLD = '\x1b[1m';
|
||||
const RESET = '\x1b[0m';
|
||||
|
||||
const SCAN_ROOTS = ['apps', 'services', 'packages'];
|
||||
const SCAN_EXTS = new Set(['.ts', '.tsx', '.mts', '.cts']);
|
||||
const SKIP_DIRS = new Set([
|
||||
'node_modules',
|
||||
'dist',
|
||||
'build',
|
||||
'.svelte-kit',
|
||||
'.next',
|
||||
'.turbo',
|
||||
'.vercel',
|
||||
'.vite',
|
||||
]);
|
||||
|
||||
/**
|
||||
* Paths (POSIX-form, relative to repo root) that are allowed to contain
|
||||
* concrete provider/model strings. Use slashes — the matcher normalises
|
||||
* Windows separators.
|
||||
*/
|
||||
const ALLOWED_PATHS = [
|
||||
// The registry itself.
|
||||
'services/mana-llm/aliases.yaml',
|
||||
|
||||
// mana-llm internals: provider adapters, router, config, tests.
|
||||
'services/mana-llm/',
|
||||
|
||||
// Picture module routes route between IMAGE generators (gpt-image,
|
||||
// gemini-3-pro-image-preview, …) — those are not LLM chat models
|
||||
// and don't go through mana-llm's chat-completions endpoint.
|
||||
'apps/api/src/modules/picture/routes.ts',
|
||||
|
||||
// generate-who-dossiers is an admin script with an explicit
|
||||
// `--model <name>` flag; the operator deliberately picks a strong
|
||||
// model for one-shot dossier authoring. Aliasing wouldn't help.
|
||||
'apps/api/scripts/generate-who-dossiers.ts',
|
||||
|
||||
// Chat / who modules INSPECT user-supplied model strings to gate
|
||||
// behaviour (prefix checks like `model.startsWith('ollama/')`); they
|
||||
// don't pick a model themselves.
|
||||
'apps/api/src/modules/chat/routes.ts',
|
||||
'apps/api/src/modules/who/routes.ts',
|
||||
|
||||
// Picture-Workbench client-side path (browser): the user picks a
|
||||
// concrete image model from a dropdown.
|
||||
'apps/mana/apps/web/src/lib/modules/picture/',
|
||||
|
||||
// llm-test playground in the web app — explicit model picker.
|
||||
'apps/mana/apps/web/src/routes/(app)/llm-test/',
|
||||
|
||||
// Validators / scripts can reference example strings in their own
|
||||
// docstrings.
|
||||
'scripts/',
|
||||
|
||||
// SSOT alias-constant file.
|
||||
'packages/shared-ai/src/llm-aliases.ts',
|
||||
];
|
||||
|
||||
/**
|
||||
* Concrete provider strings the validator hunts for. The pattern is a
|
||||
* literal `<provider>/...` token in source; matched by a regex that
|
||||
* requires a quote / backtick before the slash so we don't fire on
|
||||
* import paths like `from '../ollama/foo'`.
|
||||
*/
|
||||
const PROVIDERS = ['ollama', 'groq', 'openrouter', 'together'];
|
||||
// `google/` is intentionally not in PROVIDERS — Google is the namespace
|
||||
// for both Gemini text models AND Nano-Banana image models. Matching it
|
||||
// would yield too many false positives in image code paths.
|
||||
|
||||
const PROVIDER_RE = new RegExp(`(?<=['"\`])(?:${PROVIDERS.join('|')})/[a-zA-Z0-9_:.\\-]+`, 'g');
|
||||
|
||||
function walk(dir, hits = []) {
|
||||
let entries;
|
||||
try {
|
||||
entries = readdirSync(dir);
|
||||
} catch {
|
||||
return hits;
|
||||
}
|
||||
for (const name of entries) {
|
||||
if (SKIP_DIRS.has(name)) continue;
|
||||
const path = join(dir, name);
|
||||
let st;
|
||||
try {
|
||||
st = statSync(path);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (st.isDirectory()) walk(path, hits);
|
||||
else if (st.isFile()) {
|
||||
const dot = name.lastIndexOf('.');
|
||||
if (dot < 0) continue;
|
||||
const ext = name.slice(dot);
|
||||
if (SCAN_EXTS.has(ext)) hits.push(path);
|
||||
}
|
||||
}
|
||||
return hits;
|
||||
}
|
||||
|
||||
function isAllowed(relPath) {
|
||||
const norm = relPath.split(sep).join('/');
|
||||
return ALLOWED_PATHS.some((p) => (p.endsWith('/') ? norm.startsWith(p) : norm === p));
|
||||
}
|
||||
|
||||
const violations = [];
|
||||
let scanned = 0;
|
||||
|
||||
for (const subdir of SCAN_ROOTS) {
|
||||
const root = join(ROOT, subdir);
|
||||
for (const file of walk(root)) {
|
||||
scanned += 1;
|
||||
const rel = relative(ROOT, file);
|
||||
if (isAllowed(rel)) continue;
|
||||
const src = readFileSync(file, 'utf8');
|
||||
PROVIDER_RE.lastIndex = 0;
|
||||
let m;
|
||||
while ((m = PROVIDER_RE.exec(src)) !== null) {
|
||||
// Compute 1-based line number of the match.
|
||||
const lineNo = src.slice(0, m.index).split('\n').length;
|
||||
violations.push({ file: rel, line: lineNo, match: m[0] });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (violations.length === 0) {
|
||||
console.log(
|
||||
`${GREEN}✓${RESET} LLM strings clean — scanned ${BOLD}${scanned}${RESET} files, ` +
|
||||
`no hardcoded provider/model strings found outside the SSOT.`
|
||||
);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
console.log();
|
||||
console.log(`${RED}${BOLD}✗ Hardcoded provider/model strings found.${RESET}`);
|
||||
console.log();
|
||||
console.log(
|
||||
`${DIM}These should use ${BOLD}MANA_LLM.<class>${RESET}${DIM} aliases instead. ` +
|
||||
`The aliases resolve via ${BOLD}services/mana-llm/aliases.yaml${RESET}${DIM} ` +
|
||||
`with health-aware fallback.${RESET}`
|
||||
);
|
||||
console.log(
|
||||
`${DIM}If a site genuinely needs a concrete model, add the file path to ` +
|
||||
`${BOLD}ALLOWED_PATHS${RESET}${DIM} in ` +
|
||||
`${BOLD}scripts/validate-llm-strings.mjs${RESET}${DIM} with a comment.${RESET}`
|
||||
);
|
||||
console.log();
|
||||
|
||||
const grouped = new Map();
|
||||
for (const v of violations) {
|
||||
if (!grouped.has(v.file)) grouped.set(v.file, []);
|
||||
grouped.get(v.file).push(v);
|
||||
}
|
||||
for (const [file, list] of grouped) {
|
||||
console.log(`${YELLOW}${file}${RESET}`);
|
||||
for (const v of list) {
|
||||
console.log(` ${DIM}:${v.line}${RESET} ${v.match}`);
|
||||
}
|
||||
}
|
||||
console.log();
|
||||
console.log(
|
||||
`${RED}Total: ${violations.length} violation(s) across ${grouped.size} file(s).${RESET}`
|
||||
);
|
||||
process.exit(1);
|
||||
Loading…
Add table
Add a link
Reference in a new issue