feat(mana-mcp): Prometheus metrics for policy gate + tool invocations

Replaces the stub /metrics endpoint with a real prom-client registry
(mana_mcp_ prefix, {service="mana-mcp"} default label). Default
process metrics come along for free.

Policy-gate telemetry is the whole point — without it we can't soak
POLICY_MODE=log-only safely or decide when to flip to enforce. New
counter mana_mcp_policy_decisions_total{decision, reason, mode} buckets
every evaluatePolicy() call:

  decision ∈ {allow, deny, flagged}
  reason   ∈ {admin-scope-not-invokable, destructive-not-allowed,
              rate-limit-exceeded, injection-marker, clean, unknown}
  mode     ∈ {log-only, enforce}

So the rate of "would have been denied" during soak is visible directly
as policy_decisions_total{decision="deny", mode="log-only"}.

Also:
  - mana_mcp_tool_invocations_total{tool, outcome} — success |
    handler-error | input-invalid. Policy denies are NOT counted here
    (they're in policy_decisions_total above); this counter only counts
    calls that actually reached the handler or tripped zod validation.
  - mana_mcp_tool_duration_seconds histogram per tool/outcome.

Dep: prom-client ^15.1.3 (same version mana-ai pins).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Till JS 2026-04-23 14:23:08 +02:00
parent f07eae3c01
commit c94ab01c69
4 changed files with 89 additions and 3 deletions

View file

@ -15,6 +15,7 @@
"@modelcontextprotocol/sdk": "^1.29.0",
"hono": "^4.7.0",
"jose": "^6.1.2",
"prom-client": "^15.1.3",
"zod": "^3.25.76"
},
"devDependencies": {

View file

@ -14,6 +14,7 @@ import { registerAllModules } from '@mana/tool-registry';
import { loadConfig } from './config.ts';
import { authenticateRequest, UnauthorizedError } from './auth.ts';
import { handleMcpRequest } from './transport.ts';
import { register as metricsRegistry } from './metrics.ts';
// ─── Bootstrap ────────────────────────────────────────────────────
@ -42,9 +43,10 @@ app.get('/health', (c) =>
})
);
app.get('/metrics', (c) =>
c.text('# mana-mcp metrics stub — populated alongside Persona-Runner observability\n')
);
app.get('/metrics', async (c) => {
const body = await metricsRegistry.metrics();
return c.text(body, 200, { 'content-type': metricsRegistry.contentType });
});
// ─── MCP endpoint ─────────────────────────────────────────────────

View file

@ -25,6 +25,7 @@ import {
import type { VerifiedUser } from './auth.ts';
import type { Config } from './config.ts';
import { appendInvocation, getRecentInvocations } from './invocation-log.ts';
import { policyDecisionsTotal, toolDuration, toolInvocationsTotal } from './metrics.ts';
/**
* Shared across all sessions the client caches MKs per userId with a
@ -109,6 +110,7 @@ export function createMcpServerForUser(user: VerifiedUser, config: Config): McpS
err instanceof z.ZodError
? err.issues.map((i) => `${i.path.join('.')}: ${i.message}`).join('; ')
: String(err);
toolInvocationsTotal.inc({ tool: spec.name, outcome: 'input-invalid' });
return {
isError: true,
content: [{ type: 'text' as const, text: `Invalid input for ${spec.name}: ${msg}` }],
@ -129,6 +131,11 @@ export function createMcpServerForUser(user: VerifiedUser, config: Config): McpS
});
if (!decision.allow) {
policyDecisionsTotal.inc({
decision: 'deny',
reason: decision.reason ?? 'unknown',
mode: config.policyMode,
});
const label = config.policyMode === 'enforce' ? 'DENY' : 'WOULD-DENY';
console.warn(
`[mana-mcp policy] ${label} tool=${spec.name} user=${user.userId.slice(0, 8)} reason=${decision.reason}`
@ -143,7 +150,18 @@ export function createMcpServerForUser(user: VerifiedUser, config: Config): McpS
};
}
} else if (decision.reminder) {
policyDecisionsTotal.inc({
decision: 'flagged',
reason: 'injection-marker',
mode: config.policyMode,
});
console.info(`[mana-mcp policy] FLAG tool=${spec.name} user=${user.userId.slice(0, 8)}`);
} else {
policyDecisionsTotal.inc({
decision: 'allow',
reason: 'clean',
mode: config.policyMode,
});
}
}
@ -151,13 +169,18 @@ export function createMcpServerForUser(user: VerifiedUser, config: Config): McpS
// handler's duration doesn't open a rate-limit gap.
appendInvocation(user.userId, spec.name);
const endTimer = toolDuration.startTimer({ tool: spec.name, outcome: 'success' });
try {
const result = await spec.handler(parsed, ctxFor(spec.name));
toolInvocationsTotal.inc({ tool: spec.name, outcome: 'success' });
endTimer({ tool: spec.name, outcome: 'success' });
return {
content: [{ type: 'text' as const, text: JSON.stringify(result, null, 2) }],
};
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
toolInvocationsTotal.inc({ tool: spec.name, outcome: 'handler-error' });
endTimer({ tool: spec.name, outcome: 'handler-error' });
return {
isError: true,
content: [{ type: 'text' as const, text: `Tool ${spec.name} failed: ${msg}` }],

View file

@ -0,0 +1,60 @@
/**
* Prometheus metrics exported on GET /metrics.
*
* Mirrors the shape of `services/mana-ai/src/metrics.ts` so Grafana and
* status.mana.how recognise this service without special-casing. Metric
* names use the `mana_mcp_*` prefix; labels stay low-cardinality on
* purpose (tool name is high-cardinality but still a fixed registry, so
* it's acceptable we have ~20 tools today).
*/
import { Counter, Histogram, Registry, collectDefaultMetrics } from 'prom-client';
export const register = new Registry();
register.setDefaultLabels({ service: 'mana-mcp' });
collectDefaultMetrics({ register, prefix: 'mana_mcp_' });
// ── Policy gate ──────────────────────────────────────────────
/**
* One sample per `evaluatePolicy()` call.
*
* Labels:
* - `decision`: `allow` | `deny` | `flagged` (flagged = allow with a
* reminder, e.g. freetext injection marker hit)
* - `reason`: `admin-scope-not-invokable` | `destructive-not-allowed`
* | `rate-limit-exceeded` | `injection-marker`
* | `clean` (no reason applied; for dashboards)
* - `mode`: `log-only` | `enforce` lets us diff how many
* decisions WOULD block vs. actually blocked during soak
*/
export const policyDecisionsTotal = new Counter({
name: 'mana_mcp_policy_decisions_total',
help: 'Tool-policy gate decisions, bucketed by outcome and reason.',
labelNames: ['decision', 'reason', 'mode'] as const,
registers: [register],
});
// ── Tool invocations ─────────────────────────────────────────
/**
* Every tool that makes it past the policy gate lands here. `outcome`
* is `success` | `handler-error` | `input-invalid` so dashboards can
* differentiate "tool ran but failed" from "LLM sent malformed args".
* Policy-denied calls are NOT counted here they never reach the
* handler and are visible under `policyDecisionsTotal{decision='deny'}`.
*/
export const toolInvocationsTotal = new Counter({
name: 'mana_mcp_tool_invocations_total',
help: 'Tool handler invocations (after policy gate).',
labelNames: ['tool', 'outcome'] as const,
registers: [register],
});
export const toolDuration = new Histogram({
name: 'mana_mcp_tool_duration_seconds',
help: 'Handler wall-clock latency per tool.',
labelNames: ['tool', 'outcome'] as const,
buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
registers: [register],
});