From 76577869e1309c2df368468819c59053f42c052c Mon Sep 17 00:00:00 2001 From: Till JS Date: Thu, 16 Apr 2026 15:21:23 +0200 Subject: [PATCH] feat(mana-ai): OpenTelemetry tracing + Grafana Tempo backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add distributed tracing to the mana-ai background runner so mission execution can be visualized end-to-end in Grafana. Instrumentation (services/mana-ai/): - tracing.ts: OTel provider setup with OTLP/HTTP exporter, withSpan() helper - tick.ts: tick.planMission span with mission/agent/user attributes - client.ts: planner.complete span with LLM model, tokens, latency Infrastructure: - docker/tempo/tempo.yaml: Grafana Tempo config (OTLP HTTP on 4318) - docker-compose: tempo service + tempo_data volume + mana-ai env var - docker/grafana/provisioning/datasources/tempo.yml: auto-provisioned Trace flow: tick.planMission (root span) └── planner.complete (child span) ├── llm.model = "gpt-4o-mini" ├── llm.tokens.total = 1234 └── llm.response.length = 567 Enable: set OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 View: Grafana → Explore → Tempo datasource Also fixes: removed broken @mana/subscriptions workspace ref from arcade. Co-Authored-By: Claude Opus 4.6 (1M context) --- docker-compose.macmini.yml | 22 + .../provisioning/datasources/tempo.yml | 20 + docker/tempo/tempo.yaml | 29 ++ games/arcade/apps/web/package.json | 1 - pnpm-lock.yaml | 451 +++++++++--------- services/mana-ai/package.json | 5 + services/mana-ai/src/cron/tick.ts | 13 +- services/mana-ai/src/planner/client.ts | 78 +-- services/mana-ai/src/tracing.ts | 96 ++++ 9 files changed, 456 insertions(+), 259 deletions(-) create mode 100644 docker/grafana/provisioning/datasources/tempo.yml create mode 100644 docker/tempo/tempo.yaml create mode 100644 services/mana-ai/src/tracing.ts diff --git a/docker-compose.macmini.yml b/docker-compose.macmini.yml index 690aec151..0a7ae9ede 100644 --- a/docker-compose.macmini.yml +++ b/docker-compose.macmini.yml @@ -332,6 +332,7 @@ services: # mana-auth. Used to unwrap per-mission data keys at tick time. # Absent → all grants skip silently with reason="not-configured". MANA_AI_PRIVATE_KEY_PEM: ${MANA_AI_PRIVATE_KEY_PEM:-} + OTEL_EXPORTER_OTLP_ENDPOINT: http://tempo:4318 ports: - "3067:3067" healthcheck: @@ -1230,6 +1231,25 @@ services: retries: 3 start_period: 10s + tempo: + image: grafana/tempo:2.6.1 + container_name: mana-mon-tempo + restart: always + mem_limit: 256m + command: ["-config.file=/etc/tempo/tempo.yaml"] + volumes: + - ./docker/tempo:/etc/tempo:ro + - tempo_data:/var/tempo + ports: + - "4318:4318" # OTLP HTTP receiver + - "3200:3200" # Tempo API (for Grafana) + healthcheck: + test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://127.0.0.1:3200/ready"] + interval: 300s + timeout: 10s + retries: 3 + start_period: 10s + loki: image: grafana/loki:3.0.0 container_name: mana-mon-loki @@ -1666,3 +1686,5 @@ volumes: name: mana-loki-data stalwart_data: name: mana-stalwart-data + tempo_data: + name: mana-tempo-data diff --git a/docker/grafana/provisioning/datasources/tempo.yml b/docker/grafana/provisioning/datasources/tempo.yml new file mode 100644 index 000000000..c8d513307 --- /dev/null +++ b/docker/grafana/provisioning/datasources/tempo.yml @@ -0,0 +1,20 @@ +apiVersion: 1 + +datasources: + - name: Tempo + type: tempo + access: proxy + url: http://tempo:3200 + isDefault: false + jsonData: + httpMethod: GET + tracesToLogsV2: + datasourceUid: loki + spanStartTimeShift: "-1h" + spanEndTimeShift: "1h" + filterByTraceID: true + filterBySpanID: false + tracesToMetrics: + datasourceUid: prometheus + spanStartTimeShift: "-1h" + spanEndTimeShift: "1h" diff --git a/docker/tempo/tempo.yaml b/docker/tempo/tempo.yaml new file mode 100644 index 000000000..4839fe6bd --- /dev/null +++ b/docker/tempo/tempo.yaml @@ -0,0 +1,29 @@ +stream_over_http_enabled: true +server: + http_listen_port: 3200 + +distributor: + receivers: + otlp: + protocols: + http: + endpoint: "0.0.0.0:4318" + +storage: + trace: + backend: local + local: + path: /var/tempo/traces + wal: + path: /var/tempo/wal + +metrics_generator: + storage: + path: /var/tempo/metrics + traces_storage: + path: /var/tempo/traces + +overrides: + defaults: + metrics_generator: + processors: [] diff --git a/games/arcade/apps/web/package.json b/games/arcade/apps/web/package.json index 12fcc2d8b..452e5e65e 100644 --- a/games/arcade/apps/web/package.json +++ b/games/arcade/apps/web/package.json @@ -43,7 +43,6 @@ "@mana/shared-icons": "workspace:*", "@mana/shared-stores": "workspace:*", "@mana/shared-tags": "workspace:*", - "@mana/subscriptions": "workspace:*", "@mana/shared-tailwind": "workspace:*", "@mana/shared-theme": "workspace:*", "@mana/shared-theme-ui": "workspace:*", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fa1d321c5..fe36e190d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -138,14 +138,14 @@ importers: version: link:../../../../packages/shared-landing-ui astro: specifier: ^5.16.0 - version: 5.18.1(@azure/storage-blob@12.31.0)(@types/node@20.19.39)(ioredis@5.10.1)(jiti@1.21.7)(lightningcss@1.32.0)(rollup@4.60.1)(terser@5.46.1)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.3) + version: 5.18.1(@azure/storage-blob@12.31.0)(@types/node@20.19.39)(ioredis@5.10.1)(jiti@2.6.1)(lightningcss@1.32.0)(rollup@4.60.1)(terser@5.46.1)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.3) typescript: specifier: ^5.9.2 version: 5.9.3 devDependencies: '@astrojs/tailwind': specifier: ^6.0.2 - version: 6.0.2(astro@5.18.1(@azure/storage-blob@12.31.0)(@types/node@20.19.39)(ioredis@5.10.1)(jiti@1.21.7)(lightningcss@1.32.0)(rollup@4.60.1)(terser@5.46.1)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.3))(tailwindcss@3.4.19(tsx@4.21.0)(yaml@2.8.3)) + version: 6.0.2(astro@5.18.1(@azure/storage-blob@12.31.0)(@types/node@20.19.39)(ioredis@5.10.1)(jiti@2.6.1)(lightningcss@1.32.0)(rollup@4.60.1)(terser@5.46.1)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.3))(tailwindcss@3.4.19(tsx@4.21.0)(yaml@2.8.3)) '@tailwindcss/typography': specifier: ^0.5.18 version: 0.5.19(tailwindcss@3.4.19(tsx@4.21.0)(yaml@2.8.3)) @@ -154,13 +154,13 @@ importers: version: 20.19.39 eslint: specifier: ^9.0.0 - version: 9.39.4(jiti@1.21.7) + version: 9.39.4(jiti@2.6.1) eslint-config-prettier: specifier: ^9.1.0 - version: 9.1.2(eslint@9.39.4(jiti@1.21.7)) + version: 9.1.2(eslint@9.39.4(jiti@2.6.1)) eslint-plugin-astro: specifier: ^1.0.0 - version: 1.6.0(eslint@9.39.4(jiti@1.21.7)) + version: 1.6.0(eslint@9.39.4(jiti@2.6.1)) prettier: specifier: ^3.6.2 version: 3.8.1 @@ -193,7 +193,7 @@ importers: version: link:../../../../packages/shared-landing-ui astro: specifier: ^5.16.0 - version: 5.18.1(@azure/storage-blob@12.31.0)(@types/node@24.12.2)(ioredis@5.10.1)(jiti@2.6.1)(lightningcss@1.32.0)(rollup@4.60.1)(terser@5.46.1)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.3) + version: 5.18.1(@azure/storage-blob@12.31.0)(@types/node@24.12.2)(ioredis@5.10.1)(jiti@1.21.7)(lightningcss@1.32.0)(rollup@4.60.1)(terser@5.46.1)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.3) astro-icon: specifier: ^1.1.5 version: 1.1.5 @@ -203,7 +203,7 @@ importers: devDependencies: '@astrojs/tailwind': specifier: ^6.0.0 - version: 6.0.2(astro@5.18.1(@azure/storage-blob@12.31.0)(@types/node@24.12.2)(ioredis@5.10.1)(jiti@2.6.1)(lightningcss@1.32.0)(rollup@4.60.1)(terser@5.46.1)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.3))(tailwindcss@3.4.19(tsx@4.21.0)(yaml@2.8.3)) + version: 6.0.2(astro@5.18.1(@azure/storage-blob@12.31.0)(@types/node@24.12.2)(ioredis@5.10.1)(jiti@1.21.7)(lightningcss@1.32.0)(rollup@4.60.1)(terser@5.46.1)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.3))(tailwindcss@3.4.19(tsx@4.21.0)(yaml@2.8.3)) '@tailwindcss/typography': specifier: ^0.5.16 version: 0.5.19(tailwindcss@3.4.19(tsx@4.21.0)(yaml@2.8.3)) @@ -2518,9 +2518,6 @@ importers: '@mana/shared-utils': specifier: workspace:* version: link:../../../../packages/shared-utils - '@mana/subscriptions': - specifier: workspace:* - version: link:../../../../packages/subscriptions svelte-i18n: specifier: ^4.0.1 version: 4.0.1(svelte@5.55.1) @@ -3252,21 +3249,6 @@ importers: specifier: ^4.1.2 version: 4.1.3(@opentelemetry/api@1.9.1)(@types/node@20.19.39)(@vitest/coverage-v8@4.1.3)(@vitest/ui@4.1.3)(jsdom@29.0.2(@noble/hashes@2.0.1))(vite@6.4.2(@types/node@20.19.39)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3)) - packages/subscriptions: - devDependencies: - svelte: - specifier: ^5.0.0 - version: 5.55.1 - svelte-check: - specifier: ^4.0.0 - version: 4.4.6(picomatch@4.0.4)(svelte@5.55.1)(typescript@5.9.3) - svelte-i18n: - specifier: ^4.0.0 - version: 4.0.1(svelte@5.55.1) - typescript: - specifier: ^5.9.3 - version: 5.9.3 - packages/test-config: dependencies: '@playwright/test': @@ -3313,6 +3295,21 @@ importers: '@mana/shared-hono': specifier: workspace:* version: link:../../packages/shared-hono + '@opentelemetry/api': + specifier: ^1.9.0 + version: 1.9.1 + '@opentelemetry/exporter-trace-otlp-http': + specifier: ^0.57.0 + version: 0.57.2(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': + specifier: ^1.30.0 + version: 1.30.1(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace-base': + specifier: ^1.30.0 + version: 1.30.1(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': + specifier: ^1.28.0 + version: 1.40.0 hono: specifier: ^4.7.0 version: 4.12.12 @@ -6933,6 +6930,12 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' + '@opentelemetry/exporter-trace-otlp-http@0.57.2': + resolution: {integrity: sha512-sB/gkSYFu+0w2dVQ0PWY9fAMl172PKMZ/JrHkkW8dmjCL0CYkmXeE+ssqIL/yBUTPOvpLIpenX5T9RwXRBW/3g==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + '@opentelemetry/instrumentation-amqplib@0.46.1': resolution: {integrity: sha512-AyXVnlCf/xV3K/rNumzKxZqsULyITJH6OVLiW6730JPRqWA7Zc9bvYoVNpN6iOpTU8CasH34SU/ksVJmObFibQ==} engines: {node: '>=14'} @@ -7071,6 +7074,18 @@ packages: peerDependencies: '@opentelemetry/api': ^1.3.0 + '@opentelemetry/otlp-exporter-base@0.57.2': + resolution: {integrity: sha512-XdxEzL23Urhidyebg5E6jZoaiW5ygP/mRjxLHixogbqwDy2Faduzb5N0o/Oi+XTIJu+iyxXdVORjXax+Qgfxag==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/otlp-transformer@0.57.2': + resolution: {integrity: sha512-48IIRj49gbQVK52jYsw70+Jv+JbahT8BqT2Th7C4H7RCM9d0gZ5sgNPoMpWldmfjvIsSgiGJtjfk9MeZvjhoig==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + '@opentelemetry/redis-common@0.36.2': resolution: {integrity: sha512-faYX1N0gpLhej/6nyp6bgRjzAKXn5GOEMYY7YhciSfCoITAktLUtQ36d24QEWNA1/WA1y6qQunCe0OhHRkVl9g==} engines: {node: '>=14'} @@ -7081,6 +7096,18 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' + '@opentelemetry/sdk-logs@0.57.2': + resolution: {integrity: sha512-TXFHJ5c+BKggWbdEQ/inpgIzEmS2BGQowLE9UhsMd7YYlUfBQJ4uax0VF/B5NYigdM/75OoJGhAV3upEhK+3gg==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': '>=1.4.0 <1.10.0' + + '@opentelemetry/sdk-metrics@1.30.1': + resolution: {integrity: sha512-q9zcZ0Okl8jRgmy7eNW3Ku1XSgg3sDLa5evHZpCwjspw7E8Is4K/haRPDJrBcX3YSn/Y7gUvFnByNYEKQNbNog==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + '@opentelemetry/sdk-trace-base@1.30.1': resolution: {integrity: sha512-jVPgBbH1gCy2Lb7X0AVQ8XAfgg0pJ4nvl8/IiQA6nxOsPvS+0zMJaFSs2ltXe0J6C8dqjcnpyqINDJmU30+uOg==} engines: {node: '>=14'} @@ -18155,16 +18182,6 @@ snapshots: transitivePeerDependencies: - ts-node - '@astrojs/tailwind@6.0.2(astro@5.18.1(@azure/storage-blob@12.31.0)(@types/node@20.19.39)(ioredis@5.10.1)(jiti@1.21.7)(lightningcss@1.32.0)(rollup@4.60.1)(terser@5.46.1)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.3))(tailwindcss@3.4.19(tsx@4.21.0)(yaml@2.8.3))': - dependencies: - astro: 5.18.1(@azure/storage-blob@12.31.0)(@types/node@20.19.39)(ioredis@5.10.1)(jiti@1.21.7)(lightningcss@1.32.0)(rollup@4.60.1)(terser@5.46.1)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.3) - autoprefixer: 10.4.27(postcss@8.5.8) - postcss: 8.5.8 - postcss-load-config: 4.0.2(postcss@8.5.8) - tailwindcss: 3.4.19(tsx@4.21.0)(yaml@2.8.3) - transitivePeerDependencies: - - ts-node - '@astrojs/tailwind@6.0.2(astro@5.18.1(@azure/storage-blob@12.31.0)(@types/node@20.19.39)(ioredis@5.10.1)(jiti@2.6.1)(lightningcss@1.32.0)(rollup@4.60.1)(terser@5.46.1)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.3))(tailwindcss@3.4.19(tsx@4.21.0)(yaml@2.8.3))': dependencies: astro: 5.18.1(@azure/storage-blob@12.31.0)(@types/node@20.19.39)(ioredis@5.10.1)(jiti@2.6.1)(lightningcss@1.32.0)(rollup@4.60.1)(terser@5.46.1)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.3) @@ -18185,6 +18202,16 @@ snapshots: transitivePeerDependencies: - ts-node + '@astrojs/tailwind@6.0.2(astro@5.18.1(@azure/storage-blob@12.31.0)(@types/node@24.12.2)(ioredis@5.10.1)(jiti@1.21.7)(lightningcss@1.32.0)(rollup@4.60.1)(terser@5.46.1)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.3))(tailwindcss@3.4.19(tsx@4.21.0)(yaml@2.8.3))': + dependencies: + astro: 5.18.1(@azure/storage-blob@12.31.0)(@types/node@24.12.2)(ioredis@5.10.1)(jiti@1.21.7)(lightningcss@1.32.0)(rollup@4.60.1)(terser@5.46.1)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.3) + autoprefixer: 10.4.27(postcss@8.5.8) + postcss: 8.5.8 + postcss-load-config: 4.0.2(postcss@8.5.8) + tailwindcss: 3.4.19(tsx@4.21.0)(yaml@2.8.3) + transitivePeerDependencies: + - ts-node + '@astrojs/tailwind@6.0.2(astro@5.18.1(@azure/storage-blob@12.31.0)(@types/node@24.12.2)(ioredis@5.10.1)(jiti@2.6.1)(lightningcss@1.32.0)(rollup@4.60.1)(terser@5.46.1)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.3))(tailwindcss@3.4.19(tsx@4.21.0)(yaml@2.8.3))': dependencies: astro: 5.18.1(@azure/storage-blob@12.31.0)(@types/node@24.12.2)(ioredis@5.10.1)(jiti@2.6.1)(lightningcss@1.32.0)(rollup@4.60.1)(terser@5.46.1)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.3) @@ -20365,11 +20392,6 @@ snapshots: eslint: 8.57.1 eslint-visitor-keys: 3.4.3 - '@eslint-community/eslint-utils@4.9.1(eslint@9.39.4(jiti@1.21.7))': - dependencies: - eslint: 9.39.4(jiti@1.21.7) - eslint-visitor-keys: 3.4.3 - '@eslint-community/eslint-utils@4.9.1(eslint@9.39.4(jiti@2.6.1))': dependencies: eslint: 9.39.4(jiti@2.6.1) @@ -22302,6 +22324,15 @@ snapshots: '@opentelemetry/api': 1.9.1 '@opentelemetry/semantic-conventions': 1.28.0 + '@opentelemetry/exporter-trace-otlp-http@0.57.2(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-exporter-base': 0.57.2(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-transformer': 0.57.2(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace-base': 1.30.1(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-amqplib@0.46.1(@opentelemetry/api@1.9.1)': dependencies: '@opentelemetry/api': 1.9.1 @@ -22505,6 +22536,23 @@ snapshots: transitivePeerDependencies: - supports-color + '@opentelemetry/otlp-exporter-base@0.57.2(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-transformer': 0.57.2(@opentelemetry/api@1.9.1) + + '@opentelemetry/otlp-transformer@0.57.2(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.57.2 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-logs': 0.57.2(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-metrics': 1.30.1(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace-base': 1.30.1(@opentelemetry/api@1.9.1) + protobufjs: 7.5.4 + '@opentelemetry/redis-common@0.36.2': {} '@opentelemetry/resources@1.30.1(@opentelemetry/api@1.9.1)': @@ -22513,6 +22561,19 @@ snapshots: '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1) '@opentelemetry/semantic-conventions': 1.28.0 + '@opentelemetry/sdk-logs@0.57.2(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.57.2 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.1) + + '@opentelemetry/sdk-metrics@1.30.1(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace-base@1.30.1(@opentelemetry/api@1.9.1)': dependencies: '@opentelemetry/api': 1.9.1 @@ -26515,108 +26576,6 @@ snapshots: transitivePeerDependencies: - supports-color - astro@5.18.1(@azure/storage-blob@12.31.0)(@types/node@20.19.39)(ioredis@5.10.1)(jiti@1.21.7)(lightningcss@1.32.0)(rollup@4.60.1)(terser@5.46.1)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.3): - dependencies: - '@astrojs/compiler': 2.13.1 - '@astrojs/internal-helpers': 0.7.6 - '@astrojs/markdown-remark': 6.3.11 - '@astrojs/telemetry': 3.3.0 - '@capsizecss/unpack': 4.0.0 - '@oslojs/encoding': 1.1.0 - '@rollup/pluginutils': 5.3.0(rollup@4.60.1) - acorn: 8.16.0 - aria-query: 5.3.2 - axobject-query: 4.1.0 - boxen: 8.0.1 - ci-info: 4.4.0 - clsx: 2.1.1 - common-ancestor-path: 1.0.1 - cookie: 1.1.1 - cssesc: 3.0.0 - debug: 4.4.3 - deterministic-object-hash: 2.0.2 - devalue: 5.7.0 - diff: 8.0.4 - dlv: 1.1.3 - dset: 3.1.4 - es-module-lexer: 1.7.0 - esbuild: 0.27.7 - estree-walker: 3.0.3 - flattie: 1.1.1 - fontace: 0.4.1 - github-slugger: 2.0.0 - html-escaper: 3.0.3 - http-cache-semantics: 4.2.0 - import-meta-resolve: 4.2.0 - js-yaml: 4.1.1 - magic-string: 0.30.21 - magicast: 0.5.2 - mrmime: 2.0.1 - neotraverse: 0.6.18 - p-limit: 6.2.0 - p-queue: 8.1.1 - package-manager-detector: 1.6.0 - piccolore: 0.1.3 - picomatch: 4.0.4 - prompts: 2.4.2 - rehype: 13.0.2 - semver: 7.7.4 - shiki: 3.23.0 - smol-toml: 1.6.1 - svgo: 4.0.1 - tinyexec: 1.0.4 - tinyglobby: 0.2.15 - tsconfck: 3.1.6(typescript@5.9.3) - ultrahtml: 1.6.0 - unifont: 0.7.4 - unist-util-visit: 5.1.0 - unstorage: 1.17.5(@azure/storage-blob@12.31.0)(ioredis@5.10.1) - vfile: 6.0.3 - vite: 6.4.2(@types/node@20.19.39)(jiti@1.21.7)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3) - vitefu: 1.1.3(vite@6.4.2(@types/node@20.19.39)(jiti@1.21.7)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3)) - xxhash-wasm: 1.1.0 - yargs-parser: 21.1.1 - yocto-spinner: 0.2.3 - zod: 3.25.76 - zod-to-json-schema: 3.25.2(zod@3.25.76) - zod-to-ts: 1.2.0(typescript@5.9.3)(zod@3.25.76) - optionalDependencies: - sharp: 0.34.5 - transitivePeerDependencies: - - '@azure/app-configuration' - - '@azure/cosmos' - - '@azure/data-tables' - - '@azure/identity' - - '@azure/keyvault-secrets' - - '@azure/storage-blob' - - '@capacitor/preferences' - - '@deno/kv' - - '@netlify/blobs' - - '@planetscale/database' - - '@types/node' - - '@upstash/redis' - - '@vercel/blob' - - '@vercel/functions' - - '@vercel/kv' - - aws4fetch - - db0 - - idb-keyval - - ioredis - - jiti - - less - - lightningcss - - rollup - - sass - - sass-embedded - - stylus - - sugarss - - supports-color - - terser - - tsx - - typescript - - uploadthing - - yaml - astro@5.18.1(@azure/storage-blob@12.31.0)(@types/node@20.19.39)(ioredis@5.10.1)(jiti@2.6.1)(lightningcss@1.32.0)(rollup@4.60.1)(terser@5.46.1)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.3): dependencies: '@astrojs/compiler': 2.13.1 @@ -26821,6 +26780,108 @@ snapshots: - uploadthing - yaml + astro@5.18.1(@azure/storage-blob@12.31.0)(@types/node@24.12.2)(ioredis@5.10.1)(jiti@1.21.7)(lightningcss@1.32.0)(rollup@4.60.1)(terser@5.46.1)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.3): + dependencies: + '@astrojs/compiler': 2.13.1 + '@astrojs/internal-helpers': 0.7.6 + '@astrojs/markdown-remark': 6.3.11 + '@astrojs/telemetry': 3.3.0 + '@capsizecss/unpack': 4.0.0 + '@oslojs/encoding': 1.1.0 + '@rollup/pluginutils': 5.3.0(rollup@4.60.1) + acorn: 8.16.0 + aria-query: 5.3.2 + axobject-query: 4.1.0 + boxen: 8.0.1 + ci-info: 4.4.0 + clsx: 2.1.1 + common-ancestor-path: 1.0.1 + cookie: 1.1.1 + cssesc: 3.0.0 + debug: 4.4.3 + deterministic-object-hash: 2.0.2 + devalue: 5.7.0 + diff: 8.0.4 + dlv: 1.1.3 + dset: 3.1.4 + es-module-lexer: 1.7.0 + esbuild: 0.27.7 + estree-walker: 3.0.3 + flattie: 1.1.1 + fontace: 0.4.1 + github-slugger: 2.0.0 + html-escaper: 3.0.3 + http-cache-semantics: 4.2.0 + import-meta-resolve: 4.2.0 + js-yaml: 4.1.1 + magic-string: 0.30.21 + magicast: 0.5.2 + mrmime: 2.0.1 + neotraverse: 0.6.18 + p-limit: 6.2.0 + p-queue: 8.1.1 + package-manager-detector: 1.6.0 + piccolore: 0.1.3 + picomatch: 4.0.4 + prompts: 2.4.2 + rehype: 13.0.2 + semver: 7.7.4 + shiki: 3.23.0 + smol-toml: 1.6.1 + svgo: 4.0.1 + tinyexec: 1.0.4 + tinyglobby: 0.2.15 + tsconfck: 3.1.6(typescript@5.9.3) + ultrahtml: 1.6.0 + unifont: 0.7.4 + unist-util-visit: 5.1.0 + unstorage: 1.17.5(@azure/storage-blob@12.31.0)(ioredis@5.10.1) + vfile: 6.0.3 + vite: 6.4.2(@types/node@24.12.2)(jiti@1.21.7)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3) + vitefu: 1.1.3(vite@6.4.2(@types/node@24.12.2)(jiti@1.21.7)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3)) + xxhash-wasm: 1.1.0 + yargs-parser: 21.1.1 + yocto-spinner: 0.2.3 + zod: 3.25.76 + zod-to-json-schema: 3.25.2(zod@3.25.76) + zod-to-ts: 1.2.0(typescript@5.9.3)(zod@3.25.76) + optionalDependencies: + sharp: 0.34.5 + transitivePeerDependencies: + - '@azure/app-configuration' + - '@azure/cosmos' + - '@azure/data-tables' + - '@azure/identity' + - '@azure/keyvault-secrets' + - '@azure/storage-blob' + - '@capacitor/preferences' + - '@deno/kv' + - '@netlify/blobs' + - '@planetscale/database' + - '@types/node' + - '@upstash/redis' + - '@vercel/blob' + - '@vercel/functions' + - '@vercel/kv' + - aws4fetch + - db0 + - idb-keyval + - ioredis + - jiti + - less + - lightningcss + - rollup + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + - tsx + - typescript + - uploadthing + - yaml + astro@5.18.1(@azure/storage-blob@12.31.0)(@types/node@24.12.2)(ioredis@5.10.1)(jiti@2.6.1)(lightningcss@1.32.0)(rollup@4.60.1)(terser@5.46.1)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.3): dependencies: '@astrojs/compiler': 2.13.1 @@ -28650,11 +28711,6 @@ snapshots: eslint: 9.39.4(jiti@2.6.1) semver: 7.7.4 - eslint-compat-utils@0.6.5(eslint@9.39.4(jiti@1.21.7)): - dependencies: - eslint: 9.39.4(jiti@1.21.7) - semver: 7.7.4 - eslint-compat-utils@0.6.5(eslint@9.39.4(jiti@2.6.1)): dependencies: eslint: 9.39.4(jiti@2.6.1) @@ -28706,10 +28762,6 @@ snapshots: dependencies: eslint: 9.39.4(jiti@2.6.1) - eslint-config-prettier@9.1.2(eslint@9.39.4(jiti@1.21.7)): - dependencies: - eslint: 9.39.4(jiti@1.21.7) - eslint-config-prettier@9.1.2(eslint@9.39.4(jiti@2.6.1)): dependencies: eslint: 9.39.4(jiti@2.6.1) @@ -28893,20 +28945,6 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-plugin-astro@1.6.0(eslint@9.39.4(jiti@1.21.7)): - dependencies: - '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.4(jiti@1.21.7)) - '@jridgewell/sourcemap-codec': 1.5.5 - '@typescript-eslint/types': 8.58.0 - astro-eslint-parser: 1.4.0 - eslint: 9.39.4(jiti@1.21.7) - eslint-compat-utils: 0.6.5(eslint@9.39.4(jiti@1.21.7)) - globals: 16.5.0 - postcss: 8.5.8 - postcss-selector-parser: 7.1.1 - transitivePeerDependencies: - - supports-color - eslint-plugin-astro@1.6.0(eslint@9.39.4(jiti@2.6.1)): dependencies: '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.4(jiti@2.6.1)) @@ -29328,47 +29366,6 @@ snapshots: transitivePeerDependencies: - supports-color - eslint@9.39.4(jiti@1.21.7): - dependencies: - '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.4(jiti@1.21.7)) - '@eslint-community/regexpp': 4.12.2 - '@eslint/config-array': 0.21.2 - '@eslint/config-helpers': 0.4.2 - '@eslint/core': 0.17.0 - '@eslint/eslintrc': 3.3.5 - '@eslint/js': 9.39.4 - '@eslint/plugin-kit': 0.4.1 - '@humanfs/node': 0.16.7 - '@humanwhocodes/module-importer': 1.0.1 - '@humanwhocodes/retry': 0.4.3 - '@types/estree': 1.0.8 - ajv: 6.14.0 - chalk: 4.1.2 - cross-spawn: 7.0.6 - debug: 4.4.3 - escape-string-regexp: 4.0.0 - eslint-scope: 8.4.0 - eslint-visitor-keys: 4.2.1 - espree: 10.4.0 - esquery: 1.7.0 - esutils: 2.0.3 - fast-deep-equal: 3.1.3 - file-entry-cache: 8.0.0 - find-up: 5.0.0 - glob-parent: 6.0.2 - ignore: 5.3.2 - imurmurhash: 0.1.4 - is-glob: 4.0.3 - json-stable-stringify-without-jsonify: 1.0.1 - lodash.merge: 4.6.2 - minimatch: 3.1.5 - natural-compare: 1.4.0 - optionator: 0.9.4 - optionalDependencies: - jiti: 1.21.7 - transitivePeerDependencies: - - supports-color - eslint@9.39.4(jiti@2.6.1): dependencies: '@eslint-community/eslint-utils': 4.9.1(eslint@9.39.4(jiti@2.6.1)) @@ -37547,23 +37544,6 @@ snapshots: lightningcss: 1.32.0 terser: 5.46.1 - vite@6.4.2(@types/node@20.19.39)(jiti@1.21.7)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3): - dependencies: - esbuild: 0.25.12 - fdir: 6.5.0(picomatch@4.0.4) - picomatch: 4.0.4 - postcss: 8.5.8 - rollup: 4.60.1 - tinyglobby: 0.2.15 - optionalDependencies: - '@types/node': 20.19.39 - fsevents: 2.3.3 - jiti: 1.21.7 - lightningcss: 1.32.0 - terser: 5.46.1 - tsx: 4.21.0 - yaml: 2.8.3 - vite@6.4.2(@types/node@20.19.39)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3): dependencies: esbuild: 0.25.12 @@ -37598,6 +37578,23 @@ snapshots: tsx: 4.21.0 yaml: 2.8.3 + vite@6.4.2(@types/node@24.12.2)(jiti@1.21.7)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3): + dependencies: + esbuild: 0.25.12 + fdir: 6.5.0(picomatch@4.0.4) + picomatch: 4.0.4 + postcss: 8.5.8 + rollup: 4.60.1 + tinyglobby: 0.2.15 + optionalDependencies: + '@types/node': 24.12.2 + fsevents: 2.3.3 + jiti: 1.21.7 + lightningcss: 1.32.0 + terser: 5.46.1 + tsx: 4.21.0 + yaml: 2.8.3 + vite@6.4.2(@types/node@24.12.2)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3): dependencies: esbuild: 0.25.12 @@ -37615,10 +37612,6 @@ snapshots: tsx: 4.21.0 yaml: 2.8.3 - vitefu@1.1.3(vite@6.4.2(@types/node@20.19.39)(jiti@1.21.7)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3)): - optionalDependencies: - vite: 6.4.2(@types/node@20.19.39)(jiti@1.21.7)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3) - vitefu@1.1.3(vite@6.4.2(@types/node@20.19.39)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3)): optionalDependencies: vite: 6.4.2(@types/node@20.19.39)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3) @@ -37627,6 +37620,10 @@ snapshots: optionalDependencies: vite: 6.4.2(@types/node@22.19.17)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3) + vitefu@1.1.3(vite@6.4.2(@types/node@24.12.2)(jiti@1.21.7)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3)): + optionalDependencies: + vite: 6.4.2(@types/node@24.12.2)(jiti@1.21.7)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3) + vitefu@1.1.3(vite@6.4.2(@types/node@24.12.2)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3)): optionalDependencies: vite: 6.4.2(@types/node@24.12.2)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.3) diff --git a/services/mana-ai/package.json b/services/mana-ai/package.json index 2abdc4df5..a2eab0600 100644 --- a/services/mana-ai/package.json +++ b/services/mana-ai/package.json @@ -11,6 +11,11 @@ "dependencies": { "@mana/shared-ai": "workspace:*", "@mana/shared-hono": "workspace:*", + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/exporter-trace-otlp-http": "^0.57.0", + "@opentelemetry/resources": "^1.30.0", + "@opentelemetry/sdk-trace-base": "^1.30.0", + "@opentelemetry/semantic-conventions": "^1.28.0", "hono": "^4.7.0", "postgres": "^3.4.5", "prom-client": "^15.1.3" diff --git a/services/mana-ai/src/cron/tick.ts b/services/mana-ai/src/cron/tick.ts index bb12fb6f8..29b722e47 100644 --- a/services/mana-ai/src/cron/tick.ts +++ b/services/mana-ai/src/cron/tick.ts @@ -47,6 +47,7 @@ import { unwrapMissionGrant } from '../crypto/unwrap-grant'; import { NewsResearchClient } from '../planner/news-research-client'; import type { ResolverContext } from '../db/resolvers/types'; import type { Config } from '../config'; +import { withSpan } from '../tracing'; const ENC_PREFIX = 'enc:1:'; @@ -179,7 +180,17 @@ export async function runTickOnce(config: Config): Promise { } try { - const planResult = await planOneMission(m, planner, sql, agent, config); + const planResult = await withSpan( + 'tick.planMission', + { + 'mission.id': m.id, + 'mission.title': m.title, + 'user.id': m.userId, + 'agent.id': agent?.id ?? 'legacy', + 'agent.name': agent?.name ?? 'Mana', + }, + () => planOneMission(m, planner, sql, agent, config) + ); if (planResult === null) { parseFailures++; parseFailuresTotal.inc(); diff --git a/services/mana-ai/src/planner/client.ts b/services/mana-ai/src/planner/client.ts index 8cd2d7cf7..8f7d740db 100644 --- a/services/mana-ai/src/planner/client.ts +++ b/services/mana-ai/src/planner/client.ts @@ -8,6 +8,7 @@ */ import { plannerLatency } from '../metrics'; +import { withSpan } from '../tracing'; export interface PlannerMessages { system: string; @@ -43,38 +44,55 @@ export class PlannerClient { messages: PlannerMessages, opts: { model?: string; temperature?: number } ): Promise { - const res = await fetch(`${this.baseUrl}/v1/chat/completions`, { - method: 'POST', - headers: { - 'content-type': 'application/json', - authorization: `Bearer ${this.serviceKey}`, + return withSpan( + 'planner.complete', + { + 'llm.model': opts.model ?? 'gpt-4o-mini', + 'llm.temperature': opts.temperature ?? 0.3, }, - body: JSON.stringify({ - model: opts.model ?? 'gpt-4o-mini', - temperature: opts.temperature ?? 0.3, - messages: [ - { role: 'system', content: messages.system }, - { role: 'user', content: messages.user }, - ], - }), - }); + async (span) => { + const res = await fetch(`${this.baseUrl}/v1/chat/completions`, { + method: 'POST', + headers: { + 'content-type': 'application/json', + authorization: `Bearer ${this.serviceKey}`, + }, + body: JSON.stringify({ + model: opts.model ?? 'gpt-4o-mini', + temperature: opts.temperature ?? 0.3, + messages: [ + { role: 'system', content: messages.system }, + { role: 'user', content: messages.user }, + ], + }), + }); - if (!res.ok) { - throw new Error(`mana-llm ${res.status}: ${await res.text().catch(() => '')}`); - } - - const body = (await res.json()) as { - choices?: { message?: { content?: string } }[]; - usage?: { prompt_tokens?: number; completion_tokens?: number; total_tokens?: number }; - }; - const content = body.choices?.[0]?.message?.content ?? ''; - const usage = body.usage - ? { - promptTokens: body.usage.prompt_tokens ?? 0, - completionTokens: body.usage.completion_tokens ?? 0, - totalTokens: body.usage.total_tokens ?? 0, + if (!res.ok) { + throw new Error(`mana-llm ${res.status}: ${await res.text().catch(() => '')}`); } - : undefined; - return { content, usage }; + + const body = (await res.json()) as { + choices?: { message?: { content?: string } }[]; + usage?: { prompt_tokens?: number; completion_tokens?: number; total_tokens?: number }; + }; + const content = body.choices?.[0]?.message?.content ?? ''; + const usage = body.usage + ? { + promptTokens: body.usage.prompt_tokens ?? 0, + completionTokens: body.usage.completion_tokens ?? 0, + totalTokens: body.usage.total_tokens ?? 0, + } + : undefined; + + if (usage) { + span.setAttribute('llm.tokens.prompt', usage.promptTokens); + span.setAttribute('llm.tokens.completion', usage.completionTokens); + span.setAttribute('llm.tokens.total', usage.totalTokens); + } + span.setAttribute('llm.response.length', content.length); + + return { content, usage }; + } + ); // end withSpan } } diff --git a/services/mana-ai/src/tracing.ts b/services/mana-ai/src/tracing.ts new file mode 100644 index 000000000..dda7971c8 --- /dev/null +++ b/services/mana-ai/src/tracing.ts @@ -0,0 +1,96 @@ +/** + * OpenTelemetry tracing setup for mana-ai. + * + * Exports a tracer and initializes the trace provider on first import. + * Traces are exported via OTLP/HTTP to Grafana Tempo (or any + * OTLP-compatible backend). When no backend is configured + * (OTEL_EXPORTER_OTLP_ENDPOINT not set), tracing is a no-op. + * + * Usage in service code: + * import { tracer } from '../tracing'; + * const span = tracer.startSpan('tick.planOneMission'); + * try { ... } finally { span.end(); } + * + * Or with the helper: + * import { withSpan } from '../tracing'; + * const result = await withSpan('tick.planOneMission', { missionId }, async (span) => { + * // ... your code + * }); + */ + +import { + trace, + SpanStatusCode, + type Span, + type Tracer, + type SpanOptions, +} from '@opentelemetry/api'; +import { + BasicTracerProvider, + SimpleSpanProcessor, + BatchSpanProcessor, +} from '@opentelemetry/sdk-trace-base'; +import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'; +import { Resource } from '@opentelemetry/resources'; +import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions'; + +const OTEL_ENDPOINT = process.env.OTEL_EXPORTER_OTLP_ENDPOINT; + +// Initialize provider once on module load +if (OTEL_ENDPOINT) { + const resource = new Resource({ + [ATTR_SERVICE_NAME]: 'mana-ai', + [ATTR_SERVICE_VERSION]: '0.6.0', + }); + + const exporter = new OTLPTraceExporter({ + url: `${OTEL_ENDPOINT}/v1/traces`, + }); + + const provider = new BasicTracerProvider({ resource }); + // Use batch in production (less overhead), simple in dev (immediate export) + const isDev = process.env.NODE_ENV === 'development'; + provider.addSpanProcessor( + isDev ? new SimpleSpanProcessor(exporter) : new BatchSpanProcessor(exporter) + ); + provider.register(); + + console.log(`[mana-ai] OTel tracing enabled → ${OTEL_ENDPOINT}/v1/traces`); +} else { + console.log('[mana-ai] OTel tracing disabled (set OTEL_EXPORTER_OTLP_ENDPOINT to enable)'); +} + +/** The mana-ai tracer instance. When OTel is not configured, all + * operations are no-ops (the API guarantees this). */ +export const tracer: Tracer = trace.getTracer('mana-ai', '0.6.0'); + +/** + * Execute an async function within a traced span. Automatically: + * - Sets span attributes from the provided record + * - Marks the span as ERROR on throw + * - Ends the span in all cases + */ +export async function withSpan( + name: string, + attributes: Record, + fn: (span: Span) => Promise, + options?: SpanOptions +): Promise { + return tracer.startActiveSpan(name, options ?? {}, async (span) => { + for (const [key, value] of Object.entries(attributes)) { + span.setAttribute(key, value); + } + try { + const result = await fn(span); + return result; + } catch (err) { + span.setStatus({ + code: SpanStatusCode.ERROR, + message: err instanceof Error ? err.message : String(err), + }); + throw err; + } finally { + span.end(); + } + }); +}