diff --git a/.gitignore b/.gitignore index 6d4526ca0..3cb6c6b70 100644 --- a/.gitignore +++ b/.gitignore @@ -58,6 +58,8 @@ coverage/ .nyc_output/ test-results/ playwright-report/ +playwright-report-personas/ +test-results/ .auth-state.json # TypeScript diff --git a/docs/plans/mana-mcp-and-personas.md b/docs/plans/mana-mcp-and-personas.md index 28e87f5bc..951aeb014 100644 --- a/docs/plans/mana-mcp-and-personas.md +++ b/docs/plans/mana-mcp-and-personas.md @@ -465,7 +465,22 @@ Full tick loop live. End-to-end pipeline proven through type-check + boot smoke; **Exit criteria:** Grep nach duplizierten Tool-Definitionen → leer. Beide Consumer grün. -### M5 — Visual Suite +### M5 — Visual Suite — ✅ M5.a scaffold SHIPPED 2026-04-23 + +Single-flow foundation. Fixture + config + one spec + README + pnpm-scripts. Extension is copy-paste per module. Live baseline capture is user-side (needs running stack + seeded + persona-runner ticked). + +- [x] `tests/personas/playwright.config.ts` — own config, 2 viewports (desktop + mobile Pixel 5), 0.2 % diff threshold, animations disabled, `snapshotPathTemplate` scoped to per-spec folder, no auto-webServer (regressions only matter against a real running stack) +- [x] `tests/personas/fixtures/persona-auth.ts` — HMAC-SHA256 password derivation (bit-identical mirror of `scripts/personas/password.ts` + `services/mana-persona-runner/src/password.ts` — **3-way contract**, changing one breaks the others), Set-Cookie parsing, typed `test.extend` with `personaKey` worker option + `personaPage` fixture +- [x] `tests/personas/flows/home.spec.ts` — smoke flow, captures home-tour screenshot as `anna-adhd-student` +- [x] `tests/personas/README.md` — prerequisites, run recipe, architecture diagram, "adding a flow" steps +- [x] `pnpm test:personas` + `pnpm test:personas:update` on the root + +**Deferred** (copy-paste extensions once user has stack running): +- Per-module flows: todo, journal, notes, habits, calendar, contacts +- Additional viewports (iPad, webkit) +- Nightly CI job via GitHub Action or Mac-Mini cron + +#### Archived initial checklist - [ ] `tests/personas/` Struktur - [ ] Persona-Login-Fixture (API-Login → storageState) diff --git a/package.json b/package.json index 869cf7cfb..d533e60c3 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,8 @@ "setup:dev-user": "./scripts/dev/setup-dev-user.sh", "seed:personas": "bun run scripts/personas/seed.ts", "seed:personas:cleanup": "bun run scripts/personas/cleanup.ts", + "test:personas": "pnpm --filter @mana/tests-personas test", + "test:personas:update": "pnpm --filter @mana/tests-personas test:update", "build:packages": "pnpm --filter './packages/*' build", "postinstall": "node scripts/generate-env.mjs && pnpm run build:packages", "mana:dev": "turbo run dev --filter=mana...", diff --git a/tests/personas/README.md b/tests/personas/README.md new file mode 100644 index 000000000..54b31b7bd --- /dev/null +++ b/tests/personas/README.md @@ -0,0 +1,97 @@ +# Persona Visual Suite + +Playwright-driven visual regression tests that log in as each of the **M2 personas**, navigate through the modules the persona uses, and diff screenshots against committed baselines. Detects UI regressions before real users do. + +**Plan:** [`docs/plans/mana-mcp-and-personas.md`](../../docs/plans/mana-mcp-and-personas.md) (M5) + +## Prerequisites + +The suite assumes the full local stack is already running and **seeded with persona data**. It does *not* auto-start anything — a silent "all green" against an empty seed would hide regressions, not find them. + +```bash +# 1. Stack up + seeded +pnpm docker:up +cd services/mana-auth && bun run db:push +pnpm dev:auth # mana-auth on 3001 +pnpm dev:sync # mana-sync on 3050 +pnpm mana:dev # web app on 5173 — the suite hits this + +# 2. Seed the 10 catalog personas +export MANA_ADMIN_JWT=… +export PERSONA_SEED_SECRET=… # same value the suite uses below +pnpm seed:personas + +# 3. Optional but recommended: run the persona-runner once so each +# persona has real content. Empty accounts make empty baselines. +pnpm --filter @mana/mcp-service dev # :3069 +pnpm --filter @mana/persona-runner dev # :3070 +export MANA_SERVICE_KEY=… +export ANTHROPIC_API_KEY=… +curl -X POST localhost:3070/diag/tick +``` + +## Running + +```bash +cd tests/personas + +# First run — captures baselines (or diffs against existing ones): +pnpm test + +# Accept current render as new baseline (after intentional UI changes): +pnpm test:update + +# View the HTML report with diff visualisations: +pnpm report +``` + +## Environment + +| Var | Default | Purpose | +|---|---|---| +| `PERSONAS_AUTH_URL` | `http://localhost:3001` | mana-auth for API login | +| `PERSONAS_BASE_URL` | `http://localhost:5173` | web app the browser loads | +| `PERSONAS_COOKIE_DOMAIN` | derived from `PERSONAS_BASE_URL` host | Where better-auth's cookie lands | +| `PERSONA_SEED_SECRET` | dev fallback | MUST match `scripts/personas/password.ts` and the runner | + +## Architecture + +``` +┌──────────────────────────────────────────────┐ +│ tests/personas/fixtures/persona-auth.ts │ +│ ├── personaPassword(email) — HMAC-SHA256 │ +│ ├── loginAndGetCookies — POST /auth/login │ +│ └── test.use({ personaKey: 'anna' }) │ +└──────────────────────┬───────────────────────┘ + │ + ▼ + BrowserContext.addCookies(better-auth session) + │ + ▼ + page.goto('/') — logged in! + │ + ▼ + flow-specific assertions + toHaveScreenshot() + │ + ▼ + ./__snapshots__//-.png +``` + +## Adding a flow + +Copy `flows/home.spec.ts` to `flows/.spec.ts`, change: + +- `test.use({ personaKey: '…' })` to the right persona +- the `goto('/')` path +- the screenshot filename + +Run `pnpm test:update` once to capture the baseline, commit the new PNG, done. CI diffs on every future run. + +## Design notes + +- **No webServer in config**: the suite runs against the *running* stack, not a hermetic one. That's the point — we want to catch regressions in the app that real users would see, not just the test build. If the stack is down, all tests fail loud with a descriptive login error. +- **API-login, not form-login**: the login UI is tested elsewhere; re-driving it per persona × per spec would add minutes to every run with no signal. +- **Two viewports, one browser**: `desktop` (1440×900 Chrome) + `mobile` (Pixel 5). Enough to catch most layout regressions without quadrupling the baseline count. Add `iPad` / `webkit` after the suite has settled. +- **Diff threshold 0.2%**: tight enough to catch real regressions, loose enough to ignore antialiasing noise. Tune if false positives emerge. +- **`animations: 'disabled'`**: Playwright pauses CSS animations for the screenshot, so transitions don't bleed into diffs. +- **Live timestamps hidden**: any element with `data-testid="live-time"` gets `visibility: hidden` before the screenshot. Add that testid to clock/relative-time components. diff --git a/tests/personas/fixtures/persona-auth.ts b/tests/personas/fixtures/persona-auth.ts new file mode 100644 index 000000000..b28ca737e --- /dev/null +++ b/tests/personas/fixtures/persona-auth.ts @@ -0,0 +1,171 @@ +/** + * API-login fixture. + * + * The web app's normal login flow is a visual concern — we're testing + * downstream views, not the login UI, so we skip the form. Instead: + * + * 1. Derive the persona's password (same HMAC algorithm as the seed + * script + the persona-runner — all three must stay in sync). + * 2. POST /api/v1/auth/login to capture the better-auth session cookie. + * 3. Inject the cookie into the Playwright browser context, then goto /. + * + * Tests opt in via the typed `test.use({ persona: 'anna' })` below; + * they receive a logged-in `page` with `persona.email` available. + */ + +import { test as base, expect, type BrowserContext, type Page } from '@playwright/test'; +import { createHmac } from 'node:crypto'; + +const AUTH_URL = process.env.PERSONAS_AUTH_URL ?? 'http://localhost:3001'; +const BASE_URL = process.env.PERSONAS_BASE_URL ?? 'http://localhost:5173'; +const COOKIE_DOMAIN = process.env.PERSONAS_COOKIE_DOMAIN ?? new URL(BASE_URL).hostname; +const SEED_SECRET = process.env.PERSONA_SEED_SECRET ?? 'dev-persona-seed-secret-rotate-in-prod'; + +/** + * Must stay bit-identical to `scripts/personas/password.ts` and + * `services/mana-persona-runner/src/password.ts`. Changing one without + * the others locks the test suite out of every persona. + */ +function personaPassword(email: string): string { + const hmac = createHmac('sha256', SEED_SECRET).update(email).digest('base64'); + return hmac.replace(/[^a-zA-Z0-9]/g, '').slice(0, 32); +} + +export interface PersonaInfo { + name: string; + email: string; + archetype: string; +} + +export const PERSONAS: Record = { + anna: { name: 'Anna', email: 'persona.anna@mana.test', archetype: 'adhd-student' }, + ben: { name: 'Ben', email: 'persona.ben@mana.test', archetype: 'adhd-student' }, + marcus: { name: 'Marcus', email: 'persona.marcus@mana.test', archetype: 'ceo-busy' }, + lena: { name: 'Lena', email: 'persona.lena@mana.test', archetype: 'ceo-busy' }, + sofia: { name: 'Sofia', email: 'persona.sofia@mana.test', archetype: 'creative-parent' }, + tom: { name: 'Tom', email: 'persona.tom@mana.test', archetype: 'creative-parent' }, + kai: { name: 'Kai', email: 'persona.kai@mana.test', archetype: 'solo-dev' }, + julia: { name: 'Julia', email: 'persona.julia@mana.test', archetype: 'researcher' }, + paul: { name: 'Paul', email: 'persona.paul@mana.test', archetype: 'freelancer' }, + maya: { name: 'Maya', email: 'persona.maya@mana.test', archetype: 'overwhelmed-newbie' }, +} as const; + +export type PersonaKey = keyof typeof PERSONAS; + +interface SetCookie { + name: string; + value: string; + path?: string; + domain?: string; + expires?: number; + httpOnly?: boolean; + secure?: boolean; + sameSite?: 'Strict' | 'Lax' | 'None'; +} + +/** + * Log in over HTTP, return the Set-Cookie headers as Playwright-compatible + * cookie objects ready for `context.addCookies()`. + */ +async function loginAndGetCookies(email: string): Promise { + const res = await fetch(`${AUTH_URL}/api/v1/auth/login`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ email, password: personaPassword(email) }), + }); + if (!res.ok) { + const text = await res.text().catch(() => ''); + throw new Error( + `Persona login ${email} failed: HTTP ${res.status} — ${text.slice(0, 300)}. ` + + `Is mana-auth running at ${AUTH_URL}? Has \`pnpm seed:personas\` been run?` + ); + } + const raw = + res.headers.getSetCookie?.() ?? res.headers.get('set-cookie')?.split(/, (?=[^ ])/) ?? []; + const cookies: SetCookie[] = []; + for (const line of raw) { + const cookie = parseSetCookie(line); + if (cookie) cookies.push(cookie); + } + if (cookies.length === 0) { + throw new Error( + `Persona login ${email} succeeded but no Set-Cookie headers — better-auth config drift?` + ); + } + return cookies; +} + +function parseSetCookie(header: string): SetCookie | null { + const parts = header.split(';').map((p) => p.trim()); + const [nv, ...rest] = parts; + const eq = nv.indexOf('='); + if (eq < 0) return null; + const cookie: SetCookie = { + name: nv.slice(0, eq), + value: nv.slice(eq + 1), + path: '/', + domain: COOKIE_DOMAIN, + }; + for (const attr of rest) { + const [rk, rv] = attr.split('=', 2); + const key = rk.toLowerCase(); + if (key === 'path' && rv) cookie.path = rv; + else if (key === 'domain' && rv) cookie.domain = rv.replace(/^\./, ''); + else if (key === 'httponly') cookie.httpOnly = true; + else if (key === 'secure') cookie.secure = true; + else if (key === 'samesite' && rv) { + const s = rv as 'Strict' | 'Lax' | 'None'; + cookie.sameSite = s; + } + } + return cookie; +} + +export async function loginPersonaContext(context: BrowserContext, email: string): Promise { + const cookies = await loginAndGetCookies(email); + await context.addCookies( + cookies.map((c) => ({ + name: c.name, + value: c.value, + domain: c.domain ?? COOKIE_DOMAIN, + path: c.path ?? '/', + httpOnly: c.httpOnly, + secure: c.secure, + sameSite: c.sameSite, + })) + ); +} + +// ─── Typed test fixture ─────────────────────────────────────────── + +type PersonaTestFixtures = { + persona: PersonaInfo; + /** Pre-logged-in page pointed at `/`. Waiting for networkidle already done. */ + personaPage: Page; +}; + +type PersonaWorkerOptions = { + personaKey: PersonaKey; +}; + +/** + * Use `test.extend<{}, PersonaWorkerOptions>` and declare `test.use({ personaKey: 'anna' })` + * at the top of each spec. Each spec gets a freshly-logged-in page. + */ +export const test = base.extend({ + personaKey: ['anna', { option: true, scope: 'worker' }], + persona: async ({ personaKey }, use) => { + const info = PERSONAS[personaKey]; + if (!info) throw new Error(`Unknown persona key: ${personaKey}`); + await use(info); + }, + personaPage: async ({ context, persona }, use) => { + await loginPersonaContext(context, persona.email); + const page = await context.newPage(); + await page.goto('/'); + await page.waitForLoadState('networkidle'); + await use(page); + }, +}); + +export { expect }; diff --git a/tests/personas/flows/home.spec.ts b/tests/personas/flows/home.spec.ts new file mode 100644 index 000000000..af41a9e6c --- /dev/null +++ b/tests/personas/flows/home.spec.ts @@ -0,0 +1,40 @@ +/** + * Home-tour spec — smallest possible end-to-end signal. + * + * Runs the suite's auth fixture, lands on `/`, verifies the app loaded + * under the persona's account (dashboard chrome visible), captures a + * baseline screenshot. + * + * When this goes green, the whole persona-visual stack is plumbed: + * seed-script → mana-auth → API login → cookie injection → web app → + * screenshot. Copy this file, change the route in `page.goto`, and you + * have a new module flow. + */ + +import { test, expect } from '../fixtures/persona-auth'; + +// Worker-scoped fixture — must be top-level, not inside describe. +test.use({ personaKey: 'anna' }); + +test.describe('home — Anna', () => { + test('dashboard renders', async ({ personaPage, persona }) => { + // Sanity: we're logged in as the right user. + // The URL should be inside the authenticated (app) group, not /login. + await expect(personaPage).not.toHaveURL(/\/login(\?|$)/); + + // Give any lazy-loaded dashboard widgets a beat to settle, then + // freeze dynamic timestamps so screenshots are deterministic. + await personaPage.waitForLoadState('networkidle'); + await personaPage.evaluate(() => { + // Hide any element that renders a live clock / relative time. + // Tests can update this list if specific selectors are known. + for (const el of document.querySelectorAll('[data-testid="live-time"]')) { + el.style.visibility = 'hidden'; + } + }); + + await expect(personaPage).toHaveScreenshot(`home-${persona.archetype}.png`, { + fullPage: true, + }); + }); +}); diff --git a/tests/personas/package.json b/tests/personas/package.json new file mode 100644 index 000000000..9fc053eeb --- /dev/null +++ b/tests/personas/package.json @@ -0,0 +1,15 @@ +{ + "name": "@mana/tests-personas", + "version": "0.1.0", + "private": true, + "description": "Visual regression suite driven by the M2 personas (see docs/plans/mana-mcp-and-personas.md M5).", + "type": "module", + "scripts": { + "test": "playwright test --config=playwright.config.ts", + "test:update": "playwright test --config=playwright.config.ts --update-snapshots", + "report": "playwright show-report playwright-report-personas" + }, + "devDependencies": { + "@playwright/test": "^1.49.0" + } +} diff --git a/tests/personas/playwright.config.ts b/tests/personas/playwright.config.ts new file mode 100644 index 000000000..de353442a --- /dev/null +++ b/tests/personas/playwright.config.ts @@ -0,0 +1,55 @@ +/** + * Persona visual regression suite. + * + * Separate from the repo's main Playwright config (`../../playwright.config.ts`) + * because this suite has different defaults: fewer viewports (baselines + * cost disk + review time), tighter diff threshold, and a deterministic + * animation/font-loading setup so the same code produces the same + * pixels across runs. + * + * Plan: docs/plans/mana-mcp-and-personas.md (M5). + */ + +import { defineConfig, devices } from '@playwright/test'; + +const BASE = process.env.PERSONAS_BASE_URL ?? 'http://localhost:5173'; + +export default defineConfig({ + testDir: './flows', + // Baselines are stable across OS as long as `--project=…` is pinned + // and the server is the same. Diff threshold is tight; expect a few + // real-UI-change updates per week. + expect: { + toHaveScreenshot: { + maxDiffPixelRatio: 0.002, + animations: 'disabled', + }, + }, + snapshotPathTemplate: '{testDir}/__snapshots__/{testFilePath}/{arg}-{projectName}{ext}', + fullyParallel: true, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 1 : 0, + // Avoid cross-contamination — each persona has its own storageState, + // running in parallel is safe but keep it modest to not thrash mana-auth. + workers: process.env.CI ? 2 : undefined, + reporter: [['html', { outputFolder: 'playwright-report-personas' }], ['list']], + use: { + baseURL: BASE, + trace: 'retain-on-failure', + screenshot: 'only-on-failure', + video: 'retain-on-failure', + }, + // Two viewports as a starting point — one desktop, one mobile. Add + // more later (iPad, large desktop) once the baselines are stable. + projects: [ + { + name: 'desktop', + use: { ...devices['Desktop Chrome'], viewport: { width: 1440, height: 900 } }, + }, + { name: 'mobile', use: { ...devices['Pixel 5'] } }, + ], + // webServer stays OFF here — this suite expects the full local stack + // (mana-auth, mana-sync, web app) to already be running. Persona + // tests assume real data has been seeded; auto-starting only the web + // app would give a meaningless pass. Run-recipe lives in README.md. +});