From 610c8e37093d7cc454804c51a8fb58efee66b8cd Mon Sep 17 00:00:00 2001 From: kingwl Date: Wed, 8 Jul 2026 02:05:06 +0800 Subject: [PATCH] test(acp-snapshot): fake ACP bin + unit specs to per-file 100% coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A scripted fake ACP agent bin (tests/fixtures/fake-acp-agent.ts) speaks real newline JSON-RPC through the REAL runScenario spawn path (tsx loader, temp cwd, env plumbing); every behavior — prompt outcome, session/new rejection, persisted logs, filesystem noise — comes from a behavior.json beside the fixture, so specs script whole subprocess runs from data. harness.spec.ts drives every step op, both expect-error arms, the permission-stub default, env forwarding, workspace seeding, and the harvest ordering/noise/fallback branches. suite.spec.ts runs the factory for real at collection time: a replay suite over committed synthetic fixtures and a record suite over a temp copy (write-back never touches the committed tree; ACP_SNAPSHOT_SPEC_BOOTSTRAP=1 re-bootstraps it), plus direct cases for the exported pure helpers. The suite factory's pure helpers (childFixturePaths, fixtureContext, normalizedHeaders, headerDeltaCount) are exported for those direct specs. Two branches carry justified v8 ignores, both structurally unreachable: the waiter in-bounds guard (noUncheckedIndexedAccess) and waitForExit's already-exited race guard (both call sites sit one synchronous frame after stdin.end()/kill()). The fake bin substitutes the session/new cwd, not process.cwd(), into scripted logs — the realpath difference (/private on darwin) is exactly what the real bin's header carries. packages/support/acp-snapshot/src is at 100% statements, branches, functions, and lines under the per-file gate. --- knip.json | 2 +- packages/support/acp-snapshot/src/harness.ts | 11 +- packages/support/acp-snapshot/src/suite.ts | 30 ++- .../tests/fixtures/fake-acp-agent.ts | 232 +++++++++++++++++ .../record-suite/rec-child/behavior.json | 13 + .../record-suite/rec-child/input.json | 1 + .../record-suite/rec-child/session.1.jsonl | 2 + .../record-suite/rec-child/session.jsonl | 2 + .../rec-child/stdout.golden.jsonl | 4 + .../record-suite/rec-pin/behavior.json | 10 + .../fixtures/record-suite/rec-pin/input.json | 1 + .../record-suite/rec-pin/session.jsonl | 2 + .../record-suite/rec-pin/stdout.golden.jsonl | 4 + .../record-suite/rec-skip/behavior.json | 1 + .../fixtures/record-suite/rec-skip/input.json | 1 + .../rec-skip/replay.override.json | 1 + .../record-suite/rec-skip/session.jsonl | 1 + .../record-suite/rec-skip/stdout.golden.jsonl | 1 + .../suite/authored-error/behavior.json | 10 + .../fixtures/suite/authored-error/input.json | 1 + .../suite/authored-error/replay.override.json | 1 + .../suite/authored-error/session.jsonl | 2 + .../suite/authored-error/stdout.golden.jsonl | 4 + .../fixtures/suite/blocked-log/behavior.json | 10 + .../fixtures/suite/blocked-log/input.json | 1 + .../fixtures/suite/blocked-log/session.jsonl | 2 + .../suite/blocked-log/stdout.golden.jsonl | 4 + .../fixtures/suite/no-model/behavior.json | 1 + .../tests/fixtures/suite/no-model/input.json | 1 + .../fixtures/suite/no-model/session.jsonl | 1 + .../suite/no-model/stdout.golden.jsonl | 1 + .../fixtures/suite/pin-turn/behavior.json | 11 + .../tests/fixtures/suite/pin-turn/input.json | 1 + .../fixtures/suite/pin-turn/session.jsonl | 3 + .../suite/pin-turn/stdout.golden.jsonl | 4 + .../fixtures/suite/plain-turn/behavior.json | 15 ++ .../fixtures/suite/plain-turn/input.json | 1 + .../fixtures/suite/plain-turn/session.1.jsonl | 2 + .../fixtures/suite/plain-turn/session.jsonl | 3 + .../suite/plain-turn/stdout.golden.jsonl | 5 + .../suite/plain-turn/workspace/seed.txt | 1 + .../acp-snapshot/tests/harness.spec.ts | 233 ++++++++++++++++++ .../acp-snapshot/tests/normalize.spec.ts | 45 ++++ .../support/acp-snapshot/tests/suite.spec.ts | 145 +++++++++++ 44 files changed, 819 insertions(+), 8 deletions(-) create mode 100644 packages/support/acp-snapshot/tests/fixtures/fake-acp-agent.ts create mode 100644 packages/support/acp-snapshot/tests/fixtures/record-suite/rec-child/behavior.json create mode 100644 packages/support/acp-snapshot/tests/fixtures/record-suite/rec-child/input.json create mode 100644 packages/support/acp-snapshot/tests/fixtures/record-suite/rec-child/session.1.jsonl create mode 100644 packages/support/acp-snapshot/tests/fixtures/record-suite/rec-child/session.jsonl create mode 100644 packages/support/acp-snapshot/tests/fixtures/record-suite/rec-child/stdout.golden.jsonl create mode 100644 packages/support/acp-snapshot/tests/fixtures/record-suite/rec-pin/behavior.json create mode 100644 packages/support/acp-snapshot/tests/fixtures/record-suite/rec-pin/input.json create mode 100644 packages/support/acp-snapshot/tests/fixtures/record-suite/rec-pin/session.jsonl create mode 100644 packages/support/acp-snapshot/tests/fixtures/record-suite/rec-pin/stdout.golden.jsonl create mode 100644 packages/support/acp-snapshot/tests/fixtures/record-suite/rec-skip/behavior.json create mode 100644 packages/support/acp-snapshot/tests/fixtures/record-suite/rec-skip/input.json create mode 100644 packages/support/acp-snapshot/tests/fixtures/record-suite/rec-skip/replay.override.json create mode 100644 packages/support/acp-snapshot/tests/fixtures/record-suite/rec-skip/session.jsonl create mode 100644 packages/support/acp-snapshot/tests/fixtures/record-suite/rec-skip/stdout.golden.jsonl create mode 100644 packages/support/acp-snapshot/tests/fixtures/suite/authored-error/behavior.json create mode 100644 packages/support/acp-snapshot/tests/fixtures/suite/authored-error/input.json create mode 100644 packages/support/acp-snapshot/tests/fixtures/suite/authored-error/replay.override.json create mode 100644 packages/support/acp-snapshot/tests/fixtures/suite/authored-error/session.jsonl create mode 100644 packages/support/acp-snapshot/tests/fixtures/suite/authored-error/stdout.golden.jsonl create mode 100644 packages/support/acp-snapshot/tests/fixtures/suite/blocked-log/behavior.json create mode 100644 packages/support/acp-snapshot/tests/fixtures/suite/blocked-log/input.json create mode 100644 packages/support/acp-snapshot/tests/fixtures/suite/blocked-log/session.jsonl create mode 100644 packages/support/acp-snapshot/tests/fixtures/suite/blocked-log/stdout.golden.jsonl create mode 100644 packages/support/acp-snapshot/tests/fixtures/suite/no-model/behavior.json create mode 100644 packages/support/acp-snapshot/tests/fixtures/suite/no-model/input.json create mode 100644 packages/support/acp-snapshot/tests/fixtures/suite/no-model/session.jsonl create mode 100644 packages/support/acp-snapshot/tests/fixtures/suite/no-model/stdout.golden.jsonl create mode 100644 packages/support/acp-snapshot/tests/fixtures/suite/pin-turn/behavior.json create mode 100644 packages/support/acp-snapshot/tests/fixtures/suite/pin-turn/input.json create mode 100644 packages/support/acp-snapshot/tests/fixtures/suite/pin-turn/session.jsonl create mode 100644 packages/support/acp-snapshot/tests/fixtures/suite/pin-turn/stdout.golden.jsonl create mode 100644 packages/support/acp-snapshot/tests/fixtures/suite/plain-turn/behavior.json create mode 100644 packages/support/acp-snapshot/tests/fixtures/suite/plain-turn/input.json create mode 100644 packages/support/acp-snapshot/tests/fixtures/suite/plain-turn/session.1.jsonl create mode 100644 packages/support/acp-snapshot/tests/fixtures/suite/plain-turn/session.jsonl create mode 100644 packages/support/acp-snapshot/tests/fixtures/suite/plain-turn/stdout.golden.jsonl create mode 100644 packages/support/acp-snapshot/tests/fixtures/suite/plain-turn/workspace/seed.txt create mode 100644 packages/support/acp-snapshot/tests/harness.spec.ts create mode 100644 packages/support/acp-snapshot/tests/suite.spec.ts diff --git a/knip.json b/knip.json index b5ff3c2e5d..8c0f71f3af 100644 --- a/knip.json +++ b/knip.json @@ -22,7 +22,7 @@ "ignoreDependencies": ["cordis"] }, "packages/support/acp-snapshot": { - "entry": ["tests/**/*.spec.ts"], + "entry": ["tests/**/*.spec.ts", "tests/fixtures/fake-acp-agent.ts"], "project": ["src/**/*.ts", "tests/**/*.ts"], "ignoreDependencies": ["cordis"] }, diff --git a/packages/support/acp-snapshot/src/harness.ts b/packages/support/acp-snapshot/src/harness.ts index a46f752d5c..652e60a469 100644 --- a/packages/support/acp-snapshot/src/harness.ts +++ b/packages/support/acp-snapshot/src/harness.ts @@ -224,7 +224,12 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise sessionUpdate(params: SessionNotification): Promise { for (let i = updateWaiters.length - 1; i >= 0; i--) { const waiter = updateWaiters[i] - if (waiter !== undefined && waiter.match(params.update)) { + // The index is always in-bounds (i only decreases; splice removes at + // i, so lower entries stay valid); the guard satisfies + // noUncheckedIndexedAccess. + /* v8 ignore next 1 -- unreachable in-bounds guard, see above */ + if (waiter === undefined) continue + if (waiter.match(params.update)) { updateWaiters.splice(i, 1) waiter.resolve() } @@ -351,6 +356,10 @@ async function runStep( /** Resolve once the child process exits (any code/signal). */ function waitForExit(child: ChildProcessWithoutNullStreams): Promise { + // Race guard: both call sites run within one synchronous frame of + // stdin.end()/kill(), so the exit event cannot have been delivered yet; + // kept for any future caller that awaits in between. + /* v8 ignore next 1 -- unreachable race guard, see above */ if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve() return new Promise(resolve => child.once('exit', () => { resolve() })) } diff --git a/packages/support/acp-snapshot/src/suite.ts b/packages/support/acp-snapshot/src/suite.ts index 48d6692258..14a4df54da 100644 --- a/packages/support/acp-snapshot/src/suite.ts +++ b/packages/support/acp-snapshot/src/suite.ts @@ -101,8 +101,14 @@ export interface SnapshotSuiteOptions { mode: 'replay' | 'record' } -/** The sibling child-fixture paths for a scenario (`session.1.jsonl` …). */ -function childFixturePaths(dir: string, childSessions: number): string[] { +/** + * The sibling child-fixture paths for a scenario (`session.1.jsonl` …). + * + * @param dir The scenario's snapshots directory (`/`). + * @param childSessions How many subagent child sessions the scenario records. + * @returns One path per child, 1-based, in fixture order. + */ +export function childFixturePaths(dir: string, childSessions: number): string[] { return Array.from({ length: childSessions }, (_, i) => join(dir, `session.${i + 1}.jsonl`)) } @@ -118,8 +124,11 @@ function childFixturePaths(dir: string, childSessions: number): string[] { * is an idempotent no-op. A header with no `cwd` falls back to a sentinel that * cannot occur in a log (NOT `''`, which `String.split` would match on every * character boundary and corrupt the output). + * + * @param fixture The committed `session.jsonl` content. + * @returns The fixture's own volatile values, ready for {@link normalizeSessionLog}. */ -function fixtureContext(fixture: string): NormalizeContext { +export function fixtureContext(fixture: string): NormalizeContext { const firstLine = fixture.split('\n').find(line => line.trim().length > 0) ?? '{}' const header = JSON.parse(firstLine) as { id?: unknown; cwd?: unknown } return { @@ -134,8 +143,12 @@ function fixtureContext(fixture: string): NormalizeContext { * ({@link normalizeSessionLog}) so headers harvested from different runs — * each embedding its own temp cwd in the composed prompt — compare on equal * footing. + * + * @param rawLog The session `.jsonl` content to extract headers from. + * @param ctx The volatile values of the run that produced it. + * @returns The normalized `data.header` payloads, in log order. */ -function normalizedHeaders(rawLog: string, ctx: NormalizeContext): unknown[] { +export function normalizedHeaders(rawLog: string, ctx: NormalizeContext): unknown[] { return normalizeSessionLog(rawLog, ctx) .split('\n') .filter(line => line.trim().length > 0) @@ -144,8 +157,13 @@ function normalizedHeaders(rawLog: string, ctx: NormalizeContext): unknown[] { .map(record => record.data?.header) } -/** Count the `request/header-delta` events in a session JSONL. */ -function headerDeltaCount(rawLog: string): number { +/** + * Count the `request/header-delta` events in a session JSONL. + * + * @param rawLog The session `.jsonl` content. + * @returns How many `request/header-delta` events the log carries. + */ +export function headerDeltaCount(rawLog: string): number { return rawLog.split('\n') .filter(line => line.trim().length > 0) .filter(line => (JSON.parse(line) as { type?: unknown }).type === 'request/header-delta') diff --git a/packages/support/acp-snapshot/tests/fixtures/fake-acp-agent.ts b/packages/support/acp-snapshot/tests/fixtures/fake-acp-agent.ts new file mode 100644 index 0000000000..cf41412046 --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/fake-acp-agent.ts @@ -0,0 +1,232 @@ +/** + * Scripted fake ACP agent bin for `dsh-acp-snapshot`'s unit specs. Speaks + * newline-delimited JSON-RPC on stdio like the real `dsh-acp-agent` bin, but + * every behavior — how prompts settle, whether session/new rejects, which + * session logs get persisted, what filesystem noise to leave — comes from a + * `behavior.json` sitting NEXT to the `$DSH_SNAPSHOT_FILE` fixture, so a spec + * scripts a whole subprocess run from data. The specs launch it through the + * REAL `runScenario` spawn path (tsx loader, temp cwd, env plumbing), so the + * harness plumbing is exercised for real; only the agent behind the protocol + * is scripted. + * + * The specs (not the golden tier) own this bin: it asserts nothing, echoes + * observable facts into `session/update` text chunks (env probe, permission + * outcome, seeded-workspace listing) for the spec to read off `rawStdout`, and + * exits 0 on stdin EOF after writing the scripted logs — mirroring the real + * bin's dispose-flush-exit shape. + */ + +import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { readdirSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { randomUUID } from 'node:crypto' +import { createInterface } from 'node:readline' + +/** One scripted session log: a file path under the sessions root plus its JSONL lines. */ +interface ScriptedLog { + /** Path relative to `$DSH_SNAPSHOT_SESSIONS_ROOT`, e.g. `bucket/a.jsonl` (an empty dir segment is invalid). */ + file: string + /** + * The JSONL records. String templates `{{CWD}}` and `{{SID}}` are replaced + * with the run's real cwd and the ACP session id this bin issued, so a + * written log carries genuine volatile values for the normalizers to scrub. + */ + lines: unknown[] +} + +/** The whole scripted behavior for one run. Every field defaults to the least surprising choice. */ +interface Behavior { + /** Reject every `session/new` (exercises the expect-error step without extra dirs). */ + rejectNewSession?: boolean + /** Reject `session/new` only when `additionalDirectories` is non-empty (the real bridge's rule). */ + rejectExtraDirs?: boolean + /** How `session/prompt` settles: a clean response, a JSON-RPC error, or a hang until `session/cancel`. */ + prompt?: 'respond' | 'error' | 'hang-until-cancel' + /** Before responding to a prompt, send a `session/request_permission` request and echo its outcome as a chunk. */ + permissionProbe?: boolean + /** Echo the `DSH_SNAPSHOT_*` env the harness set as a chunk (spec-side env-plumbing assertions). */ + echoEnv?: boolean + /** Echo the sorted cwd listing as a chunk (spec-side workspace-seeding assertions). */ + echoWorkspace?: boolean + /** Write a line to stderr on boot (spec-side stderr-capture assertions). */ + stderrNote?: string + /** Session logs to persist on stdin EOF. */ + logs?: ScriptedLog[] + /** Leave a stray FILE directly under the sessions root (harvest must skip it). */ + strayRootFile?: boolean + /** Leave a stray non-`.jsonl` file inside a bucket (harvest must skip it). */ + strayBucketFile?: boolean + /** Delete the sessions root entirely (harvest must yield no logs). */ + deleteSessionsRoot?: boolean +} + +const sessionsRoot = process.env.DSH_SNAPSHOT_SESSIONS_ROOT ?? '' +const fixtureFile = process.env.DSH_SNAPSHOT_FILE ?? '' +const behavior: Behavior = fixtureFile === '' + ? {} + : JSON.parse(readFileSync(join(dirname(fixtureFile), 'behavior.json'), 'utf8')) as Behavior + +if (behavior.stderrNote !== undefined) process.stderr.write(`${behavior.stderrNote}\n`) + +let nextOutboundId = 1000 +let sessionId = '' +/** + * The cwd the client passed to `session/new` — used verbatim for `{{CWD}}` + * substitution, mirroring the real bin (whose persisted header carries the + * session cwd as given, NOT `process.cwd()`, which the OS realpaths — on + * macOS `/var/folders/…` vs `/private/var/folders/…`). + */ +let sessionCwd = '' +/** The parked prompt request id while `hang-until-cancel` waits for the cancel notification. */ +let parkedPromptId: number | string | null = null +/** Resolvers for permission-probe responses, keyed by outbound request id. */ +const pendingPermission = new Map void>() + +function send(frame: Record): void { + process.stdout.write(`${JSON.stringify({ jsonrpc: '2.0', ...frame })}\n`) +} + +function respond(id: number | string, result: unknown): void { + send({ id, result }) +} + +function respondError(id: number | string, message: string): void { + send({ id, error: { code: -32603, message } }) +} + +function chunk(text: string): void { + send({ + method: 'session/update', + params: { sessionId, update: { sessionUpdate: 'agent_message_chunk', content: { type: 'text', text } } }, + }) +} + +/** Substitute the `{{CWD}}`/`{{SID}}` templates through a scripted log record. */ +function instantiate(value: unknown): unknown { + if (typeof value === 'string') return value.split('{{CWD}}').join(sessionCwd).split('{{SID}}').join(sessionId) + if (Array.isArray(value)) return value.map(instantiate) + if (value !== null && typeof value === 'object') { + const out: Record = {} + for (const [k, v] of Object.entries(value)) out[k] = instantiate(v) + return out + } + return value +} + +async function handlePrompt(id: number | string): Promise { + if ((behavior.prompt ?? 'respond') === 'hang-until-cancel') { + // A thought chunk BEFORE any message chunk: a promptAndCancel waiter + // watches for agent_message_chunk, so this exercises its non-matching + // update path while the waiter is armed. + send({ + method: 'session/update', + params: { sessionId, update: { sessionUpdate: 'agent_thought_chunk', content: { type: 'text', text: 'mulling' } } }, + }) + } + chunk('thinking about it') + if (behavior.echoEnv === true) { + chunk(`env:${JSON.stringify({ + mode: process.env.DSH_SNAPSHOT, + override: process.env.DSH_SNAPSHOT_OVERRIDE ?? null, + childFiles: process.env.DSH_SNAPSHOT_CHILD_FILES ?? null, + })}`) + } + if (behavior.echoWorkspace === true) { + chunk(`workspace:${readdirSync(process.cwd()).sort().join(',')}`) + } + if (behavior.permissionProbe === true) { + const requestId = nextOutboundId++ + const outcome = await new Promise((resolve) => { + pendingPermission.set(requestId, resolve) + send({ + id: requestId, + method: 'session/request_permission', + params: { + sessionId, + toolCall: { toolCallId: 'call_fake_1', title: 'fake tool', kind: 'execute', status: 'pending' }, + options: [ + { optionId: 'opt-allow', name: 'Allow once', kind: 'allow_once' }, + { optionId: 'opt-reject', name: 'Reject once', kind: 'reject_once' }, + ], + }, + }) + }) + chunk(`permission:${JSON.stringify(outcome)}`) + } + switch (behavior.prompt ?? 'respond') { + case 'respond': + respond(id, { stopReason: 'end_turn' }) + return + case 'error': + respondError(id, 'model exploded') + return + case 'hang-until-cancel': + parkedPromptId = id + return + } +} + +function handleFrame(frame: Record): void { + const id = frame.id as number | string | undefined + const method = frame.method as string | undefined + const params = (frame.params ?? {}) as Record + // A response to one of OUR outbound requests (the permission probe). + if (method === undefined && id !== undefined && typeof id === 'number' && pendingPermission.has(id)) { + const resolve = pendingPermission.get(id) as (outcome: unknown) => void + pendingPermission.delete(id) + resolve((frame.result as { outcome?: unknown } | undefined)?.outcome ?? null) + return + } + switch (method) { + case 'initialize': + respond(id as number | string, { protocolVersion: 1, agentCapabilities: { loadSession: false } }) + return + case 'session/new': { + const extra = params.additionalDirectories as unknown[] | undefined + if (behavior.rejectNewSession === true || (behavior.rejectExtraDirs === true && extra !== undefined && extra.length > 0)) { + respondError(id as number | string, 'unsupported workspace scope') + return + } + sessionId = randomUUID() + sessionCwd = typeof params.cwd === 'string' ? params.cwd : process.cwd() + respond(id as number | string, { sessionId }) + return + } + case 'session/prompt': + void handlePrompt(id as number | string) + return + case 'session/cancel': + if (parkedPromptId !== null) { + const parked = parkedPromptId + parkedPromptId = null + respond(parked, { stopReason: 'cancelled' }) + } + return + default: + // Unknown method: a notification is ignored; a request gets an error so + // the SDK never waits forever on a frame this fake doesn't model. + if (id !== undefined) respondError(id, `unhandled method ${String(method)}`) + } +} + +function flushLogsAndExit(): void { + for (const log of behavior.logs ?? []) { + const target = join(sessionsRoot, log.file) + mkdirSync(dirname(target), { recursive: true }) + writeFileSync(target, log.lines.map(l => JSON.stringify(instantiate(l))).join('\n') + '\n') + } + if (behavior.strayRootFile === true) writeFileSync(join(sessionsRoot, 'stray.txt'), 'not a bucket\n') + if (behavior.strayBucketFile === true) { + mkdirSync(join(sessionsRoot, 'bucket-noise'), { recursive: true }) + writeFileSync(join(sessionsRoot, 'bucket-noise', 'notes.txt'), 'not a session log\n') + } + if (behavior.deleteSessionsRoot === true) rmSync(sessionsRoot, { recursive: true, force: true }) + process.exit(0) +} + +const rl = createInterface({ input: process.stdin }) +rl.on('line', (line) => { + if (line.trim().length === 0) return + handleFrame(JSON.parse(line) as Record) +}) +rl.on('close', () => { flushLogsAndExit() }) diff --git a/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-child/behavior.json b/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-child/behavior.json new file mode 100644 index 0000000000..d44a3a9698 --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-child/behavior.json @@ -0,0 +1,13 @@ +{ + "prompt": "respond", + "logs": [ + { "file": "b/parent.jsonl", "lines": [ + { "type": "session", "id": "{{SID}}", "createdAt": 700, "cwd": "{{CWD}}" }, + { "type": "request/header", "seq": 0, "time": 3, "data": { "header": { "config": { "model": "fake" }, "system": "SYS PROMPT", "tools": [{ "name": "t1", "description": "D1", "parameters": { "type": "object" } }] }, "reason": "initial" } } + ]}, + { "file": "b/child.jsonl", "lines": [ + { "type": "session", "id": "abababab-cdcd-4efe-8ada-badabadabada", "createdAt": 800, "cwd": "{{CWD}}", "parentSession": "{{SID}}" }, + { "type": "request/header", "seq": 0, "time": 2, "data": { "header": { "config": { "model": "fake" }, "system": "SYS PROMPT", "tools": [{ "name": "t1", "description": "D1", "parameters": { "type": "object" } }] }, "reason": "initial" } } + ]} + ] +} diff --git a/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-child/input.json b/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-child/input.json new file mode 100644 index 0000000000..6d3e49b830 --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-child/input.json @@ -0,0 +1 @@ +{ "steps": [{ "op": "initialize" }, { "op": "newSession" }, { "op": "prompt", "text": "rec child" }] } diff --git a/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-child/session.1.jsonl b/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-child/session.1.jsonl new file mode 100644 index 0000000000..1caf2610b3 --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-child/session.1.jsonl @@ -0,0 +1,2 @@ +{"type":"session","id":"abababab-cdcd-4efe-8ada-badabadabada","createdAt":800,"cwd":"/var/folders/2g/b32ct0qn1d728l_v6tdkjytr0000gn/T/acp-snap-cwd-KBQJbW","parentSession":"f6fa7fcf-dd9c-4b39-8815-b25ddcebfd88"} +{"type":"request/header","seq":0,"time":2,"data":{"header":{"config":{"model":"fake"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}} diff --git a/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-child/session.jsonl b/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-child/session.jsonl new file mode 100644 index 0000000000..a2beac360d --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-child/session.jsonl @@ -0,0 +1,2 @@ +{"type":"session","id":"f6fa7fcf-dd9c-4b39-8815-b25ddcebfd88","createdAt":700,"cwd":"/var/folders/2g/b32ct0qn1d728l_v6tdkjytr0000gn/T/acp-snap-cwd-KBQJbW"} +{"type":"request/header","seq":0,"time":3,"data":{"header":{"config":{"model":"fake"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}} diff --git a/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-child/stdout.golden.jsonl b/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-child/stdout.golden.jsonl new file mode 100644 index 0000000000..f173b45b77 --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-child/stdout.golden.jsonl @@ -0,0 +1,4 @@ +{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentCapabilities":{"loadSession":false}}} +{"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}"}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"thinking about it"}}}} +{"jsonrpc":"2.0","id":3,"result":{"stopReason":"end_turn"}} diff --git a/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-pin/behavior.json b/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-pin/behavior.json new file mode 100644 index 0000000000..a24e30d80a --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-pin/behavior.json @@ -0,0 +1,10 @@ +{ + "prompt": "respond", + "logs": [{ + "file": "b/main.jsonl", + "lines": [ + { "type": "session", "id": "{{SID}}", "createdAt": 600, "cwd": "{{CWD}}" }, + { "type": "request/header", "seq": 0, "time": 4, "data": { "header": { "config": { "model": "fake" }, "system": "SYS PROMPT", "tools": [{ "name": "t1", "description": "D1", "parameters": { "type": "object" } }] }, "reason": "initial" } } + ] + }] +} diff --git a/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-pin/input.json b/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-pin/input.json new file mode 100644 index 0000000000..9573d20b27 --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-pin/input.json @@ -0,0 +1 @@ +{ "steps": [{ "op": "initialize" }, { "op": "newSession" }, { "op": "prompt", "text": "rec pin" }] } diff --git a/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-pin/session.jsonl b/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-pin/session.jsonl new file mode 100644 index 0000000000..109a192083 --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-pin/session.jsonl @@ -0,0 +1,2 @@ +{"type":"session","id":"ccdc749f-56f3-4267-9750-598b5c60b7b2","createdAt":600,"cwd":"/var/folders/2g/b32ct0qn1d728l_v6tdkjytr0000gn/T/acp-snap-cwd-nOQ4Gy"} +{"type":"request/header","seq":0,"time":4,"data":{"header":{"config":{"model":"fake"},"system":"SYS PROMPT","tools":[{"name":"t1","description":"D1","parameters":{"type":"object"}}]},"reason":"initial"}} diff --git a/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-pin/stdout.golden.jsonl b/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-pin/stdout.golden.jsonl new file mode 100644 index 0000000000..f173b45b77 --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-pin/stdout.golden.jsonl @@ -0,0 +1,4 @@ +{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentCapabilities":{"loadSession":false}}} +{"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}"}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"thinking about it"}}}} +{"jsonrpc":"2.0","id":3,"result":{"stopReason":"end_turn"}} diff --git a/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-skip/behavior.json b/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-skip/behavior.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-skip/behavior.json @@ -0,0 +1 @@ +{} diff --git a/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-skip/input.json b/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-skip/input.json new file mode 100644 index 0000000000..d1e94c22eb --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-skip/input.json @@ -0,0 +1 @@ +{ "steps": [{ "op": "initialize" }] } diff --git a/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-skip/replay.override.json b/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-skip/replay.override.json new file mode 100644 index 0000000000..8ed00c0651 --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-skip/replay.override.json @@ -0,0 +1 @@ +[{ "kind": "hang" }] diff --git a/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-skip/session.jsonl b/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-skip/session.jsonl new file mode 100644 index 0000000000..104f2a0df2 --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-skip/session.jsonl @@ -0,0 +1 @@ +{"type":"session","id":"{{sessionId}}","createdAt":0,"cwd":"{{cwd}}"} diff --git a/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-skip/stdout.golden.jsonl b/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-skip/stdout.golden.jsonl new file mode 100644 index 0000000000..d6a1d2b232 --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-skip/stdout.golden.jsonl @@ -0,0 +1 @@ +{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentCapabilities":{"loadSession":false}}} diff --git a/packages/support/acp-snapshot/tests/fixtures/suite/authored-error/behavior.json b/packages/support/acp-snapshot/tests/fixtures/suite/authored-error/behavior.json new file mode 100644 index 0000000000..808d9672b9 --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/suite/authored-error/behavior.json @@ -0,0 +1,10 @@ +{ + "prompt": "error", + "logs": [{ + "file": "b/main.jsonl", + "lines": [ + { "type": "session", "id": "{{SID}}", "createdAt": 500, "cwd": "{{CWD}}" }, + { "type": "turn/end", "seq": 1, "time": 9, "data": { "error": "model exploded" } } + ] + }] +} diff --git a/packages/support/acp-snapshot/tests/fixtures/suite/authored-error/input.json b/packages/support/acp-snapshot/tests/fixtures/suite/authored-error/input.json new file mode 100644 index 0000000000..c281971465 --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/suite/authored-error/input.json @@ -0,0 +1 @@ +{ "steps": [{ "op": "initialize" }, { "op": "newSession" }, { "op": "promptExpectError", "text": "boom" }] } diff --git a/packages/support/acp-snapshot/tests/fixtures/suite/authored-error/replay.override.json b/packages/support/acp-snapshot/tests/fixtures/suite/authored-error/replay.override.json new file mode 100644 index 0000000000..e868115f35 --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/suite/authored-error/replay.override.json @@ -0,0 +1 @@ +[{ "kind": "throw", "chunks": [], "message": "model exploded", "code": "PROVIDER" }] diff --git a/packages/support/acp-snapshot/tests/fixtures/suite/authored-error/session.jsonl b/packages/support/acp-snapshot/tests/fixtures/suite/authored-error/session.jsonl new file mode 100644 index 0000000000..36991a214e --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/suite/authored-error/session.jsonl @@ -0,0 +1,2 @@ +{"type":"session","id":"44444444-3333-4222-8111-000000000000","createdAt":17,"cwd":"/rec/authored-cwd"} +{"type":"turn/end","seq":1,"time":17,"data":{"error":"model exploded"}} diff --git a/packages/support/acp-snapshot/tests/fixtures/suite/authored-error/stdout.golden.jsonl b/packages/support/acp-snapshot/tests/fixtures/suite/authored-error/stdout.golden.jsonl new file mode 100644 index 0000000000..2a1d69bd93 --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/suite/authored-error/stdout.golden.jsonl @@ -0,0 +1,4 @@ +{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentCapabilities":{"loadSession":false}}} +{"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}"}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"thinking about it"}}}} +{"jsonrpc":"2.0","id":3,"error":{"code":-32603,"message":"model exploded"}} diff --git a/packages/support/acp-snapshot/tests/fixtures/suite/blocked-log/behavior.json b/packages/support/acp-snapshot/tests/fixtures/suite/blocked-log/behavior.json new file mode 100644 index 0000000000..e0a438297d --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/suite/blocked-log/behavior.json @@ -0,0 +1,10 @@ +{ + "prompt": "error", + "logs": [{ + "file": "b/main.jsonl", + "lines": [ + { "type": "session", "id": "{{SID}}", "createdAt": 400, "cwd": "{{CWD}}" }, + { "type": "hook/result", "seq": 1, "time": 8, "data": { "decision": "block", "durationMs": 37 } } + ] + }] +} diff --git a/packages/support/acp-snapshot/tests/fixtures/suite/blocked-log/input.json b/packages/support/acp-snapshot/tests/fixtures/suite/blocked-log/input.json new file mode 100644 index 0000000000..0a711dca3c --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/suite/blocked-log/input.json @@ -0,0 +1 @@ +{ "steps": [{ "op": "initialize" }, { "op": "newSession" }, { "op": "promptExpectError", "text": "blocked" }] } diff --git a/packages/support/acp-snapshot/tests/fixtures/suite/blocked-log/session.jsonl b/packages/support/acp-snapshot/tests/fixtures/suite/blocked-log/session.jsonl new file mode 100644 index 0000000000..6d8474812d --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/suite/blocked-log/session.jsonl @@ -0,0 +1,2 @@ +{"type":"session","id":"99999999-8888-4777-8666-555555555555","createdAt":13,"cwd":"/rec/blocked-cwd"} +{"type":"hook/result","seq":1,"time":13,"data":{"decision":"block","durationMs":99}} diff --git a/packages/support/acp-snapshot/tests/fixtures/suite/blocked-log/stdout.golden.jsonl b/packages/support/acp-snapshot/tests/fixtures/suite/blocked-log/stdout.golden.jsonl new file mode 100644 index 0000000000..2a1d69bd93 --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/suite/blocked-log/stdout.golden.jsonl @@ -0,0 +1,4 @@ +{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentCapabilities":{"loadSession":false}}} +{"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}"}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"thinking about it"}}}} +{"jsonrpc":"2.0","id":3,"error":{"code":-32603,"message":"model exploded"}} diff --git a/packages/support/acp-snapshot/tests/fixtures/suite/no-model/behavior.json b/packages/support/acp-snapshot/tests/fixtures/suite/no-model/behavior.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/suite/no-model/behavior.json @@ -0,0 +1 @@ +{} diff --git a/packages/support/acp-snapshot/tests/fixtures/suite/no-model/input.json b/packages/support/acp-snapshot/tests/fixtures/suite/no-model/input.json new file mode 100644 index 0000000000..d1e94c22eb --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/suite/no-model/input.json @@ -0,0 +1 @@ +{ "steps": [{ "op": "initialize" }] } diff --git a/packages/support/acp-snapshot/tests/fixtures/suite/no-model/session.jsonl b/packages/support/acp-snapshot/tests/fixtures/suite/no-model/session.jsonl new file mode 100644 index 0000000000..104f2a0df2 --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/suite/no-model/session.jsonl @@ -0,0 +1 @@ +{"type":"session","id":"{{sessionId}}","createdAt":0,"cwd":"{{cwd}}"} diff --git a/packages/support/acp-snapshot/tests/fixtures/suite/no-model/stdout.golden.jsonl b/packages/support/acp-snapshot/tests/fixtures/suite/no-model/stdout.golden.jsonl new file mode 100644 index 0000000000..d6a1d2b232 --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/suite/no-model/stdout.golden.jsonl @@ -0,0 +1 @@ +{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentCapabilities":{"loadSession":false}}} diff --git a/packages/support/acp-snapshot/tests/fixtures/suite/pin-turn/behavior.json b/packages/support/acp-snapshot/tests/fixtures/suite/pin-turn/behavior.json new file mode 100644 index 0000000000..422e0a17e6 --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/suite/pin-turn/behavior.json @@ -0,0 +1,11 @@ +{ + "prompt": "respond", + "logs": [{ + "file": "b/main.jsonl", + "lines": [ + { "type": "session", "id": "{{SID}}", "createdAt": 100, "cwd": "{{CWD}}" }, + { "type": "request/header", "seq": 0, "time": 100, "data": { "header": { "config": { "model": "fake" }, "system": "SYS PROMPT", "tools": [{ "name": "t1", "description": "D1", "parameters": { "type": "object" } }] }, "reason": "initial" } }, + { "type": "turn/start", "seq": 1, "time": 100, "data": { "turn": 1 } } + ] + }] +} diff --git a/packages/support/acp-snapshot/tests/fixtures/suite/pin-turn/input.json b/packages/support/acp-snapshot/tests/fixtures/suite/pin-turn/input.json new file mode 100644 index 0000000000..b9e2d9bbc5 --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/suite/pin-turn/input.json @@ -0,0 +1 @@ +{ "steps": [{ "op": "initialize" }, { "op": "newSession" }, { "op": "prompt", "text": "pin" }] } diff --git a/packages/support/acp-snapshot/tests/fixtures/suite/pin-turn/session.jsonl b/packages/support/acp-snapshot/tests/fixtures/suite/pin-turn/session.jsonl new file mode 100644 index 0000000000..87bf09c839 --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/suite/pin-turn/session.jsonl @@ -0,0 +1,3 @@ +{"type":"session","id":"12121212-3434-4545-8686-787878787878","createdAt":7,"cwd":"/rec/pin-cwd"} +{"type":"request/header","seq":0,"time":7,"data":{"header":{"config":{"model":"fake"},"system":"SYS PROMPT","tools":[{"name":"t1","description":"D1","parameters":{"type":"object"}}]},"reason":"initial"}} +{"type":"turn/start","seq":1,"time":7,"data":{"turn":1}} diff --git a/packages/support/acp-snapshot/tests/fixtures/suite/pin-turn/stdout.golden.jsonl b/packages/support/acp-snapshot/tests/fixtures/suite/pin-turn/stdout.golden.jsonl new file mode 100644 index 0000000000..f173b45b77 --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/suite/pin-turn/stdout.golden.jsonl @@ -0,0 +1,4 @@ +{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentCapabilities":{"loadSession":false}}} +{"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}"}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"thinking about it"}}}} +{"jsonrpc":"2.0","id":3,"result":{"stopReason":"end_turn"}} diff --git a/packages/support/acp-snapshot/tests/fixtures/suite/plain-turn/behavior.json b/packages/support/acp-snapshot/tests/fixtures/suite/plain-turn/behavior.json new file mode 100644 index 0000000000..d5cbbf9d28 --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/suite/plain-turn/behavior.json @@ -0,0 +1,15 @@ +{ + "prompt": "respond", + "echoWorkspace": true, + "logs": [ + { "file": "b/parent.jsonl", "lines": [ + { "type": "session", "id": "{{SID}}", "createdAt": 200, "cwd": "{{CWD}}" }, + { "type": "request/header", "seq": 0, "time": 5, "data": { "header": { "config": { "model": "fake" }, "system": "SYS PROMPT", "tools": [{ "name": "t1", "description": "D1", "parameters": { "type": "object" } }] }, "reason": "initial" } }, + { "type": "assistant/chunk", "seq": 1, "time": 5, "data": { "turn": 1, "step": 1, "chunk": { "type": "text-delta", "index": 0, "text": "hi" } } } + ]}, + { "file": "b/child.jsonl", "lines": [ + { "type": "session", "id": "eeeeeeee-1111-4222-8333-444444444444", "createdAt": 300, "cwd": "{{CWD}}", "parentSession": "{{SID}}" }, + { "type": "request/header", "seq": 0, "time": 6, "data": { "header": { "config": { "model": "fake" }, "system": "SYS PROMPT", "tools": [{ "name": "t1", "description": "D1", "parameters": { "type": "object" } }] }, "reason": "initial" } } + ]} + ] +} diff --git a/packages/support/acp-snapshot/tests/fixtures/suite/plain-turn/input.json b/packages/support/acp-snapshot/tests/fixtures/suite/plain-turn/input.json new file mode 100644 index 0000000000..60b9e363b5 --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/suite/plain-turn/input.json @@ -0,0 +1 @@ +{ "steps": [{ "op": "initialize" }, { "op": "newSession" }, { "op": "prompt", "text": "plain" }] } diff --git a/packages/support/acp-snapshot/tests/fixtures/suite/plain-turn/session.1.jsonl b/packages/support/acp-snapshot/tests/fixtures/suite/plain-turn/session.1.jsonl new file mode 100644 index 0000000000..a844f891fc --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/suite/plain-turn/session.1.jsonl @@ -0,0 +1,2 @@ +{"type":"session","id":"eeeeeeee-1111-4222-8333-444444444444","createdAt":12,"cwd":"/rec/plain-cwd","parentSession":"56565656-7878-4989-8a9a-9b9b9b9b9b9b"} +{"type":"request/header","seq":0,"time":12,"data":{"header":{"config":{"model":"fake"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}} diff --git a/packages/support/acp-snapshot/tests/fixtures/suite/plain-turn/session.jsonl b/packages/support/acp-snapshot/tests/fixtures/suite/plain-turn/session.jsonl new file mode 100644 index 0000000000..744998f959 --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/suite/plain-turn/session.jsonl @@ -0,0 +1,3 @@ +{"type":"session","id":"56565656-7878-4989-8a9a-9b9b9b9b9b9b","createdAt":11,"cwd":"/rec/plain-cwd"} +{"type":"request/header","seq":0,"time":11,"data":{"header":{"config":{"model":"fake"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}} +{"type":"assistant/chunk","seq":1,"time":11,"data":{"turn":1,"step":1,"chunk":{"type":"text-delta","index":0,"text":"hi"}}} diff --git a/packages/support/acp-snapshot/tests/fixtures/suite/plain-turn/stdout.golden.jsonl b/packages/support/acp-snapshot/tests/fixtures/suite/plain-turn/stdout.golden.jsonl new file mode 100644 index 0000000000..d0242ae39f --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/suite/plain-turn/stdout.golden.jsonl @@ -0,0 +1,5 @@ +{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentCapabilities":{"loadSession":false}}} +{"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}"}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"thinking about it"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"workspace:seed.txt"}}}} +{"jsonrpc":"2.0","id":3,"result":{"stopReason":"end_turn"}} diff --git a/packages/support/acp-snapshot/tests/fixtures/suite/plain-turn/workspace/seed.txt b/packages/support/acp-snapshot/tests/fixtures/suite/plain-turn/workspace/seed.txt new file mode 100644 index 0000000000..c19e887d68 --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/suite/plain-turn/workspace/seed.txt @@ -0,0 +1 @@ +seeded diff --git a/packages/support/acp-snapshot/tests/harness.spec.ts b/packages/support/acp-snapshot/tests/harness.spec.ts new file mode 100644 index 0000000000..3884e36095 --- /dev/null +++ b/packages/support/acp-snapshot/tests/harness.spec.ts @@ -0,0 +1,233 @@ +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { delimiter, join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { afterAll, describe, expect, it } from 'vitest' +import { runScenario, type AgentUnderTest, type InputStep } from '../src/harness.ts' + +/** + * Unit tests for the subprocess harness, driven through the REAL spawn path + * (tsx loader, temp cwd, env plumbing) against the scripted fake ACP bin in + * ./fixtures/fake-acp-agent.ts. Each case writes a `behavior.json` next to a + * throwaway fixture path; the fake bin echoes observable facts (env, seeded + * workspace, permission outcomes) into `agent_message_chunk` text, so the + * assertions read plain `rawStdout`. + */ + +const AGENT: AgentUnderTest = { + binScript: fileURLToPath(new URL('./fixtures/fake-acp-agent.ts', import.meta.url)), + // The fake bin ignores its config argv; any real path documents the shape. + configPath: fileURLToPath(new URL('./fixtures/fake-acp-agent.ts', import.meta.url)), + tsconfigPath: fileURLToPath(new URL('../../../../tsconfig.json', import.meta.url)), +} + +/** Temp scenario dirs to drop after the suite. */ +const tempDirs: string[] = [] +afterAll(async () => { + for (const dir of tempDirs) await rm(dir, { recursive: true, force: true }) +}) + +/** Write a behavior.json into a fresh temp dir; return the sibling fixture path the harness points the bin at. */ +async function scenario(behavior: object): Promise<{ dir: string; fixtureFile: string }> { + const dir = await mkdtemp(join(tmpdir(), 'acp-snap-spec-')) + tempDirs.push(dir) + await writeFile(join(dir, 'behavior.json'), JSON.stringify(behavior)) + return { dir, fixtureFile: join(dir, 'session.jsonl') } +} + +const boot: InputStep[] = [{ op: 'initialize' }, { op: 'newSession' }] + +describe('runScenario', () => { + it('drives a full turn: initialize (terminal caps), session, prompt, permission stub, harvest', { timeout: 20_000 }, async () => { + const { fixtureFile } = await scenario({ + permissionProbe: true, + logs: [{ + file: 'bucket/main.jsonl', + lines: [ + { type: 'session', id: '{{SID}}', createdAt: 42, cwd: '{{CWD}}' }, + { type: 'turn/start', seq: 1, time: 9, data: { turn: 1 } }, + ], + }], + }) + const result = await runScenario( + { steps: [{ op: 'initialize', terminalOutput: true }, { op: 'newSession' }, { op: 'prompt', text: 'go' }] }, + { agent: AGENT, mode: 'replay', fixtureFile }, + ) + expect(result.sessionId).toBeDefined() + // The harness's client answers a permission request with `cancelled`; the + // fake bin echoes the outcome it received back as a chunk. + expect(result.rawStdout).toContain('permission:{\\"outcome\\":\\"cancelled\\"}') + expect(result.sessionLogs).toHaveLength(1) + expect(result.sessionLogs[0]?.id).toBe(result.sessionId) + expect(result.sessionLogs[0]?.createdAt).toBe(42) + expect(result.sessionLogs[0]?.content).toContain('turn/start') + // The harvested log embeds the run's REAL temp cwd (template-substituted). + expect(result.sessionLogs[0]?.content).toContain(result.cwd) + }) + + it('forwards override/child fixture paths into the child env and captures stderr', { timeout: 20_000 }, async () => { + const { dir, fixtureFile } = await scenario({ echoEnv: true, stderrNote: 'fake bin booted' }) + const childFiles = [join(dir, 'session.1.jsonl'), join(dir, 'session.2.jsonl')] + const result = await runScenario( + { steps: [...boot, { op: 'prompt', text: 'env?' }] }, + { + agent: AGENT, + mode: 'replay', + fixtureFile, + overrideFile: join(dir, 'replay.override.json'), + childFiles, + // A workspaceDir that does not exist is skipped, not an error. + workspaceDir: join(dir, 'no-such-workspace'), + }, + ) + expect(result.stderr).toContain('fake bin booted') + expect(result.rawStdout).toContain('replay.override.json') + // Child paths ride one env var, joined with the platform delimiter. + expect(result.rawStdout).toContain(JSON.stringify(childFiles.join(delimiter)).slice(1, -1)) + }) + + it('seeds the workspace dir into the temp cwd before the run', { timeout: 20_000 }, async () => { + const { dir, fixtureFile } = await scenario({ echoWorkspace: true }) + const workspaceDir = join(dir, 'workspace') + await writeFile(join(dir, 'behavior.json'), JSON.stringify({ echoWorkspace: true })) + const { mkdir } = await import('node:fs/promises') + await mkdir(workspaceDir, { recursive: true }) + await writeFile(join(workspaceDir, 'seeded.txt'), 'hello') + const result = await runScenario( + { steps: [...boot, { op: 'prompt', text: 'ls' }] }, + { agent: AGENT, mode: 'replay', fixtureFile, workspaceDir }, + ) + expect(result.rawStdout).toContain('workspace:seeded.txt') + }) + + it('promptAndCancel waits for the streamed chunk, cancels, and settles the prompt', { timeout: 20_000 }, async () => { + const { fixtureFile } = await scenario({ prompt: 'hang-until-cancel' }) + const result = await runScenario( + { steps: [...boot, { op: 'promptAndCancel', text: 'hang' }] }, + { agent: AGENT, mode: 'replay', fixtureFile }, + ) + expect(result.rawStdout).toContain('"stopReason":"cancelled"') + // The streamed chunk deterministically precedes the cancelled response. + expect(result.rawStdout.indexOf('thinking about it')).toBeLessThan(result.rawStdout.indexOf('cancelled')) + }) + + it('promptExpectError swallows a model-error response as the expected outcome', { timeout: 20_000 }, async () => { + const { fixtureFile } = await scenario({ prompt: 'error' }) + const result = await runScenario( + { steps: [...boot, { op: 'promptExpectError', text: 'boom' }] }, + { agent: AGENT, mode: 'replay', fixtureFile }, + ) + expect(result.rawStdout).toContain('model exploded') + }) + + it('promptExpectError throws when the prompt unexpectedly succeeds (and teardown kills the live child)', { timeout: 20_000 }, async () => { + const { fixtureFile } = await scenario({ prompt: 'respond' }) + await expect(runScenario( + { steps: [...boot, { op: 'promptExpectError', text: 'fine' }] }, + { agent: AGENT, mode: 'replay', fixtureFile }, + )).rejects.toThrow(/expected the prompt to fail/) + }) + + it('newSessionExpectError swallows the rejection, with and without extra dirs', { timeout: 20_000 }, async () => { + const { fixtureFile } = await scenario({ rejectExtraDirs: true }) + const result = await runScenario( + { steps: [{ op: 'initialize' }, { op: 'newSessionExpectError', additionalDirectories: ['/elsewhere'] }] }, + { agent: AGENT, mode: 'replay', fixtureFile }, + ) + // No session was created, so no id and no logs. + expect(result.sessionId).toBeUndefined() + expect(result.sessionLogs).toHaveLength(0) + + const rejectAll = await scenario({ rejectNewSession: true }) + const second = await runScenario( + { steps: [{ op: 'initialize' }, { op: 'newSessionExpectError' }] }, + { agent: AGENT, mode: 'replay', fixtureFile: rejectAll.fixtureFile }, + ) + expect(second.rawStdout).toContain('unsupported workspace scope') + }) + + it('newSessionExpectError throws when session/new unexpectedly succeeds', { timeout: 20_000 }, async () => { + const { fixtureFile } = await scenario({}) + await expect(runScenario( + { steps: [{ op: 'initialize' }, { op: 'newSessionExpectError' }] }, + { agent: AGENT, mode: 'replay', fixtureFile }, + )).rejects.toThrow(/expected session\/new to be rejected/) + }) + + it('a plain cancel step is forwarded (and ignored by an idle agent)', { timeout: 20_000 }, async () => { + const { fixtureFile } = await scenario({}) + const result = await runScenario( + { steps: [...boot, { op: 'cancel' }] }, + { agent: AGENT, mode: 'replay', fixtureFile }, + ) + expect(result.sessionId).toBeDefined() + }) + + it.each([ + [{ op: 'prompt', text: 'x' }, /prompt before newSession/], + [{ op: 'promptExpectError', text: 'x' }, /promptExpectError before newSession/], + [{ op: 'promptAndCancel', text: 'x' }, /promptAndCancel before newSession/], + [{ op: 'cancel' }, /cancel before newSession/], + ] as [InputStep, RegExp][])('rejects %j before newSession', { timeout: 20_000 }, async (step, message) => { + const { fixtureFile } = await scenario({}) + await expect(runScenario( + { steps: [{ op: 'initialize' }, step] }, + { agent: AGENT, mode: 'replay', fixtureFile }, + )).rejects.toThrow(message) + }) + + it('rejects an unknown input op', { timeout: 20_000 }, async () => { + const { fixtureFile } = await scenario({}) + const bogus = { op: 'reticulate' } as unknown as InputStep + await expect(runScenario( + { steps: [bogus] }, + { agent: AGENT, mode: 'replay', fixtureFile }, + )).rejects.toThrow(/unknown input op/) + }) + + it('harvests all logs primary-first, children by createdAt then id, skipping filesystem noise', { timeout: 20_000 }, async () => { + const { fixtureFile } = await scenario({ + strayRootFile: true, + strayBucketFile: true, + logs: [ + // File names chosen so readdir feeds the sort children-first AND + // parent-in-the-middle: the comparator then sees a parent on both + // sides of a pair, plus the same-createdAt (localeCompare) tiebreak. + { file: 'b1/aa-child-c.jsonl', lines: [{ type: 'session', id: 'cccccccc-0000-4000-8000-000000000000', createdAt: 500, parentSession: '{{SID}}' }] }, + { file: 'b1/bb-parent.jsonl', lines: [{ type: 'session', id: '{{SID}}', createdAt: 900 }] }, + { file: 'b1/cc-child-a.jsonl', lines: [{ type: 'session', id: 'aaaaaaaa-0000-4000-8000-000000000000', createdAt: 500, parentSession: '{{SID}}' }] }, + // Missing id/createdAt fall back to ''/0; earliest child by createdAt. + { file: 'b2/orphan-fields.jsonl', lines: [{ type: 'session', parentSession: '{{SID}}' }] }, + ], + }) + const result = await runScenario( + { steps: [...boot, { op: 'prompt', text: 'go' }] }, + { agent: AGENT, mode: 'replay', fixtureFile }, + ) + expect(result.sessionLogs.map(l => [l.id, l.createdAt])).toEqual([ + [result.sessionId, 900], + ['', 0], + ['aaaaaaaa-0000-4000-8000-000000000000', 500], + ['cccccccc-0000-4000-8000-000000000000', 500], + ]) + expect(result.sessionLogs[1]?.parentSession).toBe(result.sessionId) + }) + + it('treats an empty log file as a header-less primary with default fields', { timeout: 20_000 }, async () => { + const { fixtureFile } = await scenario({ logs: [{ file: 'b/empty.jsonl', lines: [] }] }) + const result = await runScenario( + { steps: boot }, + { agent: AGENT, mode: 'replay', fixtureFile }, + ) + expect(result.sessionLogs.map(l => [l.id, l.createdAt, l.parentSession])).toEqual([['', 0, undefined]]) + }) + + it('yields no logs when the sessions root vanished', { timeout: 20_000 }, async () => { + const { fixtureFile } = await scenario({ deleteSessionsRoot: true }) + const result = await runScenario( + { steps: boot }, + { agent: AGENT, mode: 'replay', fixtureFile }, + ) + expect(result.sessionLogs).toHaveLength(0) + }) +}) diff --git a/packages/support/acp-snapshot/tests/normalize.spec.ts b/packages/support/acp-snapshot/tests/normalize.spec.ts index 56c15306a3..8ebd1412b9 100644 --- a/packages/support/acp-snapshot/tests/normalize.spec.ts +++ b/packages/support/acp-snapshot/tests/normalize.spec.ts @@ -107,6 +107,17 @@ describe('normalizeSessionLog', () => { const out = normalizeSessionLog(`${header({})}\n${ev}\n`, ctx) expect(out).toContain('"durationMs":88') }) + + it('tolerates records missing the volatile fields it would zero', () => { + const bareHeader = JSON.stringify({ type: 'session', id: 's' }) + const timeless = JSON.stringify({ type: 'note', seq: 1 }) + const bareHook = JSON.stringify({ type: 'hook/result', seq: 2, time: 5, data: { decision: 'allow' } }) + const nullDataHook = JSON.stringify({ type: 'hook/result', seq: 3, time: 6, data: null }) + const out = normalizeSessionLog(`${bareHeader}\n${timeless}\n${bareHook}\n${nullDataHook}\n`, ctx) + expect(out).toContain('"type":"note","seq":1') + expect(out).toContain('"decision":"allow"') + expect(out).not.toContain('durationMs') + }) }) describe('scrubRequestHeaders', () => { @@ -135,6 +146,40 @@ describe('scrubRequestHeaders', () => { expect(out).not.toContain('{{tools}}') }) + it('scrubs a header carrying only one of system/tools, leaving the other absent', () => { + const systemOnly = scrubRequestHeaders(`${headerLine}\n${headerEvent({ system: 'secret prompt' })}\n`) + expect(systemOnly).toContain('"system":"{{system}}"') + expect(systemOnly).not.toContain('{{tools}}') + const toolsOnly = scrubRequestHeaders(`${headerLine}\n${headerEvent({ tools: [{ name: 't' }] })}\n`) + expect(toolsOnly).toContain('"tools":"{{tools}}"') + expect(toolsOnly).not.toContain('{{system}}') + }) + + it('leaves a delta with no scrubbable payload byte-identical (config-only, or non-array shapes)', () => { + const configOnly = JSON.stringify({ type: 'request/header-delta', seq: 8, time: 9, data: { config: { model: 'm2' } } }) + const oddShapes = JSON.stringify({ type: 'request/header-delta', seq: 9, time: 9, data: { system: { insert: 'not-an-array' }, tools: null } }) + const headerless = JSON.stringify({ type: 'request/header', seq: 10, time: 9, data: { reason: 'initial' } }) + const nullData = JSON.stringify({ type: 'request/header', seq: 11, time: 9, data: null }) + const raw = `${headerLine}\n${configOnly}\n${oddShapes}\n${headerless}\n${nullData}\n` + expect(scrubRequestHeaders(raw)).toBe(raw) + }) + + it('scrubs a one-sided tools delta and passes non-object schema entries through', () => { + const addedOnly = JSON.stringify({ + type: 'request/header-delta', seq: 8, time: 9, + data: { tools: { added: [null, 'weird', { name: 'x', description: 'D' }] } }, + }) + const out = scrubRequestHeaders(`${headerLine}\n${addedOnly}\n`) + // Non-object entries survive untouched; the object entry keeps only name. + expect(out).toContain('"added":[null,"weird",{"name":"x","description":"{{tools}}"}]') + const changedOnly = JSON.stringify({ + type: 'request/header-delta', seq: 8, time: 9, + data: { tools: { changed: [{ name: 'y', parameters: {} }] } }, + }) + expect(scrubRequestHeaders(`${headerLine}\n${changedOnly}\n`)) + .toContain('"changed":[{"name":"y","parameters":"{{tools}}"}]') + }) + it('scrubs a header-delta system payload but keeps its line positions and arity', () => { const delta = JSON.stringify({ type: 'request/header-delta', seq: 8, time: 9, diff --git a/packages/support/acp-snapshot/tests/suite.spec.ts b/packages/support/acp-snapshot/tests/suite.spec.ts new file mode 100644 index 0000000000..0bc15aeec2 --- /dev/null +++ b/packages/support/acp-snapshot/tests/suite.spec.ts @@ -0,0 +1,145 @@ +import { cpSync, mkdtempSync } from 'node:fs' +import { rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { afterAll, describe, expect, it } from 'vitest' +import { defineAcpSnapshotSuite, type Scenario } from '../src/index.ts' +import { childFixturePaths, fixtureContext, headerDeltaCount, normalizedHeaders } from '../src/suite.ts' + +/** + * Unit tests for the suite factory, by running it: two synthetic suites over + * the scripted fake ACP bin (./fixtures/fake-acp-agent.ts) register REAL + * describe/it trees at collection time, so every factory path — golden and log + * compares, the per-suite header pin and its uniformity guard, record-mode + * fixture write-back, skip semantics, and the fixture guard block — executes + * as an ordinary green test. The pure helpers get direct cases below. + * + * The replay suite runs against the committed fixtures in ./fixtures/suite. + * The record suite runs against a TEMP COPY of ./fixtures/record-suite + * (record mode writes session fixtures back into its snapshots dir; a run must + * never touch the committed tree). To re-bootstrap the record tree's goldens + * after changing the fake bin's output, run this spec once with + * `ACP_SNAPSHOT_SPEC_BOOTSTRAP=1` (points the record suite at the committed + * tree so vitest creates/updates the goldens and the write-back lands there), + * then commit the result. + */ + +const AGENT = { + binScript: fileURLToPath(new URL('./fixtures/fake-acp-agent.ts', import.meta.url)), + configPath: fileURLToPath(new URL('./fixtures/fake-acp-agent.ts', import.meta.url)), + tsconfigPath: fileURLToPath(new URL('../../../../tsconfig.json', import.meta.url)), +} + +const REPLAY_DIR = fileURLToPath(new URL('./fixtures/suite', import.meta.url)) +const RECORD_SRC = fileURLToPath(new URL('./fixtures/record-suite', import.meta.url)) + +const REPLAY_SCENARIOS: Scenario[] = [ + { name: 'pin-turn', hasModelTurn: true, recorded: true, pinsHeader: true }, + { name: 'plain-turn', hasModelTurn: true, recorded: true, childSessions: 1 }, + { name: 'no-model', hasModelTurn: false, recorded: false }, + { name: 'blocked-log', hasModelTurn: false, comparesLog: true, recorded: false }, + { name: 'authored-error', hasModelTurn: true, recorded: false }, +] + +const RECORD_SCENARIOS: Scenario[] = [ + { name: 'rec-pin', hasModelTurn: true, recorded: true, pinsHeader: true }, + { name: 'rec-child', hasModelTurn: true, recorded: true, childSessions: 1 }, + // recorded:false in record mode → registered but skipped (never re-recorded). + { name: 'rec-skip', hasModelTurn: true, recorded: false }, +] + +// Record mode mutates its snapshots dir, so run it on a throwaway copy — +// except under the documented bootstrap knob, which regenerates the committed +// fixtures/goldens in place. +const BOOTSTRAP = process.env.ACP_SNAPSHOT_SPEC_BOOTSTRAP === '1' +const recordDir = BOOTSTRAP ? RECORD_SRC : mkdtempSync(join(tmpdir(), 'acp-snap-record-suite-')) +if (!BOOTSTRAP) cpSync(RECORD_SRC, recordDir, { recursive: true }) +afterAll(async () => { + if (!BOOTSTRAP) await rm(recordDir, { recursive: true, force: true }) +}) + +describe('defineAcpSnapshotSuite: replay mode', () => { + defineAcpSnapshotSuite({ agent: AGENT, snapshotsDir: REPLAY_DIR, scenarios: REPLAY_SCENARIOS, mode: 'replay' }) +}) + +// The record suite's tests run in registration order: rec-pin re-records the +// pinned fixture FIRST, so rec-child's uniformity guard reads the fresh pin. +describe('defineAcpSnapshotSuite: record mode', () => { + defineAcpSnapshotSuite({ agent: AGENT, snapshotsDir: recordDir, scenarios: RECORD_SCENARIOS, mode: 'record' }) +}) + +describe('defineAcpSnapshotSuite: registration contract', () => { + it('throws when no scenario pins the request-header content', () => { + expect(() => { + defineAcpSnapshotSuite({ + agent: AGENT, + snapshotsDir: REPLAY_DIR, + scenarios: [{ name: 'pinless', hasModelTurn: true, recorded: true }], + mode: 'replay', + }) + }).toThrow(/no scenario pins/) + }) +}) + +describe('childFixturePaths', () => { + it('yields one sibling path per child, 1-based', () => { + expect(childFixturePaths('/snap/s', 2)).toEqual(['/snap/s/session.1.jsonl', '/snap/s/session.2.jsonl']) + }) + + it('yields nothing for a single-session scenario', () => { + expect(childFixturePaths('/snap/s', 0)).toEqual([]) + }) +}) + +describe('fixtureContext', () => { + it('reads the fixture header id and cwd', () => { + const ctx = fixtureContext('{"type":"session","id":"abc","cwd":"/rec"}\n{"type":"turn/start"}\n') + expect(ctx).toEqual({ sessionIds: ['abc'], cwd: '/rec' }) + }) + + it('yields no session ids for a header without a string id', () => { + expect(fixtureContext('{"type":"session","cwd":"/rec"}\n').sessionIds).toEqual([]) + }) + + it('falls back to an impossible sentinel cwd (never the empty string)', () => { + const ctx = fixtureContext('{"type":"session","id":"abc"}\n') + expect(ctx.cwd).toBe('\0no-cwd\0') + expect(ctx.cwd).not.toBe('') + }) + + it('treats an empty fixture as an empty header', () => { + expect(fixtureContext('')).toEqual({ sessionIds: [], cwd: '\0no-cwd\0' }) + }) +}) + +describe('normalizedHeaders', () => { + const header = (system: string): string => JSON.stringify({ + type: 'request/header', seq: 0, time: 9, data: { header: { config: { model: 'm' }, system }, reason: 'initial' }, + }) + + it('extracts every request/header payload in log order, normalized', () => { + const id = '11111111-2222-4333-8444-555555555555' + const log = `${JSON.stringify({ type: 'session', id, createdAt: 5, cwd: '/w' })}\n${header('one')}\n` + + `${JSON.stringify({ type: 'turn/start', seq: 1, time: 9, data: { turn: 1 } })}\n${header('two')}\n` + const headers = normalizedHeaders(log, { sessionIds: [id], cwd: '/w' }) + expect(headers).toEqual([ + { config: { model: 'm' }, system: 'one' }, + { config: { model: 'm' }, system: 'two' }, + ]) + }) + + it('yields nothing for a log without header events', () => { + const log = `${JSON.stringify({ type: 'session', id: 'a', createdAt: 5 })}\n` + expect(normalizedHeaders(log, { sessionIds: [], cwd: '/w' })).toEqual([]) + }) +}) + +describe('headerDeltaCount', () => { + it('counts request/header-delta events, ignoring blanks and other lines', () => { + const delta = JSON.stringify({ type: 'request/header-delta', seq: 2, time: 9, data: {} }) + const other = JSON.stringify({ type: 'request/header', seq: 0, time: 9, data: {} }) + expect(headerDeltaCount(`${other}\n\n${delta}\n${delta}\n`)).toBe(2) + expect(headerDeltaCount(`${other}\n`)).toBe(0) + }) +})