From 9339622d3b6da4070944ba10bc9eed2bb121fb25 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Sat, 11 Jul 2026 22:24:20 +0800 Subject: [PATCH 1/4] test(acp): snapshot system prompts as Markdown --- .../testing/2026-06-19-acp-snapshot-tests.md | 2 +- ...-request-header-content-in-one-scenario.md | 17 +- .../2026-07-08-shared-acp-snapshot-package.md | 2 +- examples/acp-agent/tests/acp.snapshot.ts | 5 +- .../snapshots/both-mode-turn/session.jsonl | 2 +- .../both-mode-turn/system-prompt.golden.md | 133 +++++++++++++ .../snapshots/code-mode-turn/session.jsonl | 2 +- .../code-mode-turn/system-prompt.golden.md | 133 +++++++++++++ .../tests/snapshots/text-turn/session.jsonl | 2 +- .../text-turn/system-prompt.golden.md | 18 ++ packages/support/acp-snapshot/README.md | 8 +- packages/support/acp-snapshot/src/index.ts | 3 +- .../support/acp-snapshot/src/normalize.ts | 87 +++++---- packages/support/acp-snapshot/src/suite.ts | 183 +++++++++++------- .../record-suite/rec-pin/session.jsonl | 2 +- .../rec-pin/system-prompt.golden.md | 1 + .../fixtures/suite/pin-turn/session.jsonl | 2 +- .../suite/pin-turn/system-prompt.golden.md | 1 + .../acp-snapshot/tests/normalize.spec.ts | 48 ++++- .../support/acp-snapshot/tests/suite.spec.ts | 27 +++ 20 files changed, 543 insertions(+), 135 deletions(-) create mode 100644 examples/acp-agent/tests/snapshots/both-mode-turn/system-prompt.golden.md create mode 100644 examples/acp-agent/tests/snapshots/code-mode-turn/system-prompt.golden.md create mode 100644 examples/acp-agent/tests/snapshots/text-turn/system-prompt.golden.md create mode 100644 packages/support/acp-snapshot/tests/fixtures/record-suite/rec-pin/system-prompt.golden.md create mode 100644 packages/support/acp-snapshot/tests/fixtures/suite/pin-turn/system-prompt.golden.md diff --git a/docs/rfc/implemented/testing/2026-06-19-acp-snapshot-tests.md b/docs/rfc/implemented/testing/2026-06-19-acp-snapshot-tests.md index ae5ea96f83..b424a253bd 100644 --- a/docs/rfc/implemented/testing/2026-06-19-acp-snapshot-tests.md +++ b/docs/rfc/implemented/testing/2026-06-19-acp-snapshot-tests.md @@ -51,7 +51,7 @@ The ACP server app loads `@deepseek-ai/dsh-llm-deepseek`, whose `apply` throws w A snapshot run asserts **two** normalized surfaces, because the harness's external surfaces are distinct: 1. The **stdout transcript** — the framed `session/update` JSON-RPC the editor sees. Catches regressions in the ACP bridge's event→update translation (`streamSessionEventUpdate`). Compared against a committed `stdout.golden.jsonl`. -2. The **re-persisted session JSONL** — the log the replay run itself persists, compared against the scenario's `session.jsonl`. Catches regressions in the loop, tool dispatch, and turn/step structure that never surface on stdout. There is no separate session golden: `session.jsonl` is BOTH the replay source (recorded scenarios) and the expected produced log. Both sides pass through `normalizeSessionLog` before comparing — the fixture is raw-harvested (its own real session id / cwd / timestamps) and the replay output has fresh ones, so each is scrubbed against ITS OWN volatile values (the fixture's read from its header line) and the comparison is on normalized form. Request-header CONTENT (the composed system prompt + tool schemas) is additionally scrubbed to `{{system}}`/`{{tools}}` tokens on both sides — in the stored fixtures too — for every scenario except the one that pins it ([pinned-header RFC](2026-07-06-pin-request-header-content-in-one-scenario.md)). For an authored override scenario the same `session.jsonl` holds the expected produced log; `replay.override.json` drives the model, and `llm-replay` ignores the fixture for model chunks when an override exists, so committing the expected log there does not affect replay. +2. The **re-persisted session JSONL** — the log the replay run itself persists, compared against the scenario's `session.jsonl`. Catches regressions in the loop, tool dispatch, and turn/step structure that never surface on stdout. There is no separate session golden: `session.jsonl` is both the replay source (recorded scenarios) and the expected produced log. Both sides pass through `normalizeSessionLog` before comparing — the fixture is raw-harvested (its own real session id / cwd / timestamps) and the replay output has fresh ones, so each is scrubbed against its own volatile values (the fixture's read from its header line) and the comparison is on normalized form. Every stored JSONL additionally scrubs composed prompt text to `{{system}}`; each header class's pinning scenario stores that prompt readably in `system-prompt.golden.md` and keeps the complete tool schemas in its JSONL, while other scenarios scrub schemas to `{{tools}}` ([pinned-header RFC](2026-07-06-pin-request-header-content-in-one-scenario.md)). For an authored override scenario the same `session.jsonl` holds the expected produced log; `replay.override.json` drives the model, and `llm-replay` ignores the fixture for model chunks when an override exists, so committing the expected log there does not affect replay. The two are genuinely additive: stdout is the bridge's *lossy projection* of the log (it drops `assistant/message.usage`, `step/*`, exact `seq`/`time`, and renders tool I/O differently), so a loop/tool/turn-structure regression can change the JSONL while leaving the stdout projection identical, and a bridge-translation regression can change stdout while the JSONL is untouched. Asserting the JSONL equality also echoes the proposed [universal replay fixture](../../proposed/testing/2026-06-11-deterministic-and-stress-testing.md) idea. diff --git a/docs/rfc/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md b/docs/rfc/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md index 862dfd42fa..00a60ad4bc 100644 --- a/docs/rfc/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md +++ b/docs/rfc/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md @@ -4,29 +4,30 @@ Status: implemented ## Problem -Every model-driving ACP snapshot fixture (`session.jsonl`) embedded the full composed system prompt and the complete tool-schema list in its `request/header` event — roughly 8 KB on one line, per fixture. That content is identical across the suite (byte-identical tool list everywhere, including subagent children; identical prompt modulo each recording's temp cwd), so any change touching a tool description or a system-prompt line had to update every fixture: re-record everything against the live API (churning model responses and stdout goldens along the way) or hand-edit ~35 giant header lines. Introducing the dynamic-workflows feature — one new tool plus one prompt paragraph — rewrote every snapshot fixture in the repo, burying the behavioral diff a reviewer should be reading. +An ACP snapshot suite needs to prove the exact composed system prompt and tool-schema list sent in each `request/header`, but duplicating that content inside every `session.jsonl` makes a prompt or schema edit rewrite dozens of giant one-line JSON records. Keeping one raw header avoids the duplication but still makes prompt review poor: prose is JSON-escaped onto one line and mixed with thousands of characters of tool schemas. ## Decision -Exactly one scenario — `text-turn`, flagged `pinsHeader` in the `acp.snapshot.ts` scenario table — commits and compares the full request-header content; the pin mechanics live in [`dsh-acp-snapshot`](../../../../packages/support/acp-snapshot/README.md), whose suite factory enforces one pin per consuming suite. Every other fixture stores and compares that content as stable tokens via the pure normalizer `scrubRequestHeaders` in that package's `normalize.ts`: a `request/header` event's `header.system` becomes `"{{system}}"` and `header.tools` becomes `"{{tools}}"`; a `request/header-delta` keeps its structural facts — the system delta's `keepStart`/`keepEnd` line positions with one `{{system}}` token per inserted line, the tools delta's added/removed/changed tool names — and tokenizes only the bulk (prompt text, schema bodies), so two different deltas still compare different. The scrub is composed in front of `normalizeSessionLog` on BOTH sides of a non-pinning scenario's log compare and applied to the harvested logs record mode writes, so a re-record cannot smuggle the content back. Absent fields stay absent — WHETHER a header carried a prompt or tools is behavior and stays visible — and `config`/`reason` stay verbatim: a model swap churns every fixture by design (it invalidates the recorded responses), while a prompt or schema edit churns none of them (replay derives model behavior exclusively from `assistant/chunk` events and never reads header content — see `dsh-llm-replay`). +Exactly one scenario per header-composition class is flagged `pinsHeader`. Its directory splits the pin by review format: `system-prompt.golden.md` contains the normalized composed prompt as ordinary Markdown, while `session.jsonl` keeps the full tool-schema list, config, and reason but stores `header.system` as `"{{system}}"`. Every other JSONL stores both the system prompt and tool list as `"{{system}}"` / `"{{tools}}"`. The pin mechanics live in [`dsh-acp-snapshot`](../../../../packages/support/acp-snapshot/README.md), whose suite factory enforces one pin per class. -A system-prompt or tool-schema change therefore lands as exactly one committed-fixture diff — the pinned `text-turn` header line — updated by hand or by re-recording that one scenario (`pnpm run test:snapshot:record` with `-t text-turn`). +The pure `scrubSystemPrompts` normalizer applies to every stored session fixture and tokenizes both an initial header's prompt and a header delta's inserted prompt lines. `scrubRequestHeaders` additionally tokenizes tool schemas and session-prefix content for non-pinning scenarios while retaining structural facts: system-delta positions and arity, added/removed/changed tool names, prefix message count, field presence, config, and reason. Record and refresh write-back apply the appropriate scrub before writing JSONL and regenerate the Markdown prompt from the normalized live header, so neither path can reintroduce prompt text into JSONL or leave the readable snapshot stale. -Guards make the split self-enforcing. On disk (fixture meta-tests): every non-pinning `session*.jsonl` must be a fixed point of `scrubRequestHeaders` (unscrubbed content crept in — apply the scrub), the pinning scenario's fixture must NOT be one (the pin lost its content), and exactly one scenario must pin. Live (every non-pinning scenario run): each `request/header` the run produces — parent, spawn child, fork child, initial or resume — must equal the pinned fixture's header after both sides normalize their own volatile values, and no `request/header-delta` may appear at all (a mid-run header change diverges from the pin by construction, and its content would be invisible under the scrub), so the single-pin premise is asserted rather than assumed. +Guards make the split self-enforcing. On disk, every `session*.jsonl` is a fixed point of `scrubSystemPrompts`, only non-pinning fixtures are fixed points of the full header scrub, `system-prompt.golden.md` exists exactly beside pinning fixtures, and each class has one pin. Live, every `request/header` produced by a parent, spawn child, fork child, initial request, or resume must match both halves of its class's pin after volatile-value normalization. A header without a string prompt or any `request/header-delta` fails loud because the two static pin artifacts cannot represent it. One pin covers the whole suite because every session — parent, spawn child, fork child — composes the identical tool list and the identical prompt modulo cwd, and the uniformity guard fails the suite the moment that stops holding. If header composition ever becomes session-dependent by design (a restricted subagent toolset, say), the divergent shape gets its own pinning scenario. ## Alternatives considered -- **Re-record or hand-edit every fixture per change** — the status quo; the churn this RFC removes. -- **Scrub at compare time only, keeping fixtures raw** — the compares go green without fixture edits, but every committed fixture then carries a permanently stale copy of the prompt and schemas: dead weight that misleads readers and still rewrites wholesale on the next re-record. Storing the tokens keeps the fixture honest about what it does and does not pin. +- **Re-record or hand-edit every fixture per change** — preserves exact headers but buries behavioral diffs under duplicated prompt and schema content. +- **Scrub at compare time only, keeping fixtures raw** — lets compares pass while committed fixtures retain stale duplicate content and rewrite wholesale on the next recording. Stored tokens state honestly what each JSONL does not pin. - **Scrub everywhere, pin nowhere** — loses the only end-to-end record of the composed header as actually sent (prompt assembly, registered-tool order, full schemas). The generated tool catalog documents each tool in isolation; only a real fixture pins the composed set. +- **Keep the one full pin entirely in JSONL** — removes suite-wide duplication but leaves system-prompt changes as an escaped one-line diff entangled with the tool list. Markdown gives prompt prose its natural review format without weakening the header assertion. - **Slim the session log itself (log a content digest, store the header elsewhere)** — violates the reconstructability contract: the product log must reproduce each request bit-for-bit ([reconstructable-requests RFC](../architecture/2026-07-05-reconstructable-requests.md)). Header bulk is a test-artifact concern, solved in test normalization; the live log is untouched. ## Verification -All 37 snapshot scenarios replay green with the scrubbed fixtures (the committed fixtures were rewritten once through `scrubRequestHeaders` itself; `text-turn` untouched). The fixed-point, pin-retains-content, exactly-one-pin, live header-uniformity, and no-unpinned-delta guards run inside the suite, and `scrubRequestHeaders` has unit coverage for both header event types, delta structure preservation (line positions, insert arity, tool names), absent-field preservation, config/reason retention, byte-for-byte pass-through of other lines, and idempotence. +The suite replays every scenario against the split pins. Unit coverage exercises both scrub levels, Markdown formatting, record/refresh regeneration, normalized prompt extraction, fixed-point enforcement, required-file symmetry, header uniformity, and delta rejection. ## Consequences -A tool-description or system-prompt change churns one committed fixture line instead of every fixture in the suite, so snapshot diffs read as behavior again, and ~270 KB of duplicated header bytes leave the repo. The cost: non-pinning fixtures no longer display header content, so reading one shows tokens where the prompt and schemas were — the pinned `text-turn` fixture is the place to look, and the live uniformity guard guarantees it speaks for every session in the suite. A header change surfaces as a suite-wide test failure whose fix is the one pinned line, rather than as ~35 fixture rewrites. +A system-prompt change produces a normal line-oriented Markdown diff in one file per affected composition class; a tool-description change produces one pinned JSONL line per class; ordinary behavioral fixtures remain untouched. Session fixtures display tokens for omitted content, and the live uniformity guard makes each split pin authoritative for every session in its class. The pinning scenario carries one extra generated artifact whose terminal newline is canonicalized for repository hygiene. diff --git a/docs/rfc/implemented/testing/2026-07-08-shared-acp-snapshot-package.md b/docs/rfc/implemented/testing/2026-07-08-shared-acp-snapshot-package.md index 910f179313..2f631a5d8a 100644 --- a/docs/rfc/implemented/testing/2026-07-08-shared-acp-snapshot-package.md +++ b/docs/rfc/implemented/testing/2026-07-08-shared-acp-snapshot-package.md @@ -16,7 +16,7 @@ The machinery lives in [`packages/support/acp-snapshot`](../../../../packages/su **`src/normalize.ts`** — the pure normalizers, hook-free by policy: when a future event carries a new volatile field (an approval duration, say), the shared normalizer learns it in the same change, keeping one home for what "normalized" means rather than per-suite scrub extensions. -**`src/suite.ts`** — the `Scenario` type and `defineAcpSnapshotSuite(options)`, registering the per-scenario compares, record-mode fixture write-back, the header pin with its live uniformity guard, and the fixture guard block (no orphan scenario dirs, required files present, exactly one pin, non-pinning fixtures are `scrubRequestHeaders` fixed points). The pinned-header contract ([pinned-header RFC](2026-07-06-pin-request-header-content-in-one-scenario.md)) is per-suite: each suite flags exactly one `pinsHeader` scenario (the factory throws on zero, a meta-test rejects more than one; WHICH scenario pins is the table's reviewable choice), and the uniformity guard compares only that suite's sessions. The pure helpers (`childFixturePaths`, `fixtureContext`, `normalizedHeaders`, `headerDeltaCount`) are exported for direct unit coverage. +**`src/suite.ts`** — the `Scenario` type and `defineAcpSnapshotSuite(options)`, registering the per-scenario compares, record/refresh fixture write-back, the header pin with its live uniformity guard, and the fixture guard block (no orphan scenario dirs, required files present, exactly one pin per class, every JSONL a `scrubSystemPrompts` fixed point, non-pinning fixtures also `scrubRequestHeaders` fixed points). The pinned-header contract ([pinned-header RFC](2026-07-06-pin-request-header-content-in-one-scenario.md)) is per-suite: each header class flags exactly one `pinsHeader` scenario, whose `system-prompt.golden.md` and JSONL tool list split the composed header into reviewable artifacts; the uniformity guard compares both against every live header in that class. The pure helpers (`childFixturePaths`, `fixtureContext`, `normalizedHeaders`, `normalizedSystemPrompts`, `formatSystemPromptSnapshot`, `headerDeltaCount`) are exported from the module for direct unit coverage. ## Alternatives considered diff --git a/examples/acp-agent/tests/acp.snapshot.ts b/examples/acp-agent/tests/acp.snapshot.ts index 7102d8ce88..7bfc2a8c62 100644 --- a/examples/acp-agent/tests/acp.snapshot.ts +++ b/examples/acp-agent/tests/acp.snapshot.ts @@ -45,9 +45,8 @@ function snapshotModeFromEnv(value: string | undefined): SnapshotSuiteOptions['m const SCENARIOS: Scenario[] = [ { name: 'handshake', hasModelTurn: false, recorded: false }, { name: 'reject-extra-dirs', hasModelTurn: false, recorded: false }, - // text-turn is the pinned-header scenario: the minimal single text turn, - // whose fixture is the ONE place the full system prompt + tool schemas are - // committed and compared verbatim. + // text-turn is the pinned-header scenario: the minimal single text turn. + // Its system-prompt.golden.md and JSONL tool list pin the composed header. { name: 'text-turn', hasModelTurn: true, recorded: true, pinsHeader: true }, { name: 'tool-call-turn', hasModelTurn: true, recorded: true }, { name: 'fs-terminal-card', hasModelTurn: true, recorded: true }, diff --git a/examples/acp-agent/tests/snapshots/both-mode-turn/session.jsonl b/examples/acp-agent/tests/snapshots/both-mode-turn/session.jsonl index df7187bbbb..65c2729e0a 100644 --- a/examples/acp-agent/tests/snapshots/both-mode-turn/session.jsonl +++ b/examples/acp-agent/tests/snapshots/both-mode-turn/session.jsonl @@ -2,7 +2,7 @@ {"type":"turn/start","seq":0,"time":1783611774323,"data":{"turn":1,"trigger":{"kind":"message","source":{"kind":"user"}}}} {"type":"user/message","seq":1,"time":1783611774323,"data":{"content":[{"type":"text","text":"Call the run_code tool (NOT the native bash tool directly) with a program that runs exactly `echo BOTH_OK` via tools.bash and returns its output. Then reply with that output only and stop."}],"source":{"kind":"user"}},"surfaceOp":"append"} {"type":"step/start","seq":2,"time":1783611774324,"data":{"turn":1,"step":1}} -{"type":"request/header","seq":3,"time":1783611774325,"data":{"header":{"config":{"model":"deepseek-v4-flash"},"system":"You are an AI agent powered by the DeepSeek Harness SDK.\n\nYou are a coding assistant powered by the deepseek-v4-flash model. Your working\ndirectory is /var/folders/2c/psb0_fmx7hbgz558xjt_f0l00000gn/T/acp-snap-cwd-52lrTl.\n\nVerify your work by running the code or tests. Keep answers brief and\nfactual.\n\n\nUse the read tool — not shell commands like cat — to inspect text files. Results include line numbers. Use offset and limit to continue reading large files.\n\nUse the write tool to create files or completely replace file contents. Existing files are overwritten, so read an existing file first (the default fs-policy requires it) and prefer edit for targeted changes.\n\nUse the edit tool for targeted changes to existing UTF-8 text files. It replaces literal old_string with new_string; by default old_string must appear exactly once. If old_string appears multiple times, provide a more specific old_string or set replace_all to true. Read the file first (the default fs-policy requires it), unless you just created or edited it in this session.\n\nCheck the [exit code: N] marker on every bash result; investigate failures before moving on.\n\nUse the workflow tool ONLY when the user explicitly asks for a workflow or for large multi-agent orchestration: you write a JavaScript script (the tool description documents the exact format) that fans work out across many subagents with phases and structured results. For one or two delegations, prefer plain subagent calls.\n\n## Writing code for run_code\n\nPass `run_code` the body of an async TypeScript function (erasable syntax only — no `enum` or namespaces; type annotations are advisory, the code runs type-stripped). Inside the program:\n\n- Call tools as `await tools.name(args)` — quoted access for exotic names: `tools[\"my-tool\"](args)`. Every call resolves to the tool's text output as a string. Tool arguments must be JSON-serializable.\n- A FAILED tool call rejects with an `Error` carrying the tool's error text — `try/catch` it to handle and continue.\n- Calls execute sequentially, even under `Promise.all`.\n- Emit results with `return` and/or `console.log(...)`. ONLY what you print or return comes back to you — intermediate tool results never enter the conversation, so extract just what you need.\n\nThe available tools:\n\n```ts\ndeclare const tools: {\n /** Execute a bash command (`bash -c`) and return its stdout/stderr. Each call runs in a fresh shell: no state (cwd, variables, functions) persists between calls — pass `workdir` instead of using `cd`. Non-zero exits are reported as `[exit code: N]`. Long output is truncated to its tail; the full output is saved to a file whose path is reported when available. Set `run_in_background: true` for long-running commands: the call returns a task id immediately; poll it with `bash_output` and stop it with `bash_kill`. */\n bash(args: {\n /** The bash command to execute. */\n command: string;\n /** Clear, concise description of what this command does in active voice, 5-10 words (shown in the UI). Examples: \"ls\" → \"List files in current directory\"; \"git status\" → \"Show working tree status\"; \"npm install\" → \"Install package dependencies\". */\n description: string;\n /** Timeout in milliseconds. The executor applies its configured default and cap, and kills the command on expiry. */\n timeoutMs?: number;\n /** Working directory for this command. Defaults to the session workspace; a relative path is resolved against it. */\n workdir?: string;\n /** Run in the background and return a task id immediately. No timeout applies. */\n run_in_background?: boolean;\n }): Promise;\n /** Ask the executor to kill a running background bash task by task id. */\n bash_kill(args: {\n /** Task id returned by the bash tool. */\n task_id: string;\n }): Promise;\n /** Read new output from a background bash task started with `bash` + `run_in_background`. Returns only output produced since the previous bash_output call, plus the task status. Tasks keep running while you do other work; poll again later for more output. */\n bash_output(args: {\n /** Task id returned by the bash tool. */\n task_id: string;\n }): Promise;\n /** Edit an existing UTF-8 text file by replacing literal text. */\n edit(args: {\n /** Path to edit, resolved by the filesystem backend. */\n file_path: string;\n /** Literal text to replace. Must match exactly. */\n old_string: string;\n /** Literal replacement text. Use an empty string to delete the match. */\n new_string: string;\n /** Replace all matches. Defaults to false; when false, old_string must appear exactly once. */\n replace_all?: boolean;\n }): Promise;\n /** Read a UTF-8 text file and return line-numbered content. */\n read(args: {\n /** Path to read, resolved by the filesystem backend. */\n file_path: string;\n /** 1-based first line to return. Defaults to 1. */\n offset?: number;\n /** Maximum number of lines to return. Defaults to 2000. */\n limit?: number;\n }): Promise;\n /** Delegate a self-contained task to a subagent (a separate agent that works in its own context) and return its final result. Use this to offload focused, independent work — research, a scoped implementation, an analysis — so it does not consume this conversation's context. The subagent runs to completion and you receive only its final answer, not its intermediate steps. Give it a complete, standalone prompt: it does not see this conversation. */\n subagent(args: {\n /** A short (3-5 word) description of the delegated task, for display. */\n description: string;\n /** The complete, self-contained task for the subagent. It does not share this conversation's context, so include everything it needs. */\n prompt: string;\n }): Promise;\n /** Delegate a task to a subagent that INHERITS this conversation: a child agent seeded with all completed turns so far (it does not see the current in-flight turn), returning only its final result. Use this when the subtask builds on this conversation's context — a follow-up analysis, a review, a continuation — without consuming this conversation's context for the work itself. You receive only its final answer, not its intermediate steps. */\n subagent_fork(args: {\n /** A short (3-5 word) description of the delegated task, for display. */\n description: string;\n /** The task for the subagent. It already sees this conversation's completed turns, so build on them freely and state only what is new. */\n prompt: string;\n }): Promise;\n /** Record and update a structured task list for the current work. Send the ENTIRE list every call — it REPLACES the previous list (there are no partial updates, no per-item edits). Use it to plan multi-step work and show progress: add one todo per concrete step before you start. Keep AT MOST ONE todo `in_progress` at a time; while work remains, exactly one active task should be `in_progress`. Mark a todo `completed` the moment it is done (do not batch completions), and allow no `in_progress` item only once all work is complete. Skip the list for trivial single-step tasks. Statuses: `pending` (not started), `in_progress` (being worked on now), `completed` (finished). */\n todo_write(args: {\n /** The COMPLETE task list, replacing any previous list. */\n todos: ({\n /** What the task is — a short imperative line. */\n content: string;\n /** pending (not started) | in_progress (now) | completed (done). */\n status: \"pending\" | \"in_progress\" | \"completed\";\n })[];\n }): Promise;\n /** Run a JavaScript workflow script that orchestrates subagents at scale. Use this for work that fans out across many independent pieces — an audit over many files, a migration, multi-angle research, adversarial verification of findings — where you write the orchestration as a script instead of delegating turn by turn. The workflow's identity rides the `meta` parameter as JSON: required `name` (short kebab-case) and `description` strings, optional `whenToUse` string and `phases` array (`{title, detail?, model?}`). The `script` parameter is the plain JavaScript body ONLY (NOT TypeScript, and NO `export const meta` statement — meta is a parameter, not code), running with top-level await; end with `return ` — the value must be JSON-serializable and is this tool's result. Script-body hooks: - `agent(prompt, opts?): Promise` — run one subagent to completion. Without `opts.schema` it resolves to the child's final text; with `opts.schema` (an object-rooted JSON Schema using ONLY type/properties/required/additionalProperties/items/enum/const — no oneOf/pattern/format/numeric bounds) it resolves to the validated object. Resolves `null` when the child fails (filter with `.filter(Boolean)`). Other opts: `label` (display), `phase` (progress group), `model` (override). Anything else (`effort`/`isolation`/`agentType`) is rejected loudly. - `pipeline(items, ...stages): Promise` — run each item through the stages independently with NO barrier between stages (prefer this for multi-stage work). Each stage receives `(prev, item, index)`. An ordinary stage throw drops that ITEM to `null` and skips its remaining stages. - `parallel(thunks): Promise` — run zero-argument functions concurrently and await ALL of them (a barrier; use only when a stage genuinely needs every prior result together). A throwing thunk resolves to `null`. - `phase(title)` — start a progress phase; `log(message)` — narrate progress; `args` — the tool call's `args` input, verbatim. Misused hooks (bad arguments, unknown options, unsupported schemas, tripped caps) throw errors that ALWAYS kill the script — they never dissolve into a per-item `null`. Constraints: concurrency and total-agent caps apply; no filesystem, network, timers, or Node.js APIs are provided — the agents do the work, the script only coordinates them. The run executes in the foreground: this call returns when the whole script finishes. */\n workflow(args: {\n /** The plain-JS workflow script body (top-level await allowed; NO `export const meta` statement; end with `return `). */\n script: string;\n /** The workflow identity block (plain JSON — never code). */\n meta: {\n /** Short kebab-case workflow name. */\n name: string;\n /** One-line description of what the workflow does. */\n description: string;\n /** Optional guidance on when this workflow applies. */\n whenToUse?: string;\n /** Optional phase declarations matched by phase() calls. */\n phases?: {\n /** The phase title phase() calls match by exact string. */\n title: string;\n /** Optional one-line description of the phase. */\n detail?: string;\n /** Optional model override this phase is expected to use. */\n model?: string;\n }[];\n };\n /** Optional JSON input exposed to the script as the `args` global (wrap a bare list as a field, e.g. {\"files\": [...]}). */\n args?: Record;\n }): Promise;\n /** Create or fully replace a UTF-8 text file. */\n write(args: {\n /** Path to write, resolved by the filesystem backend. */\n file_path: string;\n /** Full UTF-8 text content to write. */\n content: string;\n }): Promise;\n}\n```","tools":[{"name":"bash","description":"Execute a bash command (`bash -c`) and return its stdout/stderr. Each call runs in a fresh shell: no state (cwd, variables, functions) persists between calls — pass `workdir` instead of using `cd`. Non-zero exits are reported as `[exit code: N]`. Long output is truncated to its tail; the full output is saved to a file whose path is reported when available. Set `run_in_background: true` for long-running commands: the call returns a task id immediately; poll it with `bash_output` and stop it with `bash_kill`.","parameters":{"type":"object","properties":{"command":{"type":"string","description":"The bash command to execute."},"description":{"type":"string","description":"Clear, concise description of what this command does in active voice, 5-10 words (shown in the UI). Examples: \"ls\" → \"List files in current directory\"; \"git status\" → \"Show working tree status\"; \"npm install\" → \"Install package dependencies\"."},"timeoutMs":{"type":"number","description":"Timeout in milliseconds. The executor applies its configured default and cap, and kills the command on expiry."},"workdir":{"type":"string","description":"Working directory for this command. Defaults to the session workspace; a relative path is resolved against it."},"run_in_background":{"type":"boolean","description":"Run in the background and return a task id immediately. No timeout applies."}},"required":["command","description"]}},{"name":"bash_kill","description":"Ask the executor to kill a running background bash task by task id.","parameters":{"type":"object","properties":{"task_id":{"type":"string","description":"Task id returned by the bash tool."}},"required":["task_id"]}},{"name":"bash_output","description":"Read new output from a background bash task started with `bash` + `run_in_background`. Returns only output produced since the previous bash_output call, plus the task status. Tasks keep running while you do other work; poll again later for more output.","parameters":{"type":"object","properties":{"task_id":{"type":"string","description":"Task id returned by the bash tool."}},"required":["task_id"]}},{"name":"edit","description":"Edit an existing UTF-8 text file by replacing literal text.","parameters":{"type":"object","properties":{"file_path":{"type":"string","description":"Path to edit, resolved by the filesystem backend."},"old_string":{"type":"string","description":"Literal text to replace. Must match exactly."},"new_string":{"type":"string","description":"Literal replacement text. Use an empty string to delete the match."},"replace_all":{"type":"boolean","description":"Replace all matches. Defaults to false; when false, old_string must appear exactly once."}},"required":["file_path","old_string","new_string"]}},{"name":"read","description":"Read a UTF-8 text file and return line-numbered content.","parameters":{"type":"object","properties":{"file_path":{"type":"string","description":"Path to read, resolved by the filesystem backend."},"offset":{"type":"number","description":"1-based first line to return. Defaults to 1."},"limit":{"type":"number","description":"Maximum number of lines to return. Defaults to 2000."}},"required":["file_path"]}},{"name":"run_code","description":"Execute a TypeScript program against the available tools. Write the BODY of an async function (erasable syntax only; top-level `await` and `return` work) and call tools as `await tools.name(args)` per the declarations in the system prompt. Only what you print or return comes back — curate it.","parameters":{"type":"object","properties":{"code":{"type":"string","description":"The program: the body of an async TypeScript function."}},"required":["code"]}},{"name":"subagent","description":"Delegate a self-contained task to a subagent (a separate agent that works in its own context) and return its final result. Use this to offload focused, independent work — research, a scoped implementation, an analysis — so it does not consume this conversation's context. The subagent runs to completion and you receive only its final answer, not its intermediate steps. Give it a complete, standalone prompt: it does not see this conversation.","parameters":{"type":"object","properties":{"description":{"type":"string","description":"A short (3-5 word) description of the delegated task, for display."},"prompt":{"type":"string","description":"The complete, self-contained task for the subagent. It does not share this conversation's context, so include everything it needs."}},"required":["description","prompt"]}},{"name":"subagent_fork","description":"Delegate a task to a subagent that INHERITS this conversation: a child agent seeded with all completed turns so far (it does not see the current in-flight turn), returning only its final result. Use this when the subtask builds on this conversation's context — a follow-up analysis, a review, a continuation — without consuming this conversation's context for the work itself. You receive only its final answer, not its intermediate steps.","parameters":{"type":"object","properties":{"description":{"type":"string","description":"A short (3-5 word) description of the delegated task, for display."},"prompt":{"type":"string","description":"The task for the subagent. It already sees this conversation's completed turns, so build on them freely and state only what is new."}},"required":["description","prompt"]}},{"name":"todo_write","description":"Record and update a structured task list for the current work. Send the ENTIRE list every call — it REPLACES the previous list (there are no partial updates, no per-item edits). Use it to plan multi-step work and show progress: add one todo per concrete step before you start. Keep AT MOST ONE todo `in_progress` at a time; while work remains, exactly one active task should be `in_progress`. Mark a todo `completed` the moment it is done (do not batch completions), and allow no `in_progress` item only once all work is complete. Skip the list for trivial single-step tasks. Statuses: `pending` (not started), `in_progress` (being worked on now), `completed` (finished).","parameters":{"type":"object","properties":{"todos":{"type":"array","description":"The COMPLETE task list, replacing any previous list.","items":{"type":"object","properties":{"content":{"type":"string","description":"What the task is — a short imperative line."},"status":{"type":"string","description":"pending (not started) | in_progress (now) | completed (done).","enum":["pending","in_progress","completed"]}},"required":["content","status"]}}},"required":["todos"]}},{"name":"workflow","description":"Run a JavaScript workflow script that orchestrates subagents at scale. Use this for work that fans out across many independent pieces — an audit over many files, a migration, multi-angle research, adversarial verification of findings — where you write the orchestration as a script instead of delegating turn by turn.\n\nThe workflow's identity rides the `meta` parameter as JSON: required `name` (short kebab-case) and `description` strings, optional `whenToUse` string and `phases` array (`{title, detail?, model?}`). The `script` parameter is the plain JavaScript body ONLY (NOT TypeScript, and NO `export const meta` statement — meta is a parameter, not code), running with top-level await; end with `return ` — the value must be JSON-serializable and is this tool's result.\n\nScript-body hooks:\n- `agent(prompt, opts?): Promise` — run one subagent to completion. Without `opts.schema` it resolves to the child's final text; with `opts.schema` (an object-rooted JSON Schema using ONLY type/properties/required/additionalProperties/items/enum/const — no oneOf/pattern/format/numeric bounds) it resolves to the validated object. Resolves `null` when the child fails (filter with `.filter(Boolean)`). Other opts: `label` (display), `phase` (progress group), `model` (override). Anything else (`effort`/`isolation`/`agentType`) is rejected loudly.\n- `pipeline(items, ...stages): Promise` — run each item through the stages independently with NO barrier between stages (prefer this for multi-stage work). Each stage receives `(prev, item, index)`. An ordinary stage throw drops that ITEM to `null` and skips its remaining stages.\n- `parallel(thunks): Promise` — run zero-argument functions concurrently and await ALL of them (a barrier; use only when a stage genuinely needs every prior result together). A throwing thunk resolves to `null`.\n- `phase(title)` — start a progress phase; `log(message)` — narrate progress; `args` — the tool call's `args` input, verbatim.\n\nMisused hooks (bad arguments, unknown options, unsupported schemas, tripped caps) throw errors that ALWAYS kill the script — they never dissolve into a per-item `null`.\n\nConstraints: concurrency and total-agent caps apply; no filesystem, network, timers, or Node.js APIs are provided — the agents do the work, the script only coordinates them. The run executes in the foreground: this call returns when the whole script finishes.","parameters":{"type":"object","properties":{"script":{"type":"string","description":"The plain-JS workflow script body (top-level await allowed; NO `export const meta` statement; end with `return `)."},"meta":{"type":"object","description":"The workflow identity block (plain JSON — never code).","properties":{"name":{"type":"string","description":"Short kebab-case workflow name."},"description":{"type":"string","description":"One-line description of what the workflow does."},"whenToUse":{"type":"string","description":"Optional guidance on when this workflow applies."},"phases":{"type":"array","description":"Optional phase declarations matched by phase() calls.","items":{"type":"object","properties":{"title":{"type":"string","description":"The phase title phase() calls match by exact string."},"detail":{"type":"string","description":"Optional one-line description of the phase."},"model":{"type":"string","description":"Optional model override this phase is expected to use."}},"required":["title"]}}},"required":["name","description"]},"args":{"type":"object","description":"Optional JSON input exposed to the script as the `args` global (wrap a bare list as a field, e.g. {\"files\": [...]})."}},"required":["script","meta"]}},{"name":"write","description":"Create or fully replace a UTF-8 text file.","parameters":{"type":"object","properties":{"file_path":{"type":"string","description":"Path to write, resolved by the filesystem backend."},"content":{"type":"string","description":"Full UTF-8 text content to write."}},"required":["file_path","content"]}}]},"reason":"initial"}} +{"type":"request/header","seq":3,"time":1783611774325,"data":{"header":{"config":{"model":"deepseek-v4-flash"},"system":"{{system}}","tools":[{"name":"bash","description":"Execute a bash command (`bash -c`) and return its stdout/stderr. Each call runs in a fresh shell: no state (cwd, variables, functions) persists between calls — pass `workdir` instead of using `cd`. Non-zero exits are reported as `[exit code: N]`. Long output is truncated to its tail; the full output is saved to a file whose path is reported when available. Set `run_in_background: true` for long-running commands: the call returns a task id immediately; poll it with `bash_output` and stop it with `bash_kill`.","parameters":{"type":"object","properties":{"command":{"type":"string","description":"The bash command to execute."},"description":{"type":"string","description":"Clear, concise description of what this command does in active voice, 5-10 words (shown in the UI). Examples: \"ls\" → \"List files in current directory\"; \"git status\" → \"Show working tree status\"; \"npm install\" → \"Install package dependencies\"."},"timeoutMs":{"type":"number","description":"Timeout in milliseconds. The executor applies its configured default and cap, and kills the command on expiry."},"workdir":{"type":"string","description":"Working directory for this command. Defaults to the session workspace; a relative path is resolved against it."},"run_in_background":{"type":"boolean","description":"Run in the background and return a task id immediately. No timeout applies."}},"required":["command","description"]}},{"name":"bash_kill","description":"Ask the executor to kill a running background bash task by task id.","parameters":{"type":"object","properties":{"task_id":{"type":"string","description":"Task id returned by the bash tool."}},"required":["task_id"]}},{"name":"bash_output","description":"Read new output from a background bash task started with `bash` + `run_in_background`. Returns only output produced since the previous bash_output call, plus the task status. Tasks keep running while you do other work; poll again later for more output.","parameters":{"type":"object","properties":{"task_id":{"type":"string","description":"Task id returned by the bash tool."}},"required":["task_id"]}},{"name":"edit","description":"Edit an existing UTF-8 text file by replacing literal text.","parameters":{"type":"object","properties":{"file_path":{"type":"string","description":"Path to edit, resolved by the filesystem backend."},"old_string":{"type":"string","description":"Literal text to replace. Must match exactly."},"new_string":{"type":"string","description":"Literal replacement text. Use an empty string to delete the match."},"replace_all":{"type":"boolean","description":"Replace all matches. Defaults to false; when false, old_string must appear exactly once."}},"required":["file_path","old_string","new_string"]}},{"name":"read","description":"Read a UTF-8 text file and return line-numbered content.","parameters":{"type":"object","properties":{"file_path":{"type":"string","description":"Path to read, resolved by the filesystem backend."},"offset":{"type":"number","description":"1-based first line to return. Defaults to 1."},"limit":{"type":"number","description":"Maximum number of lines to return. Defaults to 2000."}},"required":["file_path"]}},{"name":"run_code","description":"Execute a TypeScript program against the available tools. Write the BODY of an async function (erasable syntax only; top-level `await` and `return` work) and call tools as `await tools.name(args)` per the declarations in the system prompt. Only what you print or return comes back — curate it.","parameters":{"type":"object","properties":{"code":{"type":"string","description":"The program: the body of an async TypeScript function."}},"required":["code"]}},{"name":"subagent","description":"Delegate a self-contained task to a subagent (a separate agent that works in its own context) and return its final result. Use this to offload focused, independent work — research, a scoped implementation, an analysis — so it does not consume this conversation's context. The subagent runs to completion and you receive only its final answer, not its intermediate steps. Give it a complete, standalone prompt: it does not see this conversation.","parameters":{"type":"object","properties":{"description":{"type":"string","description":"A short (3-5 word) description of the delegated task, for display."},"prompt":{"type":"string","description":"The complete, self-contained task for the subagent. It does not share this conversation's context, so include everything it needs."}},"required":["description","prompt"]}},{"name":"subagent_fork","description":"Delegate a task to a subagent that INHERITS this conversation: a child agent seeded with all completed turns so far (it does not see the current in-flight turn), returning only its final result. Use this when the subtask builds on this conversation's context — a follow-up analysis, a review, a continuation — without consuming this conversation's context for the work itself. You receive only its final answer, not its intermediate steps.","parameters":{"type":"object","properties":{"description":{"type":"string","description":"A short (3-5 word) description of the delegated task, for display."},"prompt":{"type":"string","description":"The task for the subagent. It already sees this conversation's completed turns, so build on them freely and state only what is new."}},"required":["description","prompt"]}},{"name":"todo_write","description":"Record and update a structured task list for the current work. Send the ENTIRE list every call — it REPLACES the previous list (there are no partial updates, no per-item edits). Use it to plan multi-step work and show progress: add one todo per concrete step before you start. Keep AT MOST ONE todo `in_progress` at a time; while work remains, exactly one active task should be `in_progress`. Mark a todo `completed` the moment it is done (do not batch completions), and allow no `in_progress` item only once all work is complete. Skip the list for trivial single-step tasks. Statuses: `pending` (not started), `in_progress` (being worked on now), `completed` (finished).","parameters":{"type":"object","properties":{"todos":{"type":"array","description":"The COMPLETE task list, replacing any previous list.","items":{"type":"object","properties":{"content":{"type":"string","description":"What the task is — a short imperative line."},"status":{"type":"string","description":"pending (not started) | in_progress (now) | completed (done).","enum":["pending","in_progress","completed"]}},"required":["content","status"]}}},"required":["todos"]}},{"name":"workflow","description":"Run a JavaScript workflow script that orchestrates subagents at scale. Use this for work that fans out across many independent pieces — an audit over many files, a migration, multi-angle research, adversarial verification of findings — where you write the orchestration as a script instead of delegating turn by turn.\n\nThe workflow's identity rides the `meta` parameter as JSON: required `name` (short kebab-case) and `description` strings, optional `whenToUse` string and `phases` array (`{title, detail?, model?}`). The `script` parameter is the plain JavaScript body ONLY (NOT TypeScript, and NO `export const meta` statement — meta is a parameter, not code), running with top-level await; end with `return ` — the value must be JSON-serializable and is this tool's result.\n\nScript-body hooks:\n- `agent(prompt, opts?): Promise` — run one subagent to completion. Without `opts.schema` it resolves to the child's final text; with `opts.schema` (an object-rooted JSON Schema using ONLY type/properties/required/additionalProperties/items/enum/const — no oneOf/pattern/format/numeric bounds) it resolves to the validated object. Resolves `null` when the child fails (filter with `.filter(Boolean)`). Other opts: `label` (display), `phase` (progress group), `model` (override). Anything else (`effort`/`isolation`/`agentType`) is rejected loudly.\n- `pipeline(items, ...stages): Promise` — run each item through the stages independently with NO barrier between stages (prefer this for multi-stage work). Each stage receives `(prev, item, index)`. An ordinary stage throw drops that ITEM to `null` and skips its remaining stages.\n- `parallel(thunks): Promise` — run zero-argument functions concurrently and await ALL of them (a barrier; use only when a stage genuinely needs every prior result together). A throwing thunk resolves to `null`.\n- `phase(title)` — start a progress phase; `log(message)` — narrate progress; `args` — the tool call's `args` input, verbatim.\n\nMisused hooks (bad arguments, unknown options, unsupported schemas, tripped caps) throw errors that ALWAYS kill the script — they never dissolve into a per-item `null`.\n\nConstraints: concurrency and total-agent caps apply; no filesystem, network, timers, or Node.js APIs are provided — the agents do the work, the script only coordinates them. The run executes in the foreground: this call returns when the whole script finishes.","parameters":{"type":"object","properties":{"script":{"type":"string","description":"The plain-JS workflow script body (top-level await allowed; NO `export const meta` statement; end with `return `)."},"meta":{"type":"object","description":"The workflow identity block (plain JSON — never code).","properties":{"name":{"type":"string","description":"Short kebab-case workflow name."},"description":{"type":"string","description":"One-line description of what the workflow does."},"whenToUse":{"type":"string","description":"Optional guidance on when this workflow applies."},"phases":{"type":"array","description":"Optional phase declarations matched by phase() calls.","items":{"type":"object","properties":{"title":{"type":"string","description":"The phase title phase() calls match by exact string."},"detail":{"type":"string","description":"Optional one-line description of the phase."},"model":{"type":"string","description":"Optional model override this phase is expected to use."}},"required":["title"]}}},"required":["name","description"]},"args":{"type":"object","description":"Optional JSON input exposed to the script as the `args` global (wrap a bare list as a field, e.g. {\"files\": [...]})."}},"required":["script","meta"]}},{"name":"write","description":"Create or fully replace a UTF-8 text file.","parameters":{"type":"object","properties":{"file_path":{"type":"string","description":"Path to write, resolved by the filesystem backend."},"content":{"type":"string","description":"Full UTF-8 text content to write."}},"required":["file_path","content"]}}]},"reason":"initial"}} {"type":"assistant/chunk","seq":4,"time":1783611774792,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"reasoning"}}} {"type":"assistant/chunk","seq":5,"time":1783611774792,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"The"}}} {"type":"assistant/chunk","seq":6,"time":1783611774879,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" user"}}} diff --git a/examples/acp-agent/tests/snapshots/both-mode-turn/system-prompt.golden.md b/examples/acp-agent/tests/snapshots/both-mode-turn/system-prompt.golden.md new file mode 100644 index 0000000000..688557b132 --- /dev/null +++ b/examples/acp-agent/tests/snapshots/both-mode-turn/system-prompt.golden.md @@ -0,0 +1,133 @@ +You are an AI agent powered by the DeepSeek Harness SDK. + +You are a coding assistant powered by the deepseek-v4-flash model. Your working +directory is {{cwd}}. + +Verify your work by running the code or tests. Keep answers brief and +factual. + + +Use the read tool — not shell commands like cat — to inspect text files. Results include line numbers. Use offset and limit to continue reading large files. + +Use the write tool to create files or completely replace file contents. Existing files are overwritten, so read an existing file first (the default fs-policy requires it) and prefer edit for targeted changes. + +Use the edit tool for targeted changes to existing UTF-8 text files. It replaces literal old_string with new_string; by default old_string must appear exactly once. If old_string appears multiple times, provide a more specific old_string or set replace_all to true. Read the file first (the default fs-policy requires it), unless you just created or edited it in this session. + +Check the [exit code: N] marker on every bash result; investigate failures before moving on. + +Use the workflow tool ONLY when the user explicitly asks for a workflow or for large multi-agent orchestration: you write a JavaScript script (the tool description documents the exact format) that fans work out across many subagents with phases and structured results. For one or two delegations, prefer plain subagent calls. + +## Writing code for run_code + +Pass `run_code` the body of an async TypeScript function (erasable syntax only — no `enum` or namespaces; type annotations are advisory, the code runs type-stripped). Inside the program: + +- Call tools as `await tools.name(args)` — quoted access for exotic names: `tools["my-tool"](args)`. Every call resolves to the tool's text output as a string. Tool arguments must be JSON-serializable. +- A FAILED tool call rejects with an `Error` carrying the tool's error text — `try/catch` it to handle and continue. +- Calls execute sequentially, even under `Promise.all`. +- Emit results with `return` and/or `console.log(...)`. ONLY what you print or return comes back to you — intermediate tool results never enter the conversation, so extract just what you need. + +The available tools: + +```ts +declare const tools: { + /** Execute a bash command (`bash -c`) and return its stdout/stderr. Each call runs in a fresh shell: no state (cwd, variables, functions) persists between calls — pass `workdir` instead of using `cd`. Non-zero exits are reported as `[exit code: N]`. Long output is truncated to its tail; the full output is saved to a file whose path is reported when available. Set `run_in_background: true` for long-running commands: the call returns a task id immediately; poll it with `bash_output` and stop it with `bash_kill`. */ + bash(args: { + /** The bash command to execute. */ + command: string; + /** Clear, concise description of what this command does in active voice, 5-10 words (shown in the UI). Examples: "ls" → "List files in current directory"; "git status" → "Show working tree status"; "npm install" → "Install package dependencies". */ + description: string; + /** Timeout in milliseconds. The executor applies its configured default and cap, and kills the command on expiry. */ + timeoutMs?: number; + /** Working directory for this command. Defaults to the session workspace; a relative path is resolved against it. */ + workdir?: string; + /** Run in the background and return a task id immediately. No timeout applies. */ + run_in_background?: boolean; + }): Promise; + /** Ask the executor to kill a running background bash task by task id. */ + bash_kill(args: { + /** Task id returned by the bash tool. */ + task_id: string; + }): Promise; + /** Read new output from a background bash task started with `bash` + `run_in_background`. Returns only output produced since the previous bash_output call, plus the task status. Tasks keep running while you do other work; poll again later for more output. */ + bash_output(args: { + /** Task id returned by the bash tool. */ + task_id: string; + }): Promise; + /** Edit an existing UTF-8 text file by replacing literal text. */ + edit(args: { + /** Path to edit, resolved by the filesystem backend. */ + file_path: string; + /** Literal text to replace. Must match exactly. */ + old_string: string; + /** Literal replacement text. Use an empty string to delete the match. */ + new_string: string; + /** Replace all matches. Defaults to false; when false, old_string must appear exactly once. */ + replace_all?: boolean; + }): Promise; + /** Read a UTF-8 text file and return line-numbered content. */ + read(args: { + /** Path to read, resolved by the filesystem backend. */ + file_path: string; + /** 1-based first line to return. Defaults to 1. */ + offset?: number; + /** Maximum number of lines to return. Defaults to 2000. */ + limit?: number; + }): Promise; + /** Delegate a self-contained task to a subagent (a separate agent that works in its own context) and return its final result. Use this to offload focused, independent work — research, a scoped implementation, an analysis — so it does not consume this conversation's context. The subagent runs to completion and you receive only its final answer, not its intermediate steps. Give it a complete, standalone prompt: it does not see this conversation. */ + subagent(args: { + /** A short (3-5 word) description of the delegated task, for display. */ + description: string; + /** The complete, self-contained task for the subagent. It does not share this conversation's context, so include everything it needs. */ + prompt: string; + }): Promise; + /** Delegate a task to a subagent that INHERITS this conversation: a child agent seeded with all completed turns so far (it does not see the current in-flight turn), returning only its final result. Use this when the subtask builds on this conversation's context — a follow-up analysis, a review, a continuation — without consuming this conversation's context for the work itself. You receive only its final answer, not its intermediate steps. */ + subagent_fork(args: { + /** A short (3-5 word) description of the delegated task, for display. */ + description: string; + /** The task for the subagent. It already sees this conversation's completed turns, so build on them freely and state only what is new. */ + prompt: string; + }): Promise; + /** Record and update a structured task list for the current work. Send the ENTIRE list every call — it REPLACES the previous list (there are no partial updates, no per-item edits). Use it to plan multi-step work and show progress: add one todo per concrete step before you start. Keep AT MOST ONE todo `in_progress` at a time; while work remains, exactly one active task should be `in_progress`. Mark a todo `completed` the moment it is done (do not batch completions), and allow no `in_progress` item only once all work is complete. Skip the list for trivial single-step tasks. Statuses: `pending` (not started), `in_progress` (being worked on now), `completed` (finished). */ + todo_write(args: { + /** The COMPLETE task list, replacing any previous list. */ + todos: ({ + /** What the task is — a short imperative line. */ + content: string; + /** pending (not started) | in_progress (now) | completed (done). */ + status: "pending" | "in_progress" | "completed"; + })[]; + }): Promise; + /** Run a JavaScript workflow script that orchestrates subagents at scale. Use this for work that fans out across many independent pieces — an audit over many files, a migration, multi-angle research, adversarial verification of findings — where you write the orchestration as a script instead of delegating turn by turn. The workflow's identity rides the `meta` parameter as JSON: required `name` (short kebab-case) and `description` strings, optional `whenToUse` string and `phases` array (`{title, detail?, model?}`). The `script` parameter is the plain JavaScript body ONLY (NOT TypeScript, and NO `export const meta` statement — meta is a parameter, not code), running with top-level await; end with `return ` — the value must be JSON-serializable and is this tool's result. Script-body hooks: - `agent(prompt, opts?): Promise` — run one subagent to completion. Without `opts.schema` it resolves to the child's final text; with `opts.schema` (an object-rooted JSON Schema using ONLY type/properties/required/additionalProperties/items/enum/const — no oneOf/pattern/format/numeric bounds) it resolves to the validated object. Resolves `null` when the child fails (filter with `.filter(Boolean)`). Other opts: `label` (display), `phase` (progress group), `model` (override). Anything else (`effort`/`isolation`/`agentType`) is rejected loudly. - `pipeline(items, ...stages): Promise` — run each item through the stages independently with NO barrier between stages (prefer this for multi-stage work). Each stage receives `(prev, item, index)`. An ordinary stage throw drops that ITEM to `null` and skips its remaining stages. - `parallel(thunks): Promise` — run zero-argument functions concurrently and await ALL of them (a barrier; use only when a stage genuinely needs every prior result together). A throwing thunk resolves to `null`. - `phase(title)` — start a progress phase; `log(message)` — narrate progress; `args` — the tool call's `args` input, verbatim. Misused hooks (bad arguments, unknown options, unsupported schemas, tripped caps) throw errors that ALWAYS kill the script — they never dissolve into a per-item `null`. Constraints: concurrency and total-agent caps apply; no filesystem, network, timers, or Node.js APIs are provided — the agents do the work, the script only coordinates them. The run executes in the foreground: this call returns when the whole script finishes. */ + workflow(args: { + /** The plain-JS workflow script body (top-level await allowed; NO `export const meta` statement; end with `return `). */ + script: string; + /** The workflow identity block (plain JSON — never code). */ + meta: { + /** Short kebab-case workflow name. */ + name: string; + /** One-line description of what the workflow does. */ + description: string; + /** Optional guidance on when this workflow applies. */ + whenToUse?: string; + /** Optional phase declarations matched by phase() calls. */ + phases?: { + /** The phase title phase() calls match by exact string. */ + title: string; + /** Optional one-line description of the phase. */ + detail?: string; + /** Optional model override this phase is expected to use. */ + model?: string; + }[]; + }; + /** Optional JSON input exposed to the script as the `args` global (wrap a bare list as a field, e.g. {"files": [...]}). */ + args?: Record; + }): Promise; + /** Create or fully replace a UTF-8 text file. */ + write(args: { + /** Path to write, resolved by the filesystem backend. */ + file_path: string; + /** Full UTF-8 text content to write. */ + content: string; + }): Promise; +} +``` diff --git a/examples/acp-agent/tests/snapshots/code-mode-turn/session.jsonl b/examples/acp-agent/tests/snapshots/code-mode-turn/session.jsonl index 72533bcdeb..9367e2deb0 100644 --- a/examples/acp-agent/tests/snapshots/code-mode-turn/session.jsonl +++ b/examples/acp-agent/tests/snapshots/code-mode-turn/session.jsonl @@ -2,7 +2,7 @@ {"type":"turn/start","seq":0,"time":1783611771394,"data":{"turn":1,"trigger":{"kind":"message","source":{"kind":"user"}}}} {"type":"user/message","seq":1,"time":1783611771394,"data":{"content":[{"type":"text","text":"Using ONE run_code program: call the bash tool twice — exactly `echo CODE_ONE` then exactly `echo CODE_TWO` — and return the two outputs joined with a plus sign. Then reply with that joined string only and stop."}],"source":{"kind":"user"}},"surfaceOp":"append"} {"type":"step/start","seq":2,"time":1783611771396,"data":{"turn":1,"step":1}} -{"type":"request/header","seq":3,"time":1783611771396,"data":{"header":{"config":{"model":"deepseek-v4-flash"},"system":"You are an AI agent powered by the DeepSeek Harness SDK.\n\nYou are a coding assistant powered by the deepseek-v4-flash model. Your working\ndirectory is /var/folders/2c/psb0_fmx7hbgz558xjt_f0l00000gn/T/acp-snap-cwd-BteTVR.\n\nVerify your work by running the code or tests. Keep answers brief and\nfactual.\n\n\nUse the read tool — not shell commands like cat — to inspect text files. Results include line numbers. Use offset and limit to continue reading large files.\n\nUse the write tool to create files or completely replace file contents. Existing files are overwritten, so read an existing file first (the default fs-policy requires it) and prefer edit for targeted changes.\n\nUse the edit tool for targeted changes to existing UTF-8 text files. It replaces literal old_string with new_string; by default old_string must appear exactly once. If old_string appears multiple times, provide a more specific old_string or set replace_all to true. Read the file first (the default fs-policy requires it), unless you just created or edited it in this session.\n\nCheck the [exit code: N] marker on every bash result; investigate failures before moving on.\n\nUse the workflow tool ONLY when the user explicitly asks for a workflow or for large multi-agent orchestration: you write a JavaScript script (the tool description documents the exact format) that fans work out across many subagents with phases and structured results. For one or two delegations, prefer plain subagent calls.\n\n## Writing code for run_code\n\nPass `run_code` the body of an async TypeScript function (erasable syntax only — no `enum` or namespaces; type annotations are advisory, the code runs type-stripped). Inside the program:\n\n- Call tools as `await tools.name(args)` — quoted access for exotic names: `tools[\"my-tool\"](args)`. Every call resolves to the tool's text output as a string. Tool arguments must be JSON-serializable.\n- A FAILED tool call rejects with an `Error` carrying the tool's error text — `try/catch` it to handle and continue.\n- Calls execute sequentially, even under `Promise.all`.\n- Emit results with `return` and/or `console.log(...)`. ONLY what you print or return comes back to you — intermediate tool results never enter the conversation, so extract just what you need.\n\nThe available tools:\n\n```ts\ndeclare const tools: {\n /** Execute a bash command (`bash -c`) and return its stdout/stderr. Each call runs in a fresh shell: no state (cwd, variables, functions) persists between calls — pass `workdir` instead of using `cd`. Non-zero exits are reported as `[exit code: N]`. Long output is truncated to its tail; the full output is saved to a file whose path is reported when available. Set `run_in_background: true` for long-running commands: the call returns a task id immediately; poll it with `bash_output` and stop it with `bash_kill`. */\n bash(args: {\n /** The bash command to execute. */\n command: string;\n /** Clear, concise description of what this command does in active voice, 5-10 words (shown in the UI). Examples: \"ls\" → \"List files in current directory\"; \"git status\" → \"Show working tree status\"; \"npm install\" → \"Install package dependencies\". */\n description: string;\n /** Timeout in milliseconds. The executor applies its configured default and cap, and kills the command on expiry. */\n timeoutMs?: number;\n /** Working directory for this command. Defaults to the session workspace; a relative path is resolved against it. */\n workdir?: string;\n /** Run in the background and return a task id immediately. No timeout applies. */\n run_in_background?: boolean;\n }): Promise;\n /** Ask the executor to kill a running background bash task by task id. */\n bash_kill(args: {\n /** Task id returned by the bash tool. */\n task_id: string;\n }): Promise;\n /** Read new output from a background bash task started with `bash` + `run_in_background`. Returns only output produced since the previous bash_output call, plus the task status. Tasks keep running while you do other work; poll again later for more output. */\n bash_output(args: {\n /** Task id returned by the bash tool. */\n task_id: string;\n }): Promise;\n /** Edit an existing UTF-8 text file by replacing literal text. */\n edit(args: {\n /** Path to edit, resolved by the filesystem backend. */\n file_path: string;\n /** Literal text to replace. Must match exactly. */\n old_string: string;\n /** Literal replacement text. Use an empty string to delete the match. */\n new_string: string;\n /** Replace all matches. Defaults to false; when false, old_string must appear exactly once. */\n replace_all?: boolean;\n }): Promise;\n /** Read a UTF-8 text file and return line-numbered content. */\n read(args: {\n /** Path to read, resolved by the filesystem backend. */\n file_path: string;\n /** 1-based first line to return. Defaults to 1. */\n offset?: number;\n /** Maximum number of lines to return. Defaults to 2000. */\n limit?: number;\n }): Promise;\n /** Delegate a self-contained task to a subagent (a separate agent that works in its own context) and return its final result. Use this to offload focused, independent work — research, a scoped implementation, an analysis — so it does not consume this conversation's context. The subagent runs to completion and you receive only its final answer, not its intermediate steps. Give it a complete, standalone prompt: it does not see this conversation. */\n subagent(args: {\n /** A short (3-5 word) description of the delegated task, for display. */\n description: string;\n /** The complete, self-contained task for the subagent. It does not share this conversation's context, so include everything it needs. */\n prompt: string;\n }): Promise;\n /** Delegate a task to a subagent that INHERITS this conversation: a child agent seeded with all completed turns so far (it does not see the current in-flight turn), returning only its final result. Use this when the subtask builds on this conversation's context — a follow-up analysis, a review, a continuation — without consuming this conversation's context for the work itself. You receive only its final answer, not its intermediate steps. */\n subagent_fork(args: {\n /** A short (3-5 word) description of the delegated task, for display. */\n description: string;\n /** The task for the subagent. It already sees this conversation's completed turns, so build on them freely and state only what is new. */\n prompt: string;\n }): Promise;\n /** Record and update a structured task list for the current work. Send the ENTIRE list every call — it REPLACES the previous list (there are no partial updates, no per-item edits). Use it to plan multi-step work and show progress: add one todo per concrete step before you start. Keep AT MOST ONE todo `in_progress` at a time; while work remains, exactly one active task should be `in_progress`. Mark a todo `completed` the moment it is done (do not batch completions), and allow no `in_progress` item only once all work is complete. Skip the list for trivial single-step tasks. Statuses: `pending` (not started), `in_progress` (being worked on now), `completed` (finished). */\n todo_write(args: {\n /** The COMPLETE task list, replacing any previous list. */\n todos: ({\n /** What the task is — a short imperative line. */\n content: string;\n /** pending (not started) | in_progress (now) | completed (done). */\n status: \"pending\" | \"in_progress\" | \"completed\";\n })[];\n }): Promise;\n /** Run a JavaScript workflow script that orchestrates subagents at scale. Use this for work that fans out across many independent pieces — an audit over many files, a migration, multi-angle research, adversarial verification of findings — where you write the orchestration as a script instead of delegating turn by turn. The workflow's identity rides the `meta` parameter as JSON: required `name` (short kebab-case) and `description` strings, optional `whenToUse` string and `phases` array (`{title, detail?, model?}`). The `script` parameter is the plain JavaScript body ONLY (NOT TypeScript, and NO `export const meta` statement — meta is a parameter, not code), running with top-level await; end with `return ` — the value must be JSON-serializable and is this tool's result. Script-body hooks: - `agent(prompt, opts?): Promise` — run one subagent to completion. Without `opts.schema` it resolves to the child's final text; with `opts.schema` (an object-rooted JSON Schema using ONLY type/properties/required/additionalProperties/items/enum/const — no oneOf/pattern/format/numeric bounds) it resolves to the validated object. Resolves `null` when the child fails (filter with `.filter(Boolean)`). Other opts: `label` (display), `phase` (progress group), `model` (override). Anything else (`effort`/`isolation`/`agentType`) is rejected loudly. - `pipeline(items, ...stages): Promise` — run each item through the stages independently with NO barrier between stages (prefer this for multi-stage work). Each stage receives `(prev, item, index)`. An ordinary stage throw drops that ITEM to `null` and skips its remaining stages. - `parallel(thunks): Promise` — run zero-argument functions concurrently and await ALL of them (a barrier; use only when a stage genuinely needs every prior result together). A throwing thunk resolves to `null`. - `phase(title)` — start a progress phase; `log(message)` — narrate progress; `args` — the tool call's `args` input, verbatim. Misused hooks (bad arguments, unknown options, unsupported schemas, tripped caps) throw errors that ALWAYS kill the script — they never dissolve into a per-item `null`. Constraints: concurrency and total-agent caps apply; no filesystem, network, timers, or Node.js APIs are provided — the agents do the work, the script only coordinates them. The run executes in the foreground: this call returns when the whole script finishes. */\n workflow(args: {\n /** The plain-JS workflow script body (top-level await allowed; NO `export const meta` statement; end with `return `). */\n script: string;\n /** The workflow identity block (plain JSON — never code). */\n meta: {\n /** Short kebab-case workflow name. */\n name: string;\n /** One-line description of what the workflow does. */\n description: string;\n /** Optional guidance on when this workflow applies. */\n whenToUse?: string;\n /** Optional phase declarations matched by phase() calls. */\n phases?: {\n /** The phase title phase() calls match by exact string. */\n title: string;\n /** Optional one-line description of the phase. */\n detail?: string;\n /** Optional model override this phase is expected to use. */\n model?: string;\n }[];\n };\n /** Optional JSON input exposed to the script as the `args` global (wrap a bare list as a field, e.g. {\"files\": [...]}). */\n args?: Record;\n }): Promise;\n /** Create or fully replace a UTF-8 text file. */\n write(args: {\n /** Path to write, resolved by the filesystem backend. */\n file_path: string;\n /** Full UTF-8 text content to write. */\n content: string;\n }): Promise;\n}\n```","tools":[{"name":"run_code","description":"Execute a TypeScript program against the available tools. Write the BODY of an async function (erasable syntax only; top-level `await` and `return` work) and call tools as `await tools.name(args)` per the declarations in the system prompt. Only what you print or return comes back — curate it.","parameters":{"type":"object","properties":{"code":{"type":"string","description":"The program: the body of an async TypeScript function."}},"required":["code"]}}]},"reason":"initial"}} +{"type":"request/header","seq":3,"time":1783611771396,"data":{"header":{"config":{"model":"deepseek-v4-flash"},"system":"{{system}}","tools":[{"name":"run_code","description":"Execute a TypeScript program against the available tools. Write the BODY of an async function (erasable syntax only; top-level `await` and `return` work) and call tools as `await tools.name(args)` per the declarations in the system prompt. Only what you print or return comes back — curate it.","parameters":{"type":"object","properties":{"code":{"type":"string","description":"The program: the body of an async TypeScript function."}},"required":["code"]}}]},"reason":"initial"}} {"type":"assistant/chunk","seq":4,"time":1783611771869,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"reasoning"}}} {"type":"assistant/chunk","seq":5,"time":1783611771869,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"The"}}} {"type":"assistant/chunk","seq":6,"time":1783611771978,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" user"}}} diff --git a/examples/acp-agent/tests/snapshots/code-mode-turn/system-prompt.golden.md b/examples/acp-agent/tests/snapshots/code-mode-turn/system-prompt.golden.md new file mode 100644 index 0000000000..688557b132 --- /dev/null +++ b/examples/acp-agent/tests/snapshots/code-mode-turn/system-prompt.golden.md @@ -0,0 +1,133 @@ +You are an AI agent powered by the DeepSeek Harness SDK. + +You are a coding assistant powered by the deepseek-v4-flash model. Your working +directory is {{cwd}}. + +Verify your work by running the code or tests. Keep answers brief and +factual. + + +Use the read tool — not shell commands like cat — to inspect text files. Results include line numbers. Use offset and limit to continue reading large files. + +Use the write tool to create files or completely replace file contents. Existing files are overwritten, so read an existing file first (the default fs-policy requires it) and prefer edit for targeted changes. + +Use the edit tool for targeted changes to existing UTF-8 text files. It replaces literal old_string with new_string; by default old_string must appear exactly once. If old_string appears multiple times, provide a more specific old_string or set replace_all to true. Read the file first (the default fs-policy requires it), unless you just created or edited it in this session. + +Check the [exit code: N] marker on every bash result; investigate failures before moving on. + +Use the workflow tool ONLY when the user explicitly asks for a workflow or for large multi-agent orchestration: you write a JavaScript script (the tool description documents the exact format) that fans work out across many subagents with phases and structured results. For one or two delegations, prefer plain subagent calls. + +## Writing code for run_code + +Pass `run_code` the body of an async TypeScript function (erasable syntax only — no `enum` or namespaces; type annotations are advisory, the code runs type-stripped). Inside the program: + +- Call tools as `await tools.name(args)` — quoted access for exotic names: `tools["my-tool"](args)`. Every call resolves to the tool's text output as a string. Tool arguments must be JSON-serializable. +- A FAILED tool call rejects with an `Error` carrying the tool's error text — `try/catch` it to handle and continue. +- Calls execute sequentially, even under `Promise.all`. +- Emit results with `return` and/or `console.log(...)`. ONLY what you print or return comes back to you — intermediate tool results never enter the conversation, so extract just what you need. + +The available tools: + +```ts +declare const tools: { + /** Execute a bash command (`bash -c`) and return its stdout/stderr. Each call runs in a fresh shell: no state (cwd, variables, functions) persists between calls — pass `workdir` instead of using `cd`. Non-zero exits are reported as `[exit code: N]`. Long output is truncated to its tail; the full output is saved to a file whose path is reported when available. Set `run_in_background: true` for long-running commands: the call returns a task id immediately; poll it with `bash_output` and stop it with `bash_kill`. */ + bash(args: { + /** The bash command to execute. */ + command: string; + /** Clear, concise description of what this command does in active voice, 5-10 words (shown in the UI). Examples: "ls" → "List files in current directory"; "git status" → "Show working tree status"; "npm install" → "Install package dependencies". */ + description: string; + /** Timeout in milliseconds. The executor applies its configured default and cap, and kills the command on expiry. */ + timeoutMs?: number; + /** Working directory for this command. Defaults to the session workspace; a relative path is resolved against it. */ + workdir?: string; + /** Run in the background and return a task id immediately. No timeout applies. */ + run_in_background?: boolean; + }): Promise; + /** Ask the executor to kill a running background bash task by task id. */ + bash_kill(args: { + /** Task id returned by the bash tool. */ + task_id: string; + }): Promise; + /** Read new output from a background bash task started with `bash` + `run_in_background`. Returns only output produced since the previous bash_output call, plus the task status. Tasks keep running while you do other work; poll again later for more output. */ + bash_output(args: { + /** Task id returned by the bash tool. */ + task_id: string; + }): Promise; + /** Edit an existing UTF-8 text file by replacing literal text. */ + edit(args: { + /** Path to edit, resolved by the filesystem backend. */ + file_path: string; + /** Literal text to replace. Must match exactly. */ + old_string: string; + /** Literal replacement text. Use an empty string to delete the match. */ + new_string: string; + /** Replace all matches. Defaults to false; when false, old_string must appear exactly once. */ + replace_all?: boolean; + }): Promise; + /** Read a UTF-8 text file and return line-numbered content. */ + read(args: { + /** Path to read, resolved by the filesystem backend. */ + file_path: string; + /** 1-based first line to return. Defaults to 1. */ + offset?: number; + /** Maximum number of lines to return. Defaults to 2000. */ + limit?: number; + }): Promise; + /** Delegate a self-contained task to a subagent (a separate agent that works in its own context) and return its final result. Use this to offload focused, independent work — research, a scoped implementation, an analysis — so it does not consume this conversation's context. The subagent runs to completion and you receive only its final answer, not its intermediate steps. Give it a complete, standalone prompt: it does not see this conversation. */ + subagent(args: { + /** A short (3-5 word) description of the delegated task, for display. */ + description: string; + /** The complete, self-contained task for the subagent. It does not share this conversation's context, so include everything it needs. */ + prompt: string; + }): Promise; + /** Delegate a task to a subagent that INHERITS this conversation: a child agent seeded with all completed turns so far (it does not see the current in-flight turn), returning only its final result. Use this when the subtask builds on this conversation's context — a follow-up analysis, a review, a continuation — without consuming this conversation's context for the work itself. You receive only its final answer, not its intermediate steps. */ + subagent_fork(args: { + /** A short (3-5 word) description of the delegated task, for display. */ + description: string; + /** The task for the subagent. It already sees this conversation's completed turns, so build on them freely and state only what is new. */ + prompt: string; + }): Promise; + /** Record and update a structured task list for the current work. Send the ENTIRE list every call — it REPLACES the previous list (there are no partial updates, no per-item edits). Use it to plan multi-step work and show progress: add one todo per concrete step before you start. Keep AT MOST ONE todo `in_progress` at a time; while work remains, exactly one active task should be `in_progress`. Mark a todo `completed` the moment it is done (do not batch completions), and allow no `in_progress` item only once all work is complete. Skip the list for trivial single-step tasks. Statuses: `pending` (not started), `in_progress` (being worked on now), `completed` (finished). */ + todo_write(args: { + /** The COMPLETE task list, replacing any previous list. */ + todos: ({ + /** What the task is — a short imperative line. */ + content: string; + /** pending (not started) | in_progress (now) | completed (done). */ + status: "pending" | "in_progress" | "completed"; + })[]; + }): Promise; + /** Run a JavaScript workflow script that orchestrates subagents at scale. Use this for work that fans out across many independent pieces — an audit over many files, a migration, multi-angle research, adversarial verification of findings — where you write the orchestration as a script instead of delegating turn by turn. The workflow's identity rides the `meta` parameter as JSON: required `name` (short kebab-case) and `description` strings, optional `whenToUse` string and `phases` array (`{title, detail?, model?}`). The `script` parameter is the plain JavaScript body ONLY (NOT TypeScript, and NO `export const meta` statement — meta is a parameter, not code), running with top-level await; end with `return ` — the value must be JSON-serializable and is this tool's result. Script-body hooks: - `agent(prompt, opts?): Promise` — run one subagent to completion. Without `opts.schema` it resolves to the child's final text; with `opts.schema` (an object-rooted JSON Schema using ONLY type/properties/required/additionalProperties/items/enum/const — no oneOf/pattern/format/numeric bounds) it resolves to the validated object. Resolves `null` when the child fails (filter with `.filter(Boolean)`). Other opts: `label` (display), `phase` (progress group), `model` (override). Anything else (`effort`/`isolation`/`agentType`) is rejected loudly. - `pipeline(items, ...stages): Promise` — run each item through the stages independently with NO barrier between stages (prefer this for multi-stage work). Each stage receives `(prev, item, index)`. An ordinary stage throw drops that ITEM to `null` and skips its remaining stages. - `parallel(thunks): Promise` — run zero-argument functions concurrently and await ALL of them (a barrier; use only when a stage genuinely needs every prior result together). A throwing thunk resolves to `null`. - `phase(title)` — start a progress phase; `log(message)` — narrate progress; `args` — the tool call's `args` input, verbatim. Misused hooks (bad arguments, unknown options, unsupported schemas, tripped caps) throw errors that ALWAYS kill the script — they never dissolve into a per-item `null`. Constraints: concurrency and total-agent caps apply; no filesystem, network, timers, or Node.js APIs are provided — the agents do the work, the script only coordinates them. The run executes in the foreground: this call returns when the whole script finishes. */ + workflow(args: { + /** The plain-JS workflow script body (top-level await allowed; NO `export const meta` statement; end with `return `). */ + script: string; + /** The workflow identity block (plain JSON — never code). */ + meta: { + /** Short kebab-case workflow name. */ + name: string; + /** One-line description of what the workflow does. */ + description: string; + /** Optional guidance on when this workflow applies. */ + whenToUse?: string; + /** Optional phase declarations matched by phase() calls. */ + phases?: { + /** The phase title phase() calls match by exact string. */ + title: string; + /** Optional one-line description of the phase. */ + detail?: string; + /** Optional model override this phase is expected to use. */ + model?: string; + }[]; + }; + /** Optional JSON input exposed to the script as the `args` global (wrap a bare list as a field, e.g. {"files": [...]}). */ + args?: Record; + }): Promise; + /** Create or fully replace a UTF-8 text file. */ + write(args: { + /** Path to write, resolved by the filesystem backend. */ + file_path: string; + /** Full UTF-8 text content to write. */ + content: string; + }): Promise; +} +``` diff --git a/examples/acp-agent/tests/snapshots/text-turn/session.jsonl b/examples/acp-agent/tests/snapshots/text-turn/session.jsonl index 8475c97896..3fcba5fd1c 100644 --- a/examples/acp-agent/tests/snapshots/text-turn/session.jsonl +++ b/examples/acp-agent/tests/snapshots/text-turn/session.jsonl @@ -2,7 +2,7 @@ {"type":"turn/start","seq":0,"time":1783600629541,"data":{"turn":1,"trigger":{"kind":"message","source":{"kind":"user"}}}} {"type":"user/message","seq":1,"time":1783600629541,"data":{"content":[{"type":"text","text":"Reply with exactly the word: PONG. Do not use any tools."}],"source":{"kind":"user"}},"surfaceOp":"append"} {"type":"step/start","seq":2,"time":1783600629542,"data":{"turn":1,"step":1}} -{"type":"request/header","seq":3,"time":1783600629542,"data":{"header":{"config":{"model":"deepseek-v4-flash"},"system":"You are an AI agent powered by the DeepSeek Harness SDK.\n\nYou are a coding assistant powered by the deepseek-v4-flash model. Your working\ndirectory is /var/folders/bn/vj1dvck95yd5jh3x4wskflxm0000gn/T/acp-snap-cwd-ka5r8w.\n\nVerify your work by running the code or tests. Keep answers brief and\nfactual.\n\n\nUse the read tool — not shell commands like cat — to inspect text files. Results include line numbers. Use offset and limit to continue reading large files.\n\nUse the write tool to create files or completely replace file contents. Existing files are overwritten, so read an existing file first (the default fs-policy requires it) and prefer edit for targeted changes.\n\nUse the edit tool for targeted changes to existing UTF-8 text files. It replaces literal old_string with new_string; by default old_string must appear exactly once. If old_string appears multiple times, provide a more specific old_string or set replace_all to true. Read the file first (the default fs-policy requires it), unless you just created or edited it in this session.\n\nCheck the [exit code: N] marker on every bash result; investigate failures before moving on.\n\nUse the workflow tool ONLY when the user explicitly asks for a workflow or for large multi-agent orchestration: you write a JavaScript script (the tool description documents the exact format) that fans work out across many subagents with phases and structured results. For one or two delegations, prefer plain subagent calls.","tools":[{"name":"bash","description":"Execute a bash command (`bash -c`) and return its stdout/stderr. Each call runs in a fresh shell: no state (cwd, variables, functions) persists between calls — pass `workdir` instead of using `cd`. Non-zero exits are reported as `[exit code: N]`. Long output is truncated to its tail; the full output is saved to a file whose path is reported when available. Set `run_in_background: true` for long-running commands: the call returns a task id immediately; poll it with `bash_output` and stop it with `bash_kill`.","parameters":{"type":"object","properties":{"command":{"type":"string","description":"The bash command to execute."},"description":{"type":"string","description":"Clear, concise description of what this command does in active voice, 5-10 words (shown in the UI). Examples: \"ls\" → \"List files in current directory\"; \"git status\" → \"Show working tree status\"; \"npm install\" → \"Install package dependencies\"."},"timeoutMs":{"type":"number","description":"Timeout in milliseconds. The executor applies its configured default and cap, and kills the command on expiry."},"workdir":{"type":"string","description":"Working directory for this command. Defaults to the session workspace; a relative path is resolved against it."},"run_in_background":{"type":"boolean","description":"Run in the background and return a task id immediately. No timeout applies."}},"required":["command","description"]}},{"name":"bash_kill","description":"Ask the executor to kill a running background bash task by task id.","parameters":{"type":"object","properties":{"task_id":{"type":"string","description":"Task id returned by the bash tool."}},"required":["task_id"]}},{"name":"bash_output","description":"Read new output from a background bash task started with `bash` + `run_in_background`. Returns only output produced since the previous bash_output call, plus the task status. Tasks keep running while you do other work; poll again later for more output.","parameters":{"type":"object","properties":{"task_id":{"type":"string","description":"Task id returned by the bash tool."}},"required":["task_id"]}},{"name":"edit","description":"Edit an existing UTF-8 text file by replacing literal text.","parameters":{"type":"object","properties":{"file_path":{"type":"string","description":"Path to edit, resolved by the filesystem backend."},"old_string":{"type":"string","description":"Literal text to replace. Must match exactly."},"new_string":{"type":"string","description":"Literal replacement text. Use an empty string to delete the match."},"replace_all":{"type":"boolean","description":"Replace all matches. Defaults to false; when false, old_string must appear exactly once."}},"required":["file_path","old_string","new_string"]}},{"name":"read","description":"Read a UTF-8 text file and return line-numbered content.","parameters":{"type":"object","properties":{"file_path":{"type":"string","description":"Path to read, resolved by the filesystem backend."},"offset":{"type":"number","description":"1-based first line to return. Defaults to 1."},"limit":{"type":"number","description":"Maximum number of lines to return. Defaults to 2000."}},"required":["file_path"]}},{"name":"subagent","description":"Delegate a self-contained task to a subagent (a separate agent that works in its own context) and return its final result. Use this to offload focused, independent work — research, a scoped implementation, an analysis — so it does not consume this conversation's context. The subagent runs to completion and you receive only its final answer, not its intermediate steps. Give it a complete, standalone prompt: it does not see this conversation.","parameters":{"type":"object","properties":{"description":{"type":"string","description":"A short (3-5 word) description of the delegated task, for display."},"prompt":{"type":"string","description":"The complete, self-contained task for the subagent. It does not share this conversation's context, so include everything it needs."}},"required":["description","prompt"]}},{"name":"subagent_fork","description":"Delegate a task to a subagent that INHERITS this conversation: a child agent seeded with all completed turns so far (it does not see the current in-flight turn), returning only its final result. Use this when the subtask builds on this conversation's context — a follow-up analysis, a review, a continuation — without consuming this conversation's context for the work itself. You receive only its final answer, not its intermediate steps.","parameters":{"type":"object","properties":{"description":{"type":"string","description":"A short (3-5 word) description of the delegated task, for display."},"prompt":{"type":"string","description":"The task for the subagent. It already sees this conversation's completed turns, so build on them freely and state only what is new."}},"required":["description","prompt"]}},{"name":"todo_write","description":"Record and update a structured task list for the current work. Send the ENTIRE list every call — it REPLACES the previous list (there are no partial updates, no per-item edits). Use it to plan multi-step work and show progress: add one todo per concrete step before you start. Keep AT MOST ONE todo `in_progress` at a time; while work remains, exactly one active task should be `in_progress`. Mark a todo `completed` the moment it is done (do not batch completions), and allow no `in_progress` item only once all work is complete. Skip the list for trivial single-step tasks. Statuses: `pending` (not started), `in_progress` (being worked on now), `completed` (finished).","parameters":{"type":"object","properties":{"todos":{"type":"array","description":"The COMPLETE task list, replacing any previous list.","items":{"type":"object","properties":{"content":{"type":"string","description":"What the task is — a short imperative line."},"status":{"type":"string","description":"pending (not started) | in_progress (now) | completed (done).","enum":["pending","in_progress","completed"]}},"required":["content","status"]}}},"required":["todos"]}},{"name":"workflow","description":"Run a JavaScript workflow script that orchestrates subagents at scale. Use this for work that fans out across many independent pieces — an audit over many files, a migration, multi-angle research, adversarial verification of findings — where you write the orchestration as a script instead of delegating turn by turn.\n\nThe workflow's identity rides the `meta` parameter as JSON: required `name` (short kebab-case) and `description` strings, optional `whenToUse` string and `phases` array (`{title, detail?, model?}`). The `script` parameter is the plain JavaScript body ONLY (NOT TypeScript, and NO `export const meta` statement — meta is a parameter, not code), running with top-level await; end with `return ` — the value must be JSON-serializable and is this tool's result.\n\nScript-body hooks:\n- `agent(prompt, opts?): Promise` — run one subagent to completion. Without `opts.schema` it resolves to the child's final text; with `opts.schema` (an object-rooted JSON Schema using ONLY type/properties/required/additionalProperties/items/enum/const — no oneOf/pattern/format/numeric bounds) it resolves to the validated object. Resolves `null` when the child fails (filter with `.filter(Boolean)`). Other opts: `label` (display), `phase` (progress group), `model` (override). Anything else (`effort`/`isolation`/`agentType`) is rejected loudly.\n- `pipeline(items, ...stages): Promise` — run each item through the stages independently with NO barrier between stages (prefer this for multi-stage work). Each stage receives `(prev, item, index)`. An ordinary stage throw drops that ITEM to `null` and skips its remaining stages.\n- `parallel(thunks): Promise` — run zero-argument functions concurrently and await ALL of them (a barrier; use only when a stage genuinely needs every prior result together). A throwing thunk resolves to `null`.\n- `phase(title)` — start a progress phase; `log(message)` — narrate progress; `args` — the tool call's `args` input, verbatim.\n\nMisused hooks (bad arguments, unknown options, unsupported schemas, tripped caps) throw errors that ALWAYS kill the script — they never dissolve into a per-item `null`.\n\nConstraints: concurrency and total-agent caps apply; no filesystem, network, timers, or Node.js APIs are provided — the agents do the work, the script only coordinates them. The run executes in the foreground: this call returns when the whole script finishes.","parameters":{"type":"object","properties":{"script":{"type":"string","description":"The plain-JS workflow script body (top-level await allowed; NO `export const meta` statement; end with `return `)."},"meta":{"type":"object","description":"The workflow identity block (plain JSON — never code).","properties":{"name":{"type":"string","description":"Short kebab-case workflow name."},"description":{"type":"string","description":"One-line description of what the workflow does."},"whenToUse":{"type":"string","description":"Optional guidance on when this workflow applies."},"phases":{"type":"array","description":"Optional phase declarations matched by phase() calls.","items":{"type":"object","properties":{"title":{"type":"string","description":"The phase title phase() calls match by exact string."},"detail":{"type":"string","description":"Optional one-line description of the phase."},"model":{"type":"string","description":"Optional model override this phase is expected to use."}},"required":["title"]}}},"required":["name","description"]},"args":{"type":"object","description":"Optional JSON input exposed to the script as the `args` global (wrap a bare list as a field, e.g. {\"files\": [...]})."}},"required":["script","meta"]}},{"name":"write","description":"Create or fully replace a UTF-8 text file.","parameters":{"type":"object","properties":{"file_path":{"type":"string","description":"Path to write, resolved by the filesystem backend."},"content":{"type":"string","description":"Full UTF-8 text content to write."}},"required":["file_path","content"]}}]},"reason":"initial"}} +{"type":"request/header","seq":3,"time":1783600629542,"data":{"header":{"config":{"model":"deepseek-v4-flash"},"system":"{{system}}","tools":[{"name":"bash","description":"Execute a bash command (`bash -c`) and return its stdout/stderr. Each call runs in a fresh shell: no state (cwd, variables, functions) persists between calls — pass `workdir` instead of using `cd`. Non-zero exits are reported as `[exit code: N]`. Long output is truncated to its tail; the full output is saved to a file whose path is reported when available. Set `run_in_background: true` for long-running commands: the call returns a task id immediately; poll it with `bash_output` and stop it with `bash_kill`.","parameters":{"type":"object","properties":{"command":{"type":"string","description":"The bash command to execute."},"description":{"type":"string","description":"Clear, concise description of what this command does in active voice, 5-10 words (shown in the UI). Examples: \"ls\" → \"List files in current directory\"; \"git status\" → \"Show working tree status\"; \"npm install\" → \"Install package dependencies\"."},"timeoutMs":{"type":"number","description":"Timeout in milliseconds. The executor applies its configured default and cap, and kills the command on expiry."},"workdir":{"type":"string","description":"Working directory for this command. Defaults to the session workspace; a relative path is resolved against it."},"run_in_background":{"type":"boolean","description":"Run in the background and return a task id immediately. No timeout applies."}},"required":["command","description"]}},{"name":"bash_kill","description":"Ask the executor to kill a running background bash task by task id.","parameters":{"type":"object","properties":{"task_id":{"type":"string","description":"Task id returned by the bash tool."}},"required":["task_id"]}},{"name":"bash_output","description":"Read new output from a background bash task started with `bash` + `run_in_background`. Returns only output produced since the previous bash_output call, plus the task status. Tasks keep running while you do other work; poll again later for more output.","parameters":{"type":"object","properties":{"task_id":{"type":"string","description":"Task id returned by the bash tool."}},"required":["task_id"]}},{"name":"edit","description":"Edit an existing UTF-8 text file by replacing literal text.","parameters":{"type":"object","properties":{"file_path":{"type":"string","description":"Path to edit, resolved by the filesystem backend."},"old_string":{"type":"string","description":"Literal text to replace. Must match exactly."},"new_string":{"type":"string","description":"Literal replacement text. Use an empty string to delete the match."},"replace_all":{"type":"boolean","description":"Replace all matches. Defaults to false; when false, old_string must appear exactly once."}},"required":["file_path","old_string","new_string"]}},{"name":"read","description":"Read a UTF-8 text file and return line-numbered content.","parameters":{"type":"object","properties":{"file_path":{"type":"string","description":"Path to read, resolved by the filesystem backend."},"offset":{"type":"number","description":"1-based first line to return. Defaults to 1."},"limit":{"type":"number","description":"Maximum number of lines to return. Defaults to 2000."}},"required":["file_path"]}},{"name":"subagent","description":"Delegate a self-contained task to a subagent (a separate agent that works in its own context) and return its final result. Use this to offload focused, independent work — research, a scoped implementation, an analysis — so it does not consume this conversation's context. The subagent runs to completion and you receive only its final answer, not its intermediate steps. Give it a complete, standalone prompt: it does not see this conversation.","parameters":{"type":"object","properties":{"description":{"type":"string","description":"A short (3-5 word) description of the delegated task, for display."},"prompt":{"type":"string","description":"The complete, self-contained task for the subagent. It does not share this conversation's context, so include everything it needs."}},"required":["description","prompt"]}},{"name":"subagent_fork","description":"Delegate a task to a subagent that INHERITS this conversation: a child agent seeded with all completed turns so far (it does not see the current in-flight turn), returning only its final result. Use this when the subtask builds on this conversation's context — a follow-up analysis, a review, a continuation — without consuming this conversation's context for the work itself. You receive only its final answer, not its intermediate steps.","parameters":{"type":"object","properties":{"description":{"type":"string","description":"A short (3-5 word) description of the delegated task, for display."},"prompt":{"type":"string","description":"The task for the subagent. It already sees this conversation's completed turns, so build on them freely and state only what is new."}},"required":["description","prompt"]}},{"name":"todo_write","description":"Record and update a structured task list for the current work. Send the ENTIRE list every call — it REPLACES the previous list (there are no partial updates, no per-item edits). Use it to plan multi-step work and show progress: add one todo per concrete step before you start. Keep AT MOST ONE todo `in_progress` at a time; while work remains, exactly one active task should be `in_progress`. Mark a todo `completed` the moment it is done (do not batch completions), and allow no `in_progress` item only once all work is complete. Skip the list for trivial single-step tasks. Statuses: `pending` (not started), `in_progress` (being worked on now), `completed` (finished).","parameters":{"type":"object","properties":{"todos":{"type":"array","description":"The COMPLETE task list, replacing any previous list.","items":{"type":"object","properties":{"content":{"type":"string","description":"What the task is — a short imperative line."},"status":{"type":"string","description":"pending (not started) | in_progress (now) | completed (done).","enum":["pending","in_progress","completed"]}},"required":["content","status"]}}},"required":["todos"]}},{"name":"workflow","description":"Run a JavaScript workflow script that orchestrates subagents at scale. Use this for work that fans out across many independent pieces — an audit over many files, a migration, multi-angle research, adversarial verification of findings — where you write the orchestration as a script instead of delegating turn by turn.\n\nThe workflow's identity rides the `meta` parameter as JSON: required `name` (short kebab-case) and `description` strings, optional `whenToUse` string and `phases` array (`{title, detail?, model?}`). The `script` parameter is the plain JavaScript body ONLY (NOT TypeScript, and NO `export const meta` statement — meta is a parameter, not code), running with top-level await; end with `return ` — the value must be JSON-serializable and is this tool's result.\n\nScript-body hooks:\n- `agent(prompt, opts?): Promise` — run one subagent to completion. Without `opts.schema` it resolves to the child's final text; with `opts.schema` (an object-rooted JSON Schema using ONLY type/properties/required/additionalProperties/items/enum/const — no oneOf/pattern/format/numeric bounds) it resolves to the validated object. Resolves `null` when the child fails (filter with `.filter(Boolean)`). Other opts: `label` (display), `phase` (progress group), `model` (override). Anything else (`effort`/`isolation`/`agentType`) is rejected loudly.\n- `pipeline(items, ...stages): Promise` — run each item through the stages independently with NO barrier between stages (prefer this for multi-stage work). Each stage receives `(prev, item, index)`. An ordinary stage throw drops that ITEM to `null` and skips its remaining stages.\n- `parallel(thunks): Promise` — run zero-argument functions concurrently and await ALL of them (a barrier; use only when a stage genuinely needs every prior result together). A throwing thunk resolves to `null`.\n- `phase(title)` — start a progress phase; `log(message)` — narrate progress; `args` — the tool call's `args` input, verbatim.\n\nMisused hooks (bad arguments, unknown options, unsupported schemas, tripped caps) throw errors that ALWAYS kill the script — they never dissolve into a per-item `null`.\n\nConstraints: concurrency and total-agent caps apply; no filesystem, network, timers, or Node.js APIs are provided — the agents do the work, the script only coordinates them. The run executes in the foreground: this call returns when the whole script finishes.","parameters":{"type":"object","properties":{"script":{"type":"string","description":"The plain-JS workflow script body (top-level await allowed; NO `export const meta` statement; end with `return `)."},"meta":{"type":"object","description":"The workflow identity block (plain JSON — never code).","properties":{"name":{"type":"string","description":"Short kebab-case workflow name."},"description":{"type":"string","description":"One-line description of what the workflow does."},"whenToUse":{"type":"string","description":"Optional guidance on when this workflow applies."},"phases":{"type":"array","description":"Optional phase declarations matched by phase() calls.","items":{"type":"object","properties":{"title":{"type":"string","description":"The phase title phase() calls match by exact string."},"detail":{"type":"string","description":"Optional one-line description of the phase."},"model":{"type":"string","description":"Optional model override this phase is expected to use."}},"required":["title"]}}},"required":["name","description"]},"args":{"type":"object","description":"Optional JSON input exposed to the script as the `args` global (wrap a bare list as a field, e.g. {\"files\": [...]})."}},"required":["script","meta"]}},{"name":"write","description":"Create or fully replace a UTF-8 text file.","parameters":{"type":"object","properties":{"file_path":{"type":"string","description":"Path to write, resolved by the filesystem backend."},"content":{"type":"string","description":"Full UTF-8 text content to write."}},"required":["file_path","content"]}}]},"reason":"initial"}} {"type":"assistant/chunk","seq":4,"time":1783600630819,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"reasoning"}}} {"type":"assistant/chunk","seq":5,"time":1783600630820,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"The"}}} {"type":"assistant/chunk","seq":6,"time":1783600630822,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" user"}}} diff --git a/examples/acp-agent/tests/snapshots/text-turn/system-prompt.golden.md b/examples/acp-agent/tests/snapshots/text-turn/system-prompt.golden.md new file mode 100644 index 0000000000..edaa7d5cc9 --- /dev/null +++ b/examples/acp-agent/tests/snapshots/text-turn/system-prompt.golden.md @@ -0,0 +1,18 @@ +You are an AI agent powered by the DeepSeek Harness SDK. + +You are a coding assistant powered by the deepseek-v4-flash model. Your working +directory is {{cwd}}. + +Verify your work by running the code or tests. Keep answers brief and +factual. + + +Use the read tool — not shell commands like cat — to inspect text files. Results include line numbers. Use offset and limit to continue reading large files. + +Use the write tool to create files or completely replace file contents. Existing files are overwritten, so read an existing file first (the default fs-policy requires it) and prefer edit for targeted changes. + +Use the edit tool for targeted changes to existing UTF-8 text files. It replaces literal old_string with new_string; by default old_string must appear exactly once. If old_string appears multiple times, provide a more specific old_string or set replace_all to true. Read the file first (the default fs-policy requires it), unless you just created or edited it in this session. + +Check the [exit code: N] marker on every bash result; investigate failures before moving on. + +Use the workflow tool ONLY when the user explicitly asks for a workflow or for large multi-agent orchestration: you write a JavaScript script (the tool description documents the exact format) that fans work out across many subagents with phases and structured results. For one or two delegations, prefer plain subagent calls. diff --git a/packages/support/acp-snapshot/README.md b/packages/support/acp-snapshot/README.md index 8ff2fe1413..2c37da4177 100644 --- a/packages/support/acp-snapshot/README.md +++ b/packages/support/acp-snapshot/README.md @@ -5,8 +5,8 @@ The ACP snapshot suite kit: the shared machinery behind the keyless snapshot tie Three layers, importable separately: - **`runScenario` (harness)** — boots the real agent bin as a subprocess via tsx (unbuilt, Loader path), drives it over ACP JSON-RPC stdio from a deterministic `input.json` script, tees raw stdout for the golden + purity check, and harvests every persisted session JSONL (parent + subagent children, primary-first) after a graceful stdin-EOF shutdown. Parameterized by `AgentUnderTest` (`binScript`, `configPath`, `tsconfigPath` — absolute paths; the subprocess cwd is a temp dir outside the repo). -- **Normalizers** — pure functions turning the two captured surfaces into stable text: `normalizeStdout` (JSON-RPC ids → first-seen sequence; UUIDs/cwd → tokens; doubles as the stdout-purity check), `normalizeSessionLog` (times zeroed, `seq` kept), and the composable `scrubRequestHeaders` (header bulk → `{{system}}`/`{{tools}}`, structure kept — [pinned-header RFC](../../../docs/rfc/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md)). -- **`defineAcpSnapshotSuite` (factory)** — registers the whole describe/it tree for a scenario table: per-scenario golden + re-persisted-log compares, record/refresh fixture write-back, the per-header-class pin with its live uniformity guard, and the fixture guard block (no orphan scenario dirs, required files present, exactly one pin per class, pinning fixtures well-formed, non-pinning fixtures header-scrubbed). Must be called at vitest collection time. +- **Normalizers** — pure functions turning the two captured surfaces into stable text: `normalizeStdout` (JSON-RPC ids → first-seen sequence; UUIDs/cwd → tokens; doubles as the stdout-purity check), `normalizeSessionLog` (times zeroed, `seq` kept), `scrubSystemPrompts` (prompt text → `{{system}}` in every JSONL), and `scrubRequestHeaders` (the remaining header bulk → `{{tools}}`/`{{messagePrefix}}` outside each pin, structure kept — [pinned-header RFC](../../../docs/rfc/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md)). +- **`defineAcpSnapshotSuite` (factory)** — registers the whole describe/it tree for a scenario table: per-scenario golden + re-persisted-log compares, record/refresh fixture write-back, the per-header-class pin (`system-prompt.golden.md` plus the JSONL's full tool schemas) with its live uniformity guard, and the fixture guard block (no orphan scenario dirs, required files present, exactly one pin per class, every JSONL prompt-scrubbed, non-pinning fixtures fully header-scrubbed). Must be called at vitest collection time. A consuming `*.snapshot.ts` is the scenario table plus one factory call: @@ -35,8 +35,8 @@ defineAcpSnapshotSuite({ }) ``` -A scenario booting a differently-composed tree sets its own `configPath` (an overlay whose basename still ends in `cordis.yml`, so the bin's replay swap finds the sibling `*cordis.snapshot.yml`) and, when that composition changes the request header, its own `headerClass` with its own pinning scenario — the acp-agent example's Code Mode scenarios are the template. +A scenario booting a differently-composed tree sets its own `configPath` (an overlay whose basename still ends in `cordis.yml`, so the bin's replay swap finds the sibling `*cordis.snapshot.yml`) and, when that composition changes the request header, its own `headerClass` with its own pinning scenario — the acp-agent example's Code Mode scenarios are the template. Each pinning directory's generated `system-prompt.golden.md` is the reviewable snapshot of the normalized composed prompt; `session.jsonl` stores `"system":"{{system}}"` while retaining the complete tool list. -The example also ships a `cordis.snapshot.yml` replay overlay next to its `cordis.yml` (the bin swaps them under `DSH_SNAPSHOT=replay` — [single-source replay config RFC](../../../docs/rfc/implemented/testing/2026-07-04-single-source-acp-replay-config.md)); replay fixtures are served by [`dsh-llm-replay`](../llm-replay/README.md), which this package points at via the `DSH_SNAPSHOT_*` env vars it sets on the child. `pnpm run test:snapshot:record` calls the live LLM and rewrites the recorded scenarios' model fixtures; `pnpm run test:snapshot:refresh` stays keyless, runs the replay overlay, and rewrites stdout plus comparable session-log goldens from the committed model scripts. Fixture roles, record/replay/refresh semantics, and scenario-table fields are documented on `Scenario` and in the [snapshot RFC](../../../docs/rfc/implemented/testing/2026-06-19-acp-snapshot-tests.md). +The example also ships a `cordis.snapshot.yml` replay overlay next to its `cordis.yml` (the bin swaps them under `DSH_SNAPSHOT=replay` — [single-source replay config RFC](../../../docs/rfc/implemented/testing/2026-07-04-single-source-acp-replay-config.md)); replay fixtures are served by [`dsh-llm-replay`](../llm-replay/README.md), which this package points at via the `DSH_SNAPSHOT_*` env vars it sets on the child. `pnpm run test:snapshot:record` calls the live LLM and rewrites the recorded scenarios' model fixtures; `pnpm run test:snapshot:refresh` stays keyless, runs the replay overlay, and rewrites stdout, comparable session-log goldens, and each pin's Markdown prompt snapshot from the committed model scripts. Fixture roles, record/replay/refresh semantics, and scenario-table fields are documented on `Scenario` and in the [snapshot RFC](../../../docs/rfc/implemented/testing/2026-06-19-acp-snapshot-tests.md). Constraints: `suite.ts` imports vitest, so the package is importable only inside a vitest run (the harness and normalizers have no such dependency but ship from the same entry). ACP-specific by design — the harness speaks the SDK's `ClientSideConnection`. Permission round-trips are scriptable: `InputScript.permissionAnswers` is a FIFO queue of option-kind selections (`allow_once`, `reject_once`, …) the client maps to the agent-issued `optionId` at answer time; an absent or exhausted queue answers `cancelled`, and a kind the request never offered rejects the run (the agent is answered `cancelled`, so a tolerant agent cannot absorb the scenario bug). diff --git a/packages/support/acp-snapshot/src/index.ts b/packages/support/acp-snapshot/src/index.ts index bbe74030f2..74bee95385 100644 --- a/packages/support/acp-snapshot/src/index.ts +++ b/packages/support/acp-snapshot/src/index.ts @@ -3,7 +3,7 @@ * tier (`pnpm run test:snapshot`). Three layers, composable per example: * the subprocess scenario harness ({@link runScenario}), the pure golden * normalizers ({@link normalizeStdout} / {@link normalizeSessionLog} / - * {@link scrubRequestHeaders}), and the suite factory + * {@link scrubRequestHeaders} / {@link scrubSystemPrompts}), and the suite factory * ({@link defineAcpSnapshotSuite}) that registers a scenario table as a full * describe/it tree. An example's `*.snapshot.ts` supplies only its * {@link AgentUnderTest} paths, its snapshots directory, and its @@ -29,6 +29,7 @@ export { normalizeSessionLog, normalizeStdout, scrubRequestHeaders, + scrubSystemPrompts, type NormalizeContext, } from './normalize.ts' export { diff --git a/packages/support/acp-snapshot/src/normalize.ts b/packages/support/acp-snapshot/src/normalize.ts index 017dd504b3..a68fcf83d3 100644 --- a/packages/support/acp-snapshot/src/normalize.ts +++ b/packages/support/acp-snapshot/src/normalize.ts @@ -12,14 +12,14 @@ * `durationMs` (wall-clock hook runtime) → 0. NOT scrubbed: the log's `seq` * (deterministic — `seq = log.length`, part of the event-log contract). * - * A separate, composable normalizer — {@link scrubRequestHeaders} — replaces - * the bulky request-header CONTENT (the composed system prompt, the tool - * schema list, and the session prefix) with - * `{{system}}`/`{{tools}}`/`{{messagePrefix}}` tokens. It is deliberately NOT - * folded into {@link normalizeSessionLog}: each suite's one header-pinning - * scenario compares that content verbatim, every other scenario composes the - * scrub in (the `pinsHeader` flag on the scenario table, consumed by the suite - * factory in ./suite.ts; see the pinned-header RFC, + * Separate, composable normalizers keep bulky request-header content out of + * session fixtures. {@link scrubSystemPrompts} replaces the composed system + * prompt in EVERY fixture; {@link scrubRequestHeaders} additionally replaces + * tool schemas and the session prefix outside each suite's header-pinning + * scenario. They are deliberately NOT folded into + * {@link normalizeSessionLog}: the suite factory composes the right scrub for + * each scenario and snapshots the pin's actual prompt as Markdown (see the + * pinned-header RFC, * docs/rfc/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md). * * See docs/rfc/implemented/testing/2026-06-19-acp-snapshot-tests.md. @@ -135,37 +135,37 @@ export function normalizeSessionLog(rawLog: string, ctx: NormalizeContext): stri } /** - * Replace request-header CONTENT in a session JSONL with stable tokens, - * keeping its structure: a `request/header` event's `data.header.system` → - * `{{system}}`, `data.header.tools` → `{{tools}}`, and - * `data.header.messagePrefix` → one `{{messagePrefix}}` token per message - * (the session prefix is model-visible bulk — an AGENTS digest, a skills - * catalog — so its COUNT stays a structural fact while its text never lands - * in a fixture); a - * `request/header-delta` event keeps every structural fact — the system - * delta's `keepStart`/`keepEnd` line positions and inserted-line COUNT (one - * `{{system}}` token per inserted line), the tools delta's - * added/removed/changed tool NAMES, the prefix replacement's message COUNT — - * and tokenizes only the bulk (prompt - * text; each added/changed schema's fields other than `name` → `{{tools}}`; - * each replacement prefix message → `{{messagePrefix}}`), - * so two different deltas still compare different. - * Absent fields stay absent — WHETHER a header carried a system prompt, - * tools, or a prefix is behavior and stays visible; `config` and `reason` - * are small and - * stable, so they stay verbatim (a model swap churns every fixture by design - * — it invalidates the recorded responses; a prompt/schema edit churns none — - * replay never reads this content, see dsh-llm-replay). - * - * Only lines with something to scrub are re-serialized; every other line - * passes through byte-for-byte, so the transform is idempotent and applying - * it to an already-scrubbed fixture is a no-op — the on-disk-fixtures guard - * in ./suite.ts relies on exactly that. + * Replace system-prompt content in request headers and header deltas with + * `{{system}}` tokens while retaining field presence and delta structure. + * Other header content stays verbatim, so a header-pinning fixture can keep + * its complete tool schemas while every JSONL fixture omits the prompt text. + * Lines without a system payload pass through byte-for-byte; the transform is + * idempotent. * * @param rawLog The raw session `.jsonl` content. - * @returns The JSONL with header content tokenized, other lines byte-identical. + * @returns The JSONL with system-prompt content tokenized. + */ +export function scrubSystemPrompts(rawLog: string): string { + return scrubHeaderContent(rawLog, false) +} + +/** + * Replace all bulky request-header content in a session JSONL with stable + * tokens. This includes the system-prompt fields handled by + * {@link scrubSystemPrompts}, tool schemas, and session-prefix messages. It + * keeps system-delta line positions and arity, tool-delta names, prefix + * message counts, field presence, config, and reason. Lines without content + * to scrub pass through byte-for-byte, and the transform is idempotent. + * + * @param rawLog The raw session `.jsonl` content. + * @returns The JSONL with all header bulk tokenized, other lines byte-identical. */ export function scrubRequestHeaders(rawLog: string): string { + return scrubHeaderContent(rawLog, true) +} + +/** Transform header content, optionally including tool schemas and the session prefix. */ +function scrubHeaderContent(rawLog: string, scrubToolsAndPrefix: boolean): string { const lines = rawLog.split('\n') const out = lines.map((line) => { if (line.trim().length === 0) return line @@ -175,11 +175,14 @@ export function scrubRequestHeaders(rawLog: string): string { if (record.type === 'request/header') { const header = data.header as Record | null | undefined if (header === null || typeof header !== 'object') return line - if (!('system' in header) && !('tools' in header) && !('messagePrefix' in header)) return line - if ('system' in header) header.system = SYSTEM - if ('tools' in header) header.tools = TOOLS - if (Array.isArray(header.messagePrefix)) header.messagePrefix = header.messagePrefix.map(() => MESSAGE_PREFIX) - return JSON.stringify(record) + let touched = false + if ('system' in header) { header.system = SYSTEM; touched = true } + if (scrubToolsAndPrefix && 'tools' in header) { header.tools = TOOLS; touched = true } + if (scrubToolsAndPrefix && Array.isArray(header.messagePrefix)) { + header.messagePrefix = header.messagePrefix.map(() => MESSAGE_PREFIX) + touched = true + } + return touched ? JSON.stringify(record) : line } if (record.type === 'request/header-delta') { let touched = false @@ -189,11 +192,11 @@ export function scrubRequestHeaders(rawLog: string): string { touched = true } const tools = data.tools as Record | null | undefined - if (tools !== null && typeof tools === 'object') { + if (scrubToolsAndPrefix && tools !== null && typeof tools === 'object') { if (Array.isArray(tools.added)) { tools.added = tools.added.map(scrubToolSchema); touched = true } if (Array.isArray(tools.changed)) { tools.changed = tools.changed.map(scrubToolSchema); touched = true } } - if (Array.isArray(data.messagePrefix)) { + if (scrubToolsAndPrefix && Array.isArray(data.messagePrefix)) { data.messagePrefix = data.messagePrefix.map(() => MESSAGE_PREFIX) touched = true } diff --git a/packages/support/acp-snapshot/src/suite.ts b/packages/support/acp-snapshot/src/suite.ts index 27de939d3d..27e6934058 100644 --- a/packages/support/acp-snapshot/src/suite.ts +++ b/packages/support/acp-snapshot/src/suite.ts @@ -10,15 +10,14 @@ * (recorded scenarios) and the expected produced log (both sides normalized * before comparing). * - * Request-header content (the composed system prompt + tool schemas riding on - * `request/header` events) is pinned by exactly ONE scenario per HEADER CLASS - * — scenarios that boot the same config compose the same header; each class's - * `pinsHeader` scenario commits it verbatim — and scrubbed to - * `{{system}}`/`{{tools}}` tokens in every other fixture and compare, so a - * prompt or tool-schema edit churns one committed line per class instead of - * every fixture. A per-run uniformity guard keeps each pin sound: every live - * header must equal its class's pinned one, and no header-delta may appear - * outside a pinning scenario (see the pinned-header RFC, + * Request-header content is pinned by exactly ONE scenario per HEADER CLASS — + * scenarios that boot the same config compose the same header. Every JSONL + * fixture scrubs the system prompt to `{{system}}`; each class's pinning + * scenario stores the readable prompt in `system-prompt.golden.md` and keeps its full + * tool schemas in `session.jsonl`, while every other fixture also scrubs tools + * to `{{tools}}`. A per-run uniformity guard compares both artifacts against + * every live header and forbids unrepresented header deltas (see the + * pinned-header RFC, * docs/rfc/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md). * * `pnpm run test:snapshot:record` (DSH_SNAPSHOT=record + -u) re-records the @@ -37,7 +36,16 @@ import { existsSync } from 'node:fs' import { join } from 'node:path' import { describe, expect, it } from 'vitest' import { type AgentUnderTest, type HarvestedLog, type InputScript, runScenario } from './harness.ts' -import { type NormalizeContext, normalizeSessionLog, normalizeStdout, scrubRequestHeaders } from './normalize.ts' +import { + type NormalizeContext, + normalizeSessionLog, + normalizeStdout, + scrubRequestHeaders, + scrubSystemPrompts, +} from './normalize.ts' + +/** The readable system-prompt snapshot beside each header-pinning fixture. */ +const SYSTEM_PROMPT_SNAPSHOT = 'system-prompt.golden.md' /** A snapshot scenario and how its fixtures are produced. */ export interface Scenario { @@ -81,14 +89,13 @@ export interface Scenario { */ childSessions?: number /** - * Whether THIS scenario's fixtures keep the full request-header content (the - * composed system prompt and tool schema list on `request/header` / - * `request/header-delta` events) and compare it verbatim. Exactly one - * scenario per HEADER CLASS ({@link headerClass}) pins it; every other - * scenario of that class stores and compares that content as - * `{{system}}`/`{{tools}}` tokens ({@link scrubRequestHeaders}), - * so a system prompt or tool-schema change shows up as ONE committed-fixture - * diff per class, not one per scenario. One pin per class suffices because + * Whether THIS scenario pins its header class's model-facing request-header + * content. Its actual composed prompt is maintained as a readable + * `system-prompt.golden.md`; its JSONL keeps full tool schemas but stores the prompt + * as `{{system}}`. Every other scenario of the class stores tools as + * `{{tools}}` too ({@link scrubRequestHeaders}). A prompt or tool-schema + * change therefore shows up in one focused artifact per class, not every + * session fixture. One pin per class suffices because * header composition is class-uniform (parent, spawn child, and fork child * all compose the same prompt-modulo-cwd and the same tools) — and that * premise is ASSERTED, not assumed: every non-pinning run's live headers @@ -124,7 +131,7 @@ export interface SnapshotSuiteOptions { agent: AgentUnderTest /** Absolute path of the suite's `snapshots/` directory (one subdir per scenario). */ snapshotsDir: string - /** The scenario table; exactly one entry must set `pinsHeader`. */ + /** The scenario table; exactly one entry per header class must set `pinsHeader`. */ scenarios: Scenario[] /** * `replay` (keyless, the default tier), `record` (live API; re-records the @@ -192,6 +199,35 @@ export function normalizedHeaders(rawLog: string, ctx: NormalizeContext): unknow .map(record => record.data?.header) } +/** + * The normalized string-valued system prompts carried by request headers in a + * session JSONL, in log order. Headers without a string prompt are omitted so + * callers can assert one prompt per header explicitly. + * + * @param rawLog The session `.jsonl` content to inspect. + * @param ctx The volatile values of the run that produced it. + * @returns The normalized system prompts, in header order. + */ +export function normalizedSystemPrompts(rawLog: string, ctx: NormalizeContext): string[] { + return normalizedHeaders(rawLog, ctx).flatMap((header) => { + if (header === null || typeof header !== 'object') return [] + const system = (header as { system?: unknown }).system + return typeof system === 'string' ? [system] : [] + }) +} + +/** + * Render a normalized prompt as a repository-friendly Markdown snapshot. + * Prompt text is unchanged except that a missing terminal newline is added so + * the committed file follows the repository newline contract. + * + * @param prompt The normalized system prompt. + * @returns Markdown snapshot text ending in a newline. + */ +export function formatSystemPromptSnapshot(prompt: string): string { + return prompt.endsWith('\n') ? prompt : `${prompt}\n` +} + /** * Count the `request/header-delta` events in a session JSONL. * @@ -289,7 +325,8 @@ export function stabilizeRefreshLog(fresh: string, existing: string, replacement * Register the suite: one `describe` per scenario (the golden/log compares and * the header-uniformity guard) plus the fixture guard block (no orphan * scenario dirs, required files present, exactly one pin per header class, - * pinning fixtures well-formed, non-pinning fixtures header-scrubbed). Must + * pinning fixtures well-formed, every JSONL prompt-scrubbed, non-pinning + * fixtures fully header-scrubbed). Must * run at vitest collection time — it calls `describe`/`it`. Throws * immediately if any header class lacks a pinning scenario or carries two * (the uniformity guard needs exactly one comparison anchor per class). @@ -364,11 +401,12 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void { // keyless replay run for every comparable log, including authored // scenarios that live record deliberately skips. The primary goes to // session.jsonl, each child to session..jsonl in harvest order. A - // non-pinning scenario's fixtures are written header-scrubbed, so a - // re-record/refresh can never smuggle the full prompt/schema content - // back into every fixture. + // Every fixture is written with its system prompt scrubbed. A pinning + // scenario keeps the remaining header content (notably tool schemas); + // every other scenario scrubs that bulk too. Record/refresh therefore + // cannot smuggle prompt text back into JSONL or duplicate schemas. const scrub = scenario.pinsHeader === true - ? (log: string): string => log + ? scrubSystemPrompts : scrubRequestHeaders const fixtureFiles = ['session.jsonl', ...Array.from({ length: childSessions }, (_, i) => `session.${i + 1}.jsonl`)] const existingFixtures = REFRESHING @@ -391,6 +429,16 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void { REFRESHING ? stabilizeRefreshLog(child, existingFixtures[i] as string, replacements) : child, )) } + if (scenario.pinsHeader === true) { + const prompts = result.sessionLogs.flatMap(log => normalizedSystemPrompts(log.content, ctx)) + expect(prompts.length, `${mode} produced no system prompt to snapshot`).toBeGreaterThan(0) + const snapshot = formatSystemPromptSnapshot(prompts[0] as string) + for (const prompt of prompts) { + expect(formatSystemPromptSnapshot(prompt), 'the pinning run produced divergent system prompts') + .toEqual(snapshot) + } + await writeFile(join(dir, SYSTEM_PROMPT_SNAPSHOT), snapshot) + } } const stdout = normalizeStdout(result.rawStdout, ctx) @@ -406,12 +454,10 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void { // 1:1. Each side passes through normalizeSessionLog, scrubbed against ITS // OWN volatile values — the live run's via `ctx`, the committed fixture's // via its own header (a committed file cannot share the live run's ids). - // Unless this scenario pins the header, both sides ALSO pass through - // scrubRequestHeaders: the live log carries the real prompt/schemas, the - // fixture carries the `{{system}}`/`{{tools}}` tokens, and the scrub is - // idempotent — so the compare checks the header's presence, position, - // reason, and config, but not its bulk content (pinned once, in the - // `pinsHeader` scenario). + // Both sides pass through the scenario's idempotent scrub: every live + // prompt becomes the fixture's `{{system}}`; non-pinning scenarios + // additionally tokenize tools/prefix. The dedicated header guard below + // compares those omitted values against their class's pin artifacts. expect(result.sessionLogs.length, 'this scenario must persist a session log').toBe(childSessions + 1) for (let i = 0; i < fixtureFiles.length; i++) { const harvested = scrub((result.sessionLogs[i] as HarvestedLog).content) @@ -421,34 +467,30 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void { } } - // Header-uniformity guard: a class's single pin is sound only while - // every session in that class composes the SAME header and keeps it - // for the whole run. Assert both halves live. (1) Every - // request/header the run produced (parent, spawn child, fork child, - // initial or resume) must equal the CLASS's pinned fixture's header - // after each side is normalized against its own volatile values. - // (2) No request/header-delta may appear at all — a mid-run header - // change diverges from the pin by construction, and its content - // would be invisible under the scrub. If either fails, either the - // header changed (update the pin: re-record or hand-edit the pinning - // scenario's fixture) or composition became session-dependent by - // design (give the divergent shape its own pinning scenario and - // class). - if (scenario.pinsHeader !== true) { - /* v8 ignore next -- construction guarantees the pin exists; a miss would fail the one-header assertion loudly. */ - const pinningScenario = pinningByClass.get(classOf(scenario)) ?? scenario - const pinnedFixture = await readFile(join(snapshotsDir, pinningScenario.name, 'session.jsonl'), 'utf8') - const pinned = normalizedHeaders(pinnedFixture, fixtureContext(pinnedFixture)) - expect(pinned.length, `the pinning fixture (${pinningScenario.name}) must carry exactly one request/header`) - .toBe(1) - for (const log of result.sessionLogs) { - expect(headerDeltaCount(log.content), `session ${log.id}: a request/header-delta in a non-pinning scenario`) - .toBe(0) - const headers = normalizedHeaders(log.content, ctx) - for (const [k, header] of headers.entries()) { - expect(header, `session ${log.id}: request/header #${k + 1} diverged from the pinned (${pinningScenario.name}) header`) - .toEqual(pinned[0]) - } + // Header-uniformity guard: every live header in a class must equal the + // class pin split across its JSONL header (system token + real tools) + // and readable Markdown prompt. No header delta is representable by + // those two static artifacts, so any delta fails loud. + /* v8 ignore next -- construction guarantees the pin exists; a miss would fail the one-header assertion loudly. */ + const pinningScenario = pinningByClass.get(classOf(scenario)) ?? scenario + const pinningDir = join(snapshotsDir, pinningScenario.name) + const pinnedFixture = await readFile(join(pinningDir, 'session.jsonl'), 'utf8') + const pinned = normalizedHeaders(pinnedFixture, fixtureContext(pinnedFixture)) + const promptSnapshot = await readFile(join(pinningDir, SYSTEM_PROMPT_SNAPSHOT), 'utf8') + expect(pinned.length, `the pinning fixture (${pinningScenario.name}) must carry exactly one request/header`) + .toBe(1) + for (const log of result.sessionLogs) { + expect(headerDeltaCount(log.content), `session ${log.id}: a request/header-delta is not represented by the class pin`) + .toBe(0) + const headers = normalizedHeaders(scrubSystemPrompts(log.content), ctx) + const prompts = normalizedSystemPrompts(log.content, ctx) + expect(prompts.length, `session ${log.id}: every request/header must carry a string system prompt`) + .toBe(headers.length) + for (const [k, header] of headers.entries()) { + expect(header, `session ${log.id}: request/header #${k + 1} diverged from the pinned (${pinningScenario.name}) header`) + .toEqual(pinned[0]) + expect(formatSystemPromptSnapshot(prompts[k] as string), `session ${log.id}: system prompt #${k + 1} diverged from ${pinningScenario.name}/${SYSTEM_PROMPT_SNAPSHOT}`) + .toEqual(promptSnapshot) } } }) @@ -479,13 +521,15 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void { // `overridden` flag: required when set, forbidden when not — the harness // forwards the file purely on existence, so an unregistered stray sidecar // would silently replace the derived script. - for (const { name, overridden, childSessions } of scenarios) { + for (const { name, overridden, childSessions, pinsHeader } of scenarios) { const dir = join(snapshotsDir, name) expect(existsSync(join(dir, 'input.json')), `${name}/input.json`).toBe(true) expect(existsSync(join(dir, 'stdout.golden.jsonl')), `${name}/stdout.golden.jsonl`).toBe(true) expect(existsSync(join(dir, 'session.jsonl')), `${name}/session.jsonl`).toBe(true) expect(existsSync(join(dir, 'replay.override.json')), `${name}/replay.override.json presence must match \`overridden\``) .toBe(overridden === true) + expect(existsSync(join(dir, SYSTEM_PROMPT_SNAPSHOT)), `${name}/${SYSTEM_PROMPT_SNAPSHOT} presence must match \`pinsHeader\``) + .toBe(pinsHeader === true) // A nested-agent scenario ships one child fixture per recorded subagent // session (`session.1.jsonl` …), the replay source for that child session. for (const childFixture of childFixturePaths(dir, childSessions ?? 0)) { @@ -511,7 +555,7 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void { } }) - it('every pinning fixture carries exactly one request/header and no deltas', async () => { + it('every pinning fixture carries exactly one request/header, one readable prompt, and no deltas', async () => { // The live uniformity guard runs only in NON-pinning scenarios, so a // class made of just its pinning scenario would otherwise accept a // re-recorded pin with several headers or a mid-run header-delta — @@ -520,19 +564,18 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void { for (const scenario of pinningByClass.values()) { const fixture = await readFile(join(snapshotsDir, scenario.name, 'session.jsonl'), 'utf8') const headers = normalizedHeaders(fixture, fixtureContext(fixture)) + const promptSnapshot = await readFile(join(snapshotsDir, scenario.name, SYSTEM_PROMPT_SNAPSHOT), 'utf8') expect(headers.length, `${scenario.name}: a pinning fixture must carry exactly one request/header`).toBe(1) + expect(promptSnapshot.length, `${scenario.name}/${SYSTEM_PROMPT_SNAPSHOT} must not be empty`).toBeGreaterThan(0) + expect(promptSnapshot.endsWith('\n'), `${scenario.name}/${SYSTEM_PROMPT_SNAPSHOT} must end in a newline`).toBe(true) expect(headerDeltaCount(fixture), `${scenario.name}: a pinning fixture must carry no request/header-delta`).toBe(0) } }) - it('committed fixtures carry request-header content ONLY in the pinning scenario', async () => { - // The whole point of the pin: a system-prompt or tool-schema change must - // churn exactly one committed line. A non-pinning fixture that carries the - // full header (a hand-recorded file, or a header line hand-edited out of - // its canonical JSON form) silently reopens the suite-wide churn, so fail - // loud here: every non-pinning session*.jsonl must be a fixed point of - // scrubRequestHeaders (apply the scrub to fix a violation), and the - // pinning scenario's fixtures must NOT be (their content IS the pin). + it('every committed JSONL omits system prompts and only pinning fixtures keep other header bulk', async () => { + // System prompts always live in the readable Markdown artifact. Header + // pins keep tool schemas/prefixes in JSONL; every other fixture tokenizes + // all header bulk. Fixed-point checks make both storage rules fail loud. for (const scenario of scenarios) { const dir = join(snapshotsDir, scenario.name) const files = [ @@ -541,8 +584,10 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void { ] for (const file of files) { const fixture = await readFile(join(dir, file), 'utf8') + expect(scrubSystemPrompts(fixture), `${scenario.name}/${file} carries an unscrubbed system prompt`) + .toEqual(fixture) if (scenario.pinsHeader === true) { - expect(scrubRequestHeaders(fixture), `${scenario.name}/${file} must PIN the full header content`) + expect(scrubRequestHeaders(fixture), `${scenario.name}/${file} must pin the non-system header content`) .not.toEqual(fixture) } else { expect(scrubRequestHeaders(fixture), `${scenario.name}/${file} carries unscrubbed header content`) diff --git a/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-pin/session.jsonl b/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-pin/session.jsonl index 109a192083..d496dbdb1d 100644 --- a/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-pin/session.jsonl +++ b/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-pin/session.jsonl @@ -1,2 +1,2 @@ {"type":"session","id":"ccdc749f-56f3-4267-9750-598b5c60b7b2","createdAt":600,"cwd":"/var/folders/2g/b32ct0qn1d728l_v6tdkjytr0000gn/T/acp-snap-cwd-nOQ4Gy"} -{"type":"request/header","seq":0,"time":4,"data":{"header":{"config":{"model":"fake"},"system":"SYS PROMPT","tools":[{"name":"t1","description":"D1","parameters":{"type":"object"}}]},"reason":"initial"}} +{"type":"request/header","seq":0,"time":4,"data":{"header":{"config":{"model":"fake"},"system":"{{system}}","tools":[{"name":"t1","description":"D1","parameters":{"type":"object"}}]},"reason":"initial"}} diff --git a/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-pin/system-prompt.golden.md b/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-pin/system-prompt.golden.md new file mode 100644 index 0000000000..63940ec7c6 --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/record-suite/rec-pin/system-prompt.golden.md @@ -0,0 +1 @@ +SYS PROMPT diff --git a/packages/support/acp-snapshot/tests/fixtures/suite/pin-turn/session.jsonl b/packages/support/acp-snapshot/tests/fixtures/suite/pin-turn/session.jsonl index 87bf09c839..8f69a32a6f 100644 --- a/packages/support/acp-snapshot/tests/fixtures/suite/pin-turn/session.jsonl +++ b/packages/support/acp-snapshot/tests/fixtures/suite/pin-turn/session.jsonl @@ -1,3 +1,3 @@ {"type":"session","id":"12121212-3434-4545-8686-787878787878","createdAt":7,"cwd":"/rec/pin-cwd"} -{"type":"request/header","seq":0,"time":7,"data":{"header":{"config":{"model":"fake"},"system":"SYS PROMPT","tools":[{"name":"t1","description":"D1","parameters":{"type":"object"}}]},"reason":"initial"}} +{"type":"request/header","seq":0,"time":7,"data":{"header":{"config":{"model":"fake"},"system":"{{system}}","tools":[{"name":"t1","description":"D1","parameters":{"type":"object"}}]},"reason":"initial"}} {"type":"turn/start","seq":1,"time":7,"data":{"turn":1}} diff --git a/packages/support/acp-snapshot/tests/fixtures/suite/pin-turn/system-prompt.golden.md b/packages/support/acp-snapshot/tests/fixtures/suite/pin-turn/system-prompt.golden.md new file mode 100644 index 0000000000..63940ec7c6 --- /dev/null +++ b/packages/support/acp-snapshot/tests/fixtures/suite/pin-turn/system-prompt.golden.md @@ -0,0 +1 @@ +SYS PROMPT diff --git a/packages/support/acp-snapshot/tests/normalize.spec.ts b/packages/support/acp-snapshot/tests/normalize.spec.ts index daa9f8342d..bdde120a76 100644 --- a/packages/support/acp-snapshot/tests/normalize.spec.ts +++ b/packages/support/acp-snapshot/tests/normalize.spec.ts @@ -1,5 +1,11 @@ import { describe, expect, it } from 'vitest' -import { type NormalizeContext, normalizeSessionLog, normalizeStdout, scrubRequestHeaders } from '../src/normalize.ts' +import { + type NormalizeContext, + normalizeSessionLog, + normalizeStdout, + scrubRequestHeaders, + scrubSystemPrompts, +} from '../src/normalize.ts' /** * Unit tests for the pure snapshot normalizers. Live as a *.spec.ts (runs in @@ -260,3 +266,43 @@ describe('scrubRequestHeaders', () => { expect(scrubRequestHeaders(once)).toBe(once) }) }) + +describe('scrubSystemPrompts', () => { + it('scrubs only system prompt payloads while keeping tools and prefixes verbatim', () => { + const header = JSON.stringify({ + type: 'request/header', seq: 1, time: 2, + data: { + header: { + system: 'full prompt', + tools: [{ name: 'read', description: 'full schema' }], + messagePrefix: [{ role: 'user', content: [{ type: 'text', text: 'full prefix' }] }], + }, + reason: 'initial', + }, + }) + const delta = JSON.stringify({ + type: 'request/header-delta', seq: 2, time: 3, + data: { + system: { keepStart: 1, keepEnd: 2, insert: ['new prompt line'] }, + tools: { changed: [{ name: 'read', description: 'changed schema' }] }, + messagePrefix: [{ role: 'user', content: [{ type: 'text', text: 'changed prefix' }] }], + }, + }) + const toolsOnly = JSON.stringify({ + type: 'request/header', seq: 3, time: 4, + data: { header: { tools: [{ name: 'read', description: 'schema only' }] }, reason: 'resume' }, + }) + + const out = scrubSystemPrompts(`${header}\n${delta}\n${toolsOnly}\n`) + expect(out).toContain('"system":"{{system}}"') + expect(out).toContain('"insert":["{{system}}"]') + expect(out).not.toContain('full prompt') + expect(out).not.toContain('new prompt line') + expect(out).toContain('full schema') + expect(out).toContain('full prefix') + expect(out).toContain('changed schema') + expect(out).toContain('changed prefix') + expect(out.split('\n')[2]).toBe(toolsOnly) + expect(scrubSystemPrompts(out)).toBe(out) + }) +}) diff --git a/packages/support/acp-snapshot/tests/suite.spec.ts b/packages/support/acp-snapshot/tests/suite.spec.ts index ae01190c01..0eb08d05f5 100644 --- a/packages/support/acp-snapshot/tests/suite.spec.ts +++ b/packages/support/acp-snapshot/tests/suite.spec.ts @@ -8,8 +8,10 @@ import { defineAcpSnapshotSuite, type HarvestedLog, type Scenario } from '../src import { childFixturePaths, fixtureContext, + formatSystemPromptSnapshot, headerDeltaCount, normalizedHeaders, + normalizedSystemPrompts, refreshFixtureReplacements, stabilizeRefreshLog, } from '../src/suite.ts' @@ -78,6 +80,7 @@ afterAll(async () => { function staleRefreshFixtures(dir: string): void { writeFileSync(join(dir, 'plain-turn', 'stdout.golden.jsonl'), 'stale stdout\n') + writeFileSync(join(dir, 'pin-turn', 'system-prompt.golden.md'), 'STALE PROMPT\n') const plainBehaviorFile = join(dir, 'plain-turn', 'behavior.json') const plainBehavior = JSON.parse(readFileSync(plainBehaviorFile, 'utf8')) as Record @@ -124,6 +127,8 @@ describe('defineAcpSnapshotSuite: refresh write-back', () => { const authored = readFileSync(join(refreshDir, 'authored-error', 'session.jsonl'), 'utf8') expect(authored).toContain('"error":"model exploded"') expect(authored).not.toContain('"error":"stale"') + + expect(readFileSync(join(refreshDir, 'pin-turn', 'system-prompt.golden.md'), 'utf8')).toBe('SYS PROMPT\n') }) }) @@ -219,6 +224,28 @@ describe('normalizedHeaders', () => { }) }) +describe('normalizedSystemPrompts', () => { + it('extracts normalized string prompts and omits absent or non-string fields', () => { + const log = [ + '{"type":"session","id":"a","createdAt":5,"cwd":"/w"}', + '{"type":"request/header","seq":0,"time":9,"data":{"header":{"system":"work in /w"}}}', + '{"type":"request/header","seq":1,"time":9,"data":{"header":{}}}', + '{"type":"request/header","seq":2,"time":9,"data":{"header":{"system":null}}}', + '{"type":"request/header","seq":3,"time":9,"data":{"header":null}}', + '{"type":"request/header","seq":4,"time":9,"data":{"header":"invalid"}}', + '', + ].join('\n') + expect(normalizedSystemPrompts(log, { sessionIds: [], cwd: '/w' })).toEqual(['work in {{cwd}}']) + }) +}) + +describe('formatSystemPromptSnapshot', () => { + it('adds a missing terminal newline without changing an existing one', () => { + expect(formatSystemPromptSnapshot('prompt')).toBe('prompt\n') + expect(formatSystemPromptSnapshot('prompt\n')).toBe('prompt\n') + }) +}) + describe('headerDeltaCount', () => { it('counts request/header-delta events, ignoring blanks and other lines', () => { const delta = JSON.stringify({ type: 'request/header-delta', seq: 2, time: 9, data: {} }) From 169f86966a2cb2aec97ed2613cb8f766b6caed4f Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Sat, 11 Jul 2026 22:33:33 +0800 Subject: [PATCH 2/4] fix(review): lint prompt Markdown paragraphs --- .../acp-agent/both-mode.cordis.snapshot.yml | 6 ++--- examples/acp-agent/both-mode.cordis.yml | 6 ++--- .../acp-agent/code-mode.cordis.snapshot.yml | 6 ++--- examples/acp-agent/code-mode.cordis.yml | 6 ++--- examples/acp-agent/cordis.yml | 6 ++--- .../both-mode-turn/system-prompt.golden.md | 6 ++--- .../code-mode-turn/system-prompt.golden.md | 6 ++--- .../text-turn/system-prompt.golden.md | 6 ++--- scripts/verify-md-wrap.ts | 22 ++++++++++++++----- 9 files changed, 33 insertions(+), 37 deletions(-) diff --git a/examples/acp-agent/both-mode.cordis.snapshot.yml b/examples/acp-agent/both-mode.cordis.snapshot.yml index 67044b8066..8ee54b3078 100644 --- a/examples/acp-agent/both-mode.cordis.snapshot.yml +++ b/examples/acp-agent/both-mode.cordis.snapshot.yml @@ -20,11 +20,9 @@ tools: mode: both persona: | - You are a coding assistant powered by the {{model}} model. Your working - directory is {{cwd}}. + You are a coding assistant powered by the {{model}} model. Your working directory is {{cwd}}. - Verify your work by running the code or tests. Keep answers brief and - factual. + Verify your work by running the code or tests. Keep answers brief and factual. - insert: - id: code-runtime name: '@deepseek-ai/dsh-code-runtime-worker' diff --git a/examples/acp-agent/both-mode.cordis.yml b/examples/acp-agent/both-mode.cordis.yml index b449a568ec..3dff66d60a 100644 --- a/examples/acp-agent/both-mode.cordis.yml +++ b/examples/acp-agent/both-mode.cordis.yml @@ -19,11 +19,9 @@ tools: mode: both persona: | - You are a coding assistant powered by the {{model}} model. Your working - directory is {{cwd}}. + You are a coding assistant powered by the {{model}} model. Your working directory is {{cwd}}. - Verify your work by running the code or tests. Keep answers brief and - factual. + Verify your work by running the code or tests. Keep answers brief and factual. - insert: - id: code-runtime name: '@deepseek-ai/dsh-code-runtime-worker' diff --git a/examples/acp-agent/code-mode.cordis.snapshot.yml b/examples/acp-agent/code-mode.cordis.snapshot.yml index bcaa225eba..d525afc5d6 100644 --- a/examples/acp-agent/code-mode.cordis.snapshot.yml +++ b/examples/acp-agent/code-mode.cordis.snapshot.yml @@ -20,11 +20,9 @@ tools: mode: code persona: | - You are a coding assistant powered by the {{model}} model. Your working - directory is {{cwd}}. + You are a coding assistant powered by the {{model}} model. Your working directory is {{cwd}}. - Verify your work by running the code or tests. Keep answers brief and - factual. + Verify your work by running the code or tests. Keep answers brief and factual. - insert: - id: code-runtime name: '@deepseek-ai/dsh-code-runtime-worker' diff --git a/examples/acp-agent/code-mode.cordis.yml b/examples/acp-agent/code-mode.cordis.yml index d46e490494..323c35b5b4 100644 --- a/examples/acp-agent/code-mode.cordis.yml +++ b/examples/acp-agent/code-mode.cordis.yml @@ -20,11 +20,9 @@ tools: mode: code persona: | - You are a coding assistant powered by the {{model}} model. Your working - directory is {{cwd}}. + You are a coding assistant powered by the {{model}} model. Your working directory is {{cwd}}. - Verify your work by running the code or tests. Keep answers brief and - factual. + Verify your work by running the code or tests. Keep answers brief and factual. - insert: - id: code-runtime name: '@deepseek-ai/dsh-code-runtime-worker' diff --git a/examples/acp-agent/cordis.yml b/examples/acp-agent/cordis.yml index 0ff736dd59..dc03ea6b03 100644 --- a/examples/acp-agent/cordis.yml +++ b/examples/acp-agent/cordis.yml @@ -44,11 +44,9 @@ # loop resolves per session (every ACP session carries the client's cwd, # so the persona can state the workspace). persona: | - You are a coding assistant powered by the {{model}} model. Your working - directory is {{cwd}}. + You are a coding assistant powered by the {{model}} model. Your working directory is {{cwd}}. - Verify your work by running the code or tests. Keep answers brief and - factual. + Verify your work by running the code or tests. Keep answers brief and factual. # The subagent seam + both in-process backends + two model-facing tools, as leaf # entries after the app (which provides ctx.agents/ctx.tools). spawn (a fresh diff --git a/examples/acp-agent/tests/snapshots/both-mode-turn/system-prompt.golden.md b/examples/acp-agent/tests/snapshots/both-mode-turn/system-prompt.golden.md index 688557b132..d817cf1a80 100644 --- a/examples/acp-agent/tests/snapshots/both-mode-turn/system-prompt.golden.md +++ b/examples/acp-agent/tests/snapshots/both-mode-turn/system-prompt.golden.md @@ -1,10 +1,8 @@ You are an AI agent powered by the DeepSeek Harness SDK. -You are a coding assistant powered by the deepseek-v4-flash model. Your working -directory is {{cwd}}. +You are a coding assistant powered by the deepseek-v4-flash model. Your working directory is {{cwd}}. -Verify your work by running the code or tests. Keep answers brief and -factual. +Verify your work by running the code or tests. Keep answers brief and factual. Use the read tool — not shell commands like cat — to inspect text files. Results include line numbers. Use offset and limit to continue reading large files. diff --git a/examples/acp-agent/tests/snapshots/code-mode-turn/system-prompt.golden.md b/examples/acp-agent/tests/snapshots/code-mode-turn/system-prompt.golden.md index 688557b132..d817cf1a80 100644 --- a/examples/acp-agent/tests/snapshots/code-mode-turn/system-prompt.golden.md +++ b/examples/acp-agent/tests/snapshots/code-mode-turn/system-prompt.golden.md @@ -1,10 +1,8 @@ You are an AI agent powered by the DeepSeek Harness SDK. -You are a coding assistant powered by the deepseek-v4-flash model. Your working -directory is {{cwd}}. +You are a coding assistant powered by the deepseek-v4-flash model. Your working directory is {{cwd}}. -Verify your work by running the code or tests. Keep answers brief and -factual. +Verify your work by running the code or tests. Keep answers brief and factual. Use the read tool — not shell commands like cat — to inspect text files. Results include line numbers. Use offset and limit to continue reading large files. diff --git a/examples/acp-agent/tests/snapshots/text-turn/system-prompt.golden.md b/examples/acp-agent/tests/snapshots/text-turn/system-prompt.golden.md index edaa7d5cc9..18f0cbcd07 100644 --- a/examples/acp-agent/tests/snapshots/text-turn/system-prompt.golden.md +++ b/examples/acp-agent/tests/snapshots/text-turn/system-prompt.golden.md @@ -1,10 +1,8 @@ You are an AI agent powered by the DeepSeek Harness SDK. -You are a coding assistant powered by the deepseek-v4-flash model. Your working -directory is {{cwd}}. +You are a coding assistant powered by the deepseek-v4-flash model. Your working directory is {{cwd}}. -Verify your work by running the code or tests. Keep answers brief and -factual. +Verify your work by running the code or tests. Keep answers brief and factual. Use the read tool — not shell commands like cat — to inspect text files. Results include line numbers. Use offset and limit to continue reading large files. diff --git a/scripts/verify-md-wrap.ts b/scripts/verify-md-wrap.ts index 2d8845e030..3143ee5617 100644 --- a/scripts/verify-md-wrap.ts +++ b/scripts/verify-md-wrap.ts @@ -18,9 +18,11 @@ * A wrapped paragraph inside a list item or blockquote is still a `paragraph` * node, so those are caught too. Scope mirrors doc-typecheck plus the two * AGENTS.md files that doc-sync does NOT otherwise cover (the convention itself - * lives there): README.md, docs/** /*.md, packages/* /*.md, AGENTS.md, - * packages/AGENTS.md. The root and packages/ CLAUDE.md are symlinks to the - * AGENTS.md files, so they are deduped by real path. + * lives there), plus generated system-prompt Markdown goldens: README.md, + * docs/** /*.md, packages/* /*.md, examples/** /system-prompt.golden.md, + * packages/** /system-prompt.golden.md, AGENTS.md, packages/AGENTS.md. The root + * and packages/ CLAUDE.md are symlinks to the AGENTS.md files, so they are + * deduped by real path. * * Run: `tsx scripts/verify-md-wrap.ts`. */ @@ -34,8 +36,18 @@ import type { Nodes } from 'mdast' const root = resolve(import.meta.dirname, '..') -/** Files to check: doc-typecheck's scope plus the AGENTS.md pair. */ -const PATTERNS = ['README.md', 'README.zh.md', 'docs/**/*.md', 'packages/*/*.md', 'packages/*/*/*.md', 'AGENTS.md', 'packages/AGENTS.md'] +/** Files to check: doc-typecheck's scope, prompt goldens, and the AGENTS.md pair. */ +const PATTERNS = [ + 'README.md', + 'README.zh.md', + 'docs/**/*.md', + 'packages/*/*.md', + 'packages/*/*/*.md', + 'examples/**/system-prompt.golden.md', + 'packages/**/system-prompt.golden.md', + 'AGENTS.md', + 'packages/AGENTS.md', +] /** A located hard-wrap: a prose paragraph spanning more than one source line. */ interface Violation { From bac97a5b2e1ae105eaa3a50c3aa131f8ead02fbb Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Sat, 11 Jul 2026 23:55:56 +0800 Subject: [PATCH 3/4] test(acp-snapshot): cover the delta-insert trailing-newline branch formatSystemPromptSnapshot's insert-join ternary had its already-newline- terminated arm unexercised (a delta whose insert ends in a blank line), failing the per-file 100% branch gate on suite.ts (99.24%). --- packages/support/acp-snapshot/tests/suite.spec.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/support/acp-snapshot/tests/suite.spec.ts b/packages/support/acp-snapshot/tests/suite.spec.ts index dc6b71ea00..d6e3e2b912 100644 --- a/packages/support/acp-snapshot/tests/suite.spec.ts +++ b/packages/support/acp-snapshot/tests/suite.spec.ts @@ -273,6 +273,12 @@ describe('formatSystemPromptSnapshot', () => { { keepStart: 1, keepEnd: 0, insert: ['new', 'lines'] }, ])).toBe('prompt\n\n\n\nnew\nlines\n') }) + + it('does not double the newline of a delta insert with a trailing blank line', () => { + expect(formatSystemPromptSnapshot('prompt\n', [ + { keepStart: 2, keepEnd: 1, insert: ['tail', ''] }, + ])).toBe('prompt\n\n\n\ntail\n') + }) }) describe('headerDeltaCount', () => { From 382c243e78893814cb751eaf0fc09b35caf4e65b Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Sat, 11 Jul 2026 23:56:16 +0800 Subject: [PATCH 4/4] test(e2e): widen keyless boot-smoke kill budgets to 30s MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The keyless Loader-path smokes killed the child 10s after spawn, but a loaded CI e2e runner routinely needs longer just to boot the unbuilt tsx tree: on this branch's run the coding-agent smoke burned both retries and failed at 30s wall-clock, and the sibling smokes passed only on retry x2 (master's latest run shows the same near-misses). The budget guards against a HANG, not slowness — raise kill to 30s and the vitest test timeout to 45s so a slow boot no longer masquerades as one. --- .../tests/code-mode-keyless-smoke.e2e.ts | 6 +++--- examples/coding-agent/tests/keyless-smoke.e2e.ts | 6 +++--- examples/cordis-agent/tests/keyless-smoke.e2e.ts | 6 +++--- examples/echo-agent/tests/echo.e2e.ts | 12 ++++++------ 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/examples/coding-agent/tests/code-mode-keyless-smoke.e2e.ts b/examples/coding-agent/tests/code-mode-keyless-smoke.e2e.ts index 7894e9ff9f..6c0c6afd3a 100644 --- a/examples/coding-agent/tests/code-mode-keyless-smoke.e2e.ts +++ b/examples/coding-agent/tests/code-mode-keyless-smoke.e2e.ts @@ -67,8 +67,8 @@ async function bootAndEof(): Promise<{ stdout: string; code: number }> { const timer = setTimeout(() => { proc.kill('SIGKILL') - reject(new Error(`code-mode overlay did not exit within 10s. stdout:\n${stdout}\nstderr:\n${stderr}`)) - }, 10_000) + reject(new Error(`code-mode overlay did not exit within 30s. stdout:\n${stdout}\nstderr:\n${stderr}`)) + }, 30_000) proc.on('exit', (code) => { clearTimeout(timer) @@ -87,5 +87,5 @@ describe('code-mode overlay keyless smoke (real code-mode.cordis.yml via the Loa const { stdout, code } = await bootAndEof() expect(code).toBe(0) expect(stdout).toContain('code-mode agent ready.') - }, 15_000) + }, 45_000) }) diff --git a/examples/coding-agent/tests/keyless-smoke.e2e.ts b/examples/coding-agent/tests/keyless-smoke.e2e.ts index b4d73f4d90..219144e904 100644 --- a/examples/coding-agent/tests/keyless-smoke.e2e.ts +++ b/examples/coding-agent/tests/keyless-smoke.e2e.ts @@ -78,8 +78,8 @@ async function bootAndEof(): Promise<{ stdout: string; code: number }> { const timer = setTimeout(() => { proc.kill('SIGKILL') - reject(new Error(`coding-agent did not exit within 10s. stdout:\n${stdout}\nstderr:\n${stderr}`)) - }, 10_000) + reject(new Error(`coding-agent did not exit within 30s. stdout:\n${stdout}\nstderr:\n${stderr}`)) + }, 30_000) proc.on('exit', (code) => { clearTimeout(timer) @@ -98,5 +98,5 @@ describe('coding-agent keyless smoke (real cordis.yml via the Loader)', () => { const { stdout, code } = await bootAndEof() expect(code).toBe(0) expect(stdout).toContain('agent REPL ready.') - }, 15_000) + }, 45_000) }) diff --git a/examples/cordis-agent/tests/keyless-smoke.e2e.ts b/examples/cordis-agent/tests/keyless-smoke.e2e.ts index b37ea8d83e..1211b4dce5 100644 --- a/examples/cordis-agent/tests/keyless-smoke.e2e.ts +++ b/examples/cordis-agent/tests/keyless-smoke.e2e.ts @@ -70,8 +70,8 @@ async function bootAndEof(): Promise<{ stdout: string; code: number }> { const timer = setTimeout(() => { proc.kill('SIGKILL') - reject(new Error(`cordis-agent did not exit within 10s. stdout:\n${stdout}\nstderr:\n${stderr}`)) - }, 10_000) + reject(new Error(`cordis-agent did not exit within 30s. stdout:\n${stdout}\nstderr:\n${stderr}`)) + }, 30_000) proc.on('exit', (code) => { clearTimeout(timer) @@ -90,5 +90,5 @@ describe('cordis-agent keyless smoke (real cordis.yml via the Loader)', () => { const { stdout, code } = await bootAndEof() expect(code).toBe(0) expect(stdout).toContain('cordis-agent ready.') - }, 15_000) + }, 45_000) }) diff --git a/examples/echo-agent/tests/echo.e2e.ts b/examples/echo-agent/tests/echo.e2e.ts index b446b2b191..0a2bb36288 100644 --- a/examples/echo-agent/tests/echo.e2e.ts +++ b/examples/echo-agent/tests/echo.e2e.ts @@ -51,7 +51,7 @@ afterEach(async () => { /** * Boot echo-agent, write `lines` to its stdin, close stdin, and resolve with * the full stdout once the process exits (the stdio UI exits on EOF after the - * agent settles). Rejects on a non-zero exit or a 10s timeout. + * agent settles). Rejects on a non-zero exit or a 30s timeout. */ async function runEcho(lines: string[]): Promise<{ stdout: string; code: number }> { workdir = await mkdtemp(join(tmpdir(), 'echo-smoke-')) @@ -84,8 +84,8 @@ async function runEcho(lines: string[]): Promise<{ stdout: string; code: number const timer = setTimeout(() => { proc.kill('SIGKILL') - reject(new Error(`echo-agent did not exit within 10s. stdout:\n${stdout}\nstderr:\n${stderr}`)) - }, 10_000) + reject(new Error(`echo-agent did not exit within 30s. stdout:\n${stdout}\nstderr:\n${stderr}`)) + }, 30_000) proc.on('exit', (code) => { clearTimeout(timer) @@ -105,19 +105,19 @@ describe('echo-agent keyless smoke (real cordis.yml via the Loader)', () => { const { stdout, code } = await runEcho([]) expect(code).toBe(0) expect(stdout).toContain('echo-agent ready.') - }, 15_000) + }, 45_000) it('runs the echo tool round-trip for an "echo …" line', async () => { const { stdout } = await runEcho(['echo hello world']) // mock-llm.ts emits a tool-call for the echo tool; echo-tool.ts uppercases. expect(stdout).toContain('[tool call] echo') expect(stdout).toContain('[tool result] ECHO: HELLO WORLD') - }, 15_000) + }, 45_000) it('streams a direct canned reply for a non-echo line', async () => { const { stdout } = await runEcho(['just chatting']) // The direct-response branch of mock-llm.ts quotes the input back. expect(stdout).toContain('just chatting') expect(stdout).not.toContain('[tool call]') - }, 15_000) + }, 45_000) })