From 7ef21239caeb9a554bf3f407f00090d263ed5627 Mon Sep 17 00:00:00 2001 From: kingwl Date: Wed, 15 Jul 2026 21:26:36 +0800 Subject: [PATCH 1/9] feat(session): opt-in packed chunk rows in the JSONL log Providers stream token-sized deltas, so a session log stores hundreds of near-identical assistant/chunk lines whose JSON envelopes dwarf their payloads (~56x measured on a real DeepSeek session, 73% of file bytes). Add a lossless storage codec to dsh-session: packChunkRuns() folds each run of >=3 consecutive same-block delta chunks into one storage row -- text-chunks / reasoning-chunks / tool-call-chunks, bare slash-less tags like the header line's 'session' so rows cannot be confused with session events -- and decodeStorageRecord() expands rows back to the exact original events (seq0/time0 + dt gap array reconstruct every member's seq/time; tool-call rows carry the run-constant id/name). The encoder whitelists exact shapes and stores anything unrecognized verbatim; the decoder validates row-tagged values and fails loud on malformation. The JSONL backend gains a packChunks config (default false). Writing packs only when enabled -- default-off output stays byte-identical to the previous layout, so snapshot goldens are untouched. Reading is layout-blind: scanLog always decodes rows and now checks seq contiguity with a cursor instead of the line index, so packed, unpacked, and mixed files all load identically. Fixture readers (llm-replay parseSessionLog, acp-snapshot normalizeSessionLog) share the codec; the normalizer zeroes a row's time0/dt exactly like an event's time. The two demo bundles plumb packChunks from cordis.yml to the backend. Measured on a real coding session: 105 KB -> 42 KB (-60%), 475 lines -> 74, with reasoning/tool-call heavy sessions saving the most. Covered by example + fast-check round-trip codec tests, backend packed/mixed/torn- tail specs, and an end-to-end demo run loading a packed log through a default-config backend. --- docs/config-catalog.md | 17 +- docs/cordis-catalog/events.md | 8 +- docs/cordis-catalog/services.md | 2 +- docs/core-data-structures/session.md | 2 +- docs/event-producer-consumer.md | 8 +- packages/core/session/README.md | 4 + packages/core/session/src/chunk-rows.ts | 326 ++++++++++++++++++ packages/core/session/src/index.ts | 2 + .../core/session/tests/chunk-rows.spec.ts | 208 +++++++++++ packages/examples/acp-demo/README.md | 1 + packages/examples/acp-demo/src/index.ts | 8 +- packages/examples/stdio-demo/README.md | 1 + packages/examples/stdio-demo/src/index.ts | 8 +- .../session-persistence-jsonl/README.md | 6 +- .../session-persistence-jsonl/src/format.ts | 74 ++-- .../session-persistence-jsonl/src/index.ts | 22 +- .../tests/jsonl.spec.ts | 115 +++++- .../support/acp-snapshot/src/normalize.ts | 13 +- .../acp-snapshot/tests/normalize.spec.ts | 21 ++ packages/support/llm-replay/src/index.ts | 8 +- .../llm-replay/tests/llm-replay.spec.ts | 13 + 21 files changed, 816 insertions(+), 51 deletions(-) create mode 100644 packages/core/session/src/chunk-rows.ts create mode 100644 packages/core/session/tests/chunk-rows.spec.ts diff --git a/docs/config-catalog.md b/docs/config-catalog.md index af8ef0832c..4ebe48c045 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -50,6 +50,8 @@ export interface Config { tools?: ToolsConfig /** Directory the JSONL session backend writes under. Defaults to `./.sessions`. */ persistenceRoot?: string + /** Write delta-chunk runs as packed storage rows (the JSONL backend's `packChunks`). Defaults to `false`. */ + packChunks?: boolean /** Skill registry, local-provider, and model-facing consumer config forwarded to agent-spine-demo. */ skills?: agentCore.SkillConfig } @@ -417,7 +419,7 @@ export interface Config { } ``` -Source: [`packages/support/llm-replay/src/index.ts:306`](../packages/support/llm-replay/src/index.ts) +Source: [`packages/support/llm-replay/src/index.ts:308`](../packages/support/llm-replay/src/index.ts) ## `@deepseek-ai/dsh-mcp-client` @@ -567,7 +569,7 @@ Source: [`packages/sandbox/sandbox-local/src/index.ts:20`](../packages/sandbox/s Requires: `sessions` ```ts config-catalog -/** Plugin config: where the JSONL backend keeps its session logs (`root` is required — no default). */ +/** Plugin config: where the JSONL backend keeps its session logs, and the packed-row write switch. */ export interface Config { /** * Root directory for all session files. Required (no default): a default of @@ -575,6 +577,15 @@ export interface Config { * (bash calls, subprocesses). Sessions group under per-cwd subdirectories. */ root: string + /** + * Write runs of consecutive `assistant/chunk` delta events as packed + * `text-chunks`/`reasoning-chunks`/`tool-call-chunks` rows (lossless, + * ~60% smaller logs measured on a real session). Off by default while + * snapshot fixtures stay in the one-event-per-line layout: recording with + * packing on rewrites every golden `session.jsonl`. READING packed rows is + * unconditional — a log's layout never depends on this switch. + */ + packChunks?: boolean } ``` @@ -699,6 +710,8 @@ export interface Config { tools?: ToolsConfig /** Directory the JSONL session backend writes under. Defaults to `./.sessions`. */ persistenceRoot?: string + /** Write delta-chunk runs as packed storage rows (the JSONL backend's `packChunks`). Defaults to `false`. */ + packChunks?: boolean /** stdin-chat banner printed once on start. Defaults to `'ready.'`. */ welcome?: string /** Skill registry, local-provider, and model-facing consumer config forwarded to agent-spine-demo. */ diff --git a/docs/cordis-catalog/events.md b/docs/cordis-catalog/events.md index ebd7b75bc0..de755f2cee 100644 --- a/docs/cordis-catalog/events.md +++ b/docs/cordis-catalog/events.md @@ -245,7 +245,7 @@ Creation announcement during session publication. A synchronous throw vetoes and 'session/created'(this: Scoped, session: Session): void ``` -Source: [`packages/core/session/src/index.ts:47`](../../packages/core/session/src/index.ts) +Source: [`packages/core/session/src/index.ts:49`](../../packages/core/session/src/index.ts) ### `session/disposed` — emit @@ -255,7 +255,7 @@ Emitted once when an announced session leaves the store, including publication r 'session/disposed'(this: Scoped, session: Session): void ``` -Source: [`packages/core/session/src/index.ts:57`](../../packages/core/session/src/index.ts) +Source: [`packages/core/session/src/index.ts:59`](../../packages/core/session/src/index.ts) ### `session/event` — emit @@ -267,7 +267,7 @@ Post-commit, fire-and-forget append feed. The listener snapshot resolves before Types: [SessionEvent](../core-data-structures/core.md) -Source: [`packages/core/session/src/index.ts:69`](../../packages/core/session/src/index.ts) +Source: [`packages/core/session/src/index.ts:71`](../../packages/core/session/src/index.ts) ### `session/flush` — parallel @@ -277,7 +277,7 @@ Awaited parallel durability checkpoint: every listener runs and the caller await 'session/flush'(this: Scoped, session: Session): Promise | void ``` -Source: [`packages/core/session/src/index.ts:79`](../../packages/core/session/src/index.ts) +Source: [`packages/core/session/src/index.ts:81`](../../packages/core/session/src/index.ts) ## `subagent/*` diff --git a/docs/cordis-catalog/services.md b/docs/cordis-catalog/services.md index 83888c02e1..be2eec01ac 100644 --- a/docs/cordis-catalog/services.md +++ b/docs/cordis-catalog/services.md @@ -200,7 +200,7 @@ list(): Session[] fork(source: SessionForkSource, boundary?: number, childSessionId?: SessionId): Session ``` -Source: [`packages/core/session/src/index.ts:564`](../../packages/core/session/src/index.ts) +Source: [`packages/core/session/src/index.ts:566`](../../packages/core/session/src/index.ts) ## `ctx.skills` — `SkillService` diff --git a/docs/core-data-structures/session.md b/docs/core-data-structures/session.md index d8292c49b3..188aa5084b 100644 --- a/docs/core-data-structures/session.md +++ b/docs/core-data-structures/session.md @@ -305,6 +305,6 @@ The hook bridges' `hook/invoked` / `hook/result` provenance pairs (from `@deepse ## Durability contract -What a persistence backend relies on: the durable log persists every event verbatim, **including** `assistant/chunk` — `seq` must stay contiguous, so chunks cannot be filtered out of the canonical log. All `event.data` must be JSON-serializable; `Session.append` enforces this at the source (throwing on non-serializable data), so a bad event never enters the log and `session.events` always equals what a backend can persist. Adding an event type that carries non-serializable data, or that breaks the turn/step nesting the invariants plugin checks, is a breaking change to the on-disk format. +What a persistence backend relies on: the durable log persists every event losslessly, **including** `assistant/chunk` — `seq` must stay contiguous, so chunks cannot be filtered out of the canonical log. A backend may choose its own storage encoding for an event batch as long as `load` returns the exact appended events (the JSONL backend's opt-in packed chunk rows are such an encoding — see [persistence.md](persistence.md)). All `event.data` must be JSON-serializable; `Session.append` enforces this at the source (throwing on non-serializable data), so a bad event never enters the log and `session.events` always equals what a backend can persist. Adding an event type that carries non-serializable data, or that breaks the turn/step nesting the invariants plugin checks, is a breaking change to the on-disk format. The backends that consume this contract are on [persistence.md](persistence.md). diff --git a/docs/event-producer-consumer.md b/docs/event-producer-consumer.md index ec2da1408d..b9d8a2a2e6 100644 --- a/docs/event-producer-consumer.md +++ b/docs/event-producer-consumer.md @@ -25,10 +25,10 @@ This matrix shows which packages dispatch each harness-owned event and which pac | `fs/observed` | `emit` | [`packages/fs/fs/src/index.ts:68`](../packages/fs/fs/src/index.ts) | [`tool-fs`](../packages/fs/tool-fs) (`emit`) | [`fs-policy`](../packages/fs/fs-policy) | | `fs/write-intent` | `waterfall` | [`packages/fs/fs/src/index.ts:51`](../packages/fs/fs/src/index.ts) | [`tool-fs`](../packages/fs/tool-fs) (`waterfall`) | [`fs-policy`](../packages/fs/fs-policy) | | `llm/stream` | `waterfall` | [`packages/llm/llm/src/index.ts:39`](../packages/llm/llm/src/index.ts) | [`llm`](../packages/llm/llm) (`waterfall`) | [`invariants`](../packages/support/invariants), [`llm-replay`](../packages/support/llm-replay) | -| `session/created` | `emit` | [`packages/core/session/src/index.ts:47`](../packages/core/session/src/index.ts) | [`session`](../packages/core/session) (`events.dispatch`) | [`invariants`](../packages/support/invariants), [`jsonrpc`](../packages/ui/jsonrpc), [`session-persistence`](../packages/session-persistence/session-persistence) | -| `session/disposed` | `emit` | [`packages/core/session/src/index.ts:57`](../packages/core/session/src/index.ts) | [`session`](../packages/core/session) (`events.dispatch`) | - | -| `session/event` | `emit` | [`packages/core/session/src/index.ts:69`](../packages/core/session/src/index.ts) | [`session`](../packages/core/session) (`events.dispatch`) | [`acp`](../packages/ui/acp), [`invariants`](../packages/support/invariants), [`jsonrpc`](../packages/ui/jsonrpc), [`session-persistence`](../packages/session-persistence/session-persistence), [`stdio`](../packages/ui/stdio) | -| `session/flush` | `parallel` | [`packages/core/session/src/index.ts:79`](../packages/core/session/src/index.ts) | [`session`](../packages/core/session) (`events.dispatch`) | [`session-persistence`](../packages/session-persistence/session-persistence) | +| `session/created` | `emit` | [`packages/core/session/src/index.ts:49`](../packages/core/session/src/index.ts) | [`session`](../packages/core/session) (`events.dispatch`) | [`invariants`](../packages/support/invariants), [`jsonrpc`](../packages/ui/jsonrpc), [`session-persistence`](../packages/session-persistence/session-persistence) | +| `session/disposed` | `emit` | [`packages/core/session/src/index.ts:59`](../packages/core/session/src/index.ts) | [`session`](../packages/core/session) (`events.dispatch`) | - | +| `session/event` | `emit` | [`packages/core/session/src/index.ts:71`](../packages/core/session/src/index.ts) | [`session`](../packages/core/session) (`events.dispatch`) | [`acp`](../packages/ui/acp), [`invariants`](../packages/support/invariants), [`jsonrpc`](../packages/ui/jsonrpc), [`session-persistence`](../packages/session-persistence/session-persistence), [`stdio`](../packages/ui/stdio) | +| `session/flush` | `parallel` | [`packages/core/session/src/index.ts:81`](../packages/core/session/src/index.ts) | [`session`](../packages/core/session) (`events.dispatch`) | [`session-persistence`](../packages/session-persistence/session-persistence) | | `subagent/end` | `emit` | [`packages/subagent/subagent/src/index.ts:92`](../packages/subagent/subagent/src/index.ts) | [`subagent`](../packages/subagent/subagent) (`events.dispatch`) | [`hooks-claude`](../packages/hooks/hooks-claude), [`jsonrpc`](../packages/ui/jsonrpc) | | `subagent/provider-added` | `emit` | [`packages/subagent/subagent/src/index.ts:66`](../packages/subagent/subagent/src/index.ts) | [`subagent`](../packages/subagent/subagent) (`emit`) | [`tool-subagent`](../packages/subagent/tool-subagent) | | `subagent/provider-removed` | `emit` | [`packages/subagent/subagent/src/index.ts:72`](../packages/subagent/subagent/src/index.ts) | [`subagent`](../packages/subagent/subagent) (`events.dispatch`) | [`tool-subagent`](../packages/subagent/tool-subagent) | diff --git a/packages/core/session/README.md b/packages/core/session/README.md index 7e201ab647..0db0a72e80 100644 --- a/packages/core/session/README.md +++ b/packages/core/session/README.md @@ -44,6 +44,10 @@ Plain class (not a Cordis Service). Create via `ctx.sessions.create()`. Durable values need one accepted representation, not a check followed by a second read. `isJsonValue(value)` is the boolean predicate; `snapshotJsonValue(value)` recursively validates and copies a plain value in one pass, returning `undefined` for invalid input and propagating a throwing getter. The snapshot helper accepts finite JSON numbers except `-0` (JSON rewrites it to `0`), dense ordinary arrays, and plain or null-prototype objects; it rejects cycles, unsupported scalars, and exotic prototypes before normalization. +### Chunk-row storage codec (`chunk-rows.ts`) + +Providers stream token-sized deltas, so a raw log stores hundreds of `assistant/chunk` lines whose JSON envelopes dwarf their payloads. `packChunkRuns(events)` packs each run of ≥3 consecutive same-block delta chunks into one storage row — `text-chunks`, `reasoning-chunks`, or `tool-call-chunks` (bare slash-less tags: storage vocabulary, not `SessionEventMap` members) — and `decodeStorageRecord(value)` expands a parsed line back into its exact events (`seq0`/`time0` + per-member `dt` gaps reconstruct every `seq`/`time`). The encoder whitelists exact shapes and stores anything unrecognized verbatim; the decoder validates row-tagged values and throws on malformation. Owned here so the JSONL backend and the fixture readers (`dsh-llm-replay`, `dsh-acp-snapshot`) share one codec; the write-side switch is the backend's `packChunks` config. + ### Surface types - `SurfaceOp` — how a surface node entered the linked list: `'append'` (normal tail append) or `{ op: 'replace', start, end }` (replace nodes from `start` through `end` inclusive — both must be valid surface node seqs; `start === end` replaces a single node). Used by compaction to shadow old nodes without deleting them. diff --git a/packages/core/session/src/chunk-rows.ts b/packages/core/session/src/chunk-rows.ts new file mode 100644 index 0000000000..c42532946c --- /dev/null +++ b/packages/core/session/src/chunk-rows.ts @@ -0,0 +1,326 @@ +/** + * Lossless storage packing for `assistant/chunk` delta runs. Providers stream + * token-sized deltas, so a log stores hundreds of near-identical event lines + * whose JSON envelopes dwarf their payloads (~56× measured on a real DeepSeek + * session). This module packs each run of consecutive same-block delta chunks + * into ONE storage row — `text-chunks`, `reasoning-chunks`, or + * `tool-call-chunks` — and expands rows back to the exact original events. + * + * Storage rows are a durable-encoding vocabulary, NOT session events: they + * never enter `Session.events`, have no `SessionEventMap` entry, and use bare + * (slash-less) type tags so a reader cannot confuse them with the event + * taxonomy (precedent: the JSONL header line's `session` tag). The encoder + * whitelists exact shapes — anything it does not fully recognize is stored + * verbatim, so unknown fields or future chunk variants lose compression, never + * data. The decoder validates before expanding and fails loud on a malformed + * row-tagged value instead of silently dropping a whole run. + * + * @module @deepseek-ai/dsh-session/chunk-rows + */ + +import { CallId, assertNever } from '@deepseek-ai/dsh-llm' +import type { StreamChunk } from '@deepseek-ai/dsh-llm' +import type { SessionEvent } from './types.ts' + +/** The chunk kinds that may pack; block boundaries, usage, and finish chunks always stay one event per line. */ +type DeltaKind = 'text-delta' | 'reasoning-delta' | 'tool-call-delta' + +/** A run member: an `assistant/chunk` event whose exact shape the encoder whitelisted. */ +type DeltaEvent = SessionEvent<'assistant/chunk'> + +/** + * Fields shared by every packed run: placement, block correlation, and member + * timestamps as gaps. Member `k` reconstructs as seq `seq0 + k` and time + * `time0` plus the first `k` gaps; a gap may be negative when the wall clock + * stepped backwards between events. + */ +interface RunDataBase { + turn: number + step: number + /** The stream block index every member shares. */ + index: number + /** Epoch-ms gaps between consecutive members; length is one less than the member count. */ + dt: number[] +} + +/** Payload of a `text-chunks`/`reasoning-chunks` row: one entry per member, never joined — token boundaries are data. */ +interface TextRunData extends RunDataBase { + texts: string[] +} + +/** Payload of a `tool-call-chunks` row: the run-constant call identity plus each member's raw arguments fragment. */ +interface ToolCallRunData extends RunDataBase { + id: CallId + /** Present iff every member carried it, with one uniform value (a mixed run never packs). */ + name?: string + args: string[] +} + +/** + * A packed run of consecutive delta chunk events, discriminated on `type`. + * `seq0`/`time0` anchor the first member; text and reasoning rows share the + * {@link TextRunData} payload, tool-call rows carry {@link ToolCallRunData}. + */ +export type ChunkRow = + | { type: 'text-chunks'; seq0: number; time0: number; data: TextRunData } + | { type: 'reasoning-chunks'; seq0: number; time0: number; data: TextRunData } + | { type: 'tool-call-chunks'; seq0: number; time0: number; data: ToolCallRunData } + +/** One durable log line's JSON value: a session event verbatim, or a packed chunk row. */ +export type StorageRecord = SessionEvent | ChunkRow + +/** + * Minimum members before a run packs. Below it a row's envelope rivals the + * event lines it replaces. A format constant, not a tunable: both layouts + * decode identically, so changing it never invalidates stored logs. + */ +const MIN_RUN = 3 + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null +} + +/** Exact-key check: `value` has every key in `keys` and nothing else. */ +function hasExactKeys(value: object, keys: readonly string[]): boolean { + return Object.keys(value).length === keys.length && keys.every(k => Object.hasOwn(value, k)) +} + +/** + * Classify an event for packing: its delta kind when the ENTIRE shape + * (envelope, data, chunk — exact keys, primitive types, integer seq/time) is + * whitelisted, else `undefined` (store verbatim). Inputs come from live typed + * appends AND parsed fixture files, so the checks are structural, not + * type-trusted. Integer times keep gap encoding exact: a fractional time would + * reconstruct through float subtraction/addition, which need not round-trip. + */ +function classify(event: SessionEvent): DeltaKind | undefined { + if (event.type !== 'assistant/chunk') return undefined + if (!hasExactKeys(event, ['type', 'seq', 'time', 'data'])) return undefined + if (!Number.isSafeInteger(event.seq) || event.seq < 0 || !Number.isSafeInteger(event.time)) return undefined + const data: unknown = event.data + if (!isRecord(data) || !hasExactKeys(data, ['turn', 'step', 'chunk'])) return undefined + if (typeof data.turn !== 'number' || typeof data.step !== 'number') return undefined + const chunk = data.chunk + if (!isRecord(chunk) || typeof chunk.index !== 'number') return undefined + switch (chunk.type) { + case 'text-delta': + case 'reasoning-delta': + return hasExactKeys(chunk, ['type', 'index', 'text']) && typeof chunk.text === 'string' + ? chunk.type + : undefined + case 'tool-call-delta': { + const shapeOk = hasExactKeys(chunk, ['type', 'index', 'id', 'argumentsDelta']) + || (hasExactKeys(chunk, ['type', 'index', 'id', 'name', 'argumentsDelta']) && typeof chunk.name === 'string') + return shapeOk && typeof chunk.id === 'string' && typeof chunk.argumentsDelta === 'string' + ? chunk.type + : undefined + } + // Whitelist fall-through over parsed data: block-start/end, usage, finish, + // and any future chunk variant stay one event per line. + default: + return undefined + } +} + +/** The tool-call fields of a whitelisted delta chunk (only after {@link classify} returned `'tool-call-delta'`). */ +function toolCallOf(event: DeltaEvent): { id: string; name?: string } { + return event.data.chunk as { id: string; name?: string } +} + +/** The block index of a whitelisted delta chunk (not every {@link StreamChunk} variant carries one). */ +function indexOf(event: DeltaEvent): number { + return (event.data.chunk as { index: number }).index +} + +/** Whether `next` extends a run ending in `prev` (same kind already checked by the caller). */ +function continues(prev: DeltaEvent, next: DeltaEvent, kind: DeltaKind): boolean { + if (next.seq !== prev.seq + 1) return false + if (next.data.turn !== prev.data.turn || next.data.step !== prev.data.step) return false + if (indexOf(next) !== indexOf(prev)) return false + if (kind !== 'tool-call-delta') return true + const a = toolCallOf(prev) + const b = toolCallOf(next) + // `name` must match in presence AND value — a mixed run is not representable. + return a.id === b.id && Object.hasOwn(a, 'name') === Object.hasOwn(b, 'name') && a.name === b.name +} + +/** Build the row for a completed run (`run.length >= MIN_RUN`, uniform per {@link continues}). */ +function buildRow(kind: DeltaKind, run: readonly DeltaEvent[]): ChunkRow { + const first = run[0] as DeltaEvent + const base = { + turn: first.data.turn, + step: first.data.step, + index: indexOf(first), + dt: run.slice(1).map((event, i) => event.time - (run[i] as DeltaEvent).time), + } + const envelope = { seq0: first.seq, time0: first.time } + if (kind === 'tool-call-delta') { + const call = toolCallOf(first) + return { + type: 'tool-call-chunks', + ...envelope, + data: { + ...base, + id: CallId(call.id), + ...Object.hasOwn(call, 'name') ? { name: call.name as string } : {}, + args: run.map(event => (event.data.chunk as { argumentsDelta: string }).argumentsDelta), + }, + } + } + const data = { ...base, texts: run.map(event => (event.data.chunk as { text: string }).text) } + return kind === 'text-delta' + ? { type: 'text-chunks', ...envelope, data } + : { type: 'reasoning-chunks', ...envelope, data } +} + +/** + * Pack an event batch for storage: each run of at least {@link MIN_RUN} + * consecutive whitelisted same-kind, same-block delta chunk events becomes one + * {@link ChunkRow}; every other event passes through verbatim, in order. + * Pure and stateless — safe over any array, including a batch whose runs were + * split by flush boundaries (the split runs simply pack per batch). + * + * @param events - the batch to encode, in log order. + * @returns the storage records to write, one JSONL line each. + */ +export function packChunkRuns(events: readonly SessionEvent[]): StorageRecord[] { + const out: StorageRecord[] = [] + let kind: DeltaKind | undefined + let run: DeltaEvent[] = [] + const flush = (): void => { + if (kind !== undefined && run.length >= MIN_RUN) out.push(buildRow(kind, run)) + else out.push(...run) + kind = undefined + run = [] + } + for (const event of events) { + const k = classify(event) + if (k === undefined) { + flush() + out.push(event) + continue + } + const delta = event as DeltaEvent + const last = run[run.length - 1] + if (k === kind && last !== undefined && continues(last, delta, k)) { + run.push(delta) + continue + } + flush() + kind = k + run = [delta] + } + flush() + return out +} + +/** Throw the uniform malformed-row diagnostic. */ +function malformed(tag: string, why: string): never { + throw new Error(`malformed ${tag} storage row: ${why}`) +} + +/** Validate the shared run-data fields and the payload/dt arity. */ +function validateRunData(tag: string, data: Record, payloadKey: 'texts' | 'args'): void { + if (typeof data.turn !== 'number' || typeof data.step !== 'number' || typeof data.index !== 'number') { + malformed(tag, 'turn/step/index must be numbers') + } + const payload = data[payloadKey] + if (!Array.isArray(payload) || payload.length === 0 || payload.some(entry => typeof entry !== 'string')) { + malformed(tag, `${payloadKey} must be a non-empty string array`) + } + const dt = data.dt + if (!Array.isArray(dt) || dt.some(gap => typeof gap !== 'number' || !Number.isFinite(gap))) { + malformed(tag, 'dt must be an array of finite numbers') + } + if (dt.length !== payload.length - 1) { + malformed(tag, `dt length ${dt.length} does not match ${payload.length} members`) + } +} + +/** Validate a row-tagged parsed value's envelope and data, throwing on any malformation. */ +function validateRow(value: Record, tag: ChunkRow['type']): ChunkRow { + if (!hasExactKeys(value, ['type', 'seq0', 'time0', 'data'])) { + malformed(tag, 'envelope must be exactly {type, seq0, time0, data}') + } + if (!Number.isSafeInteger(value.seq0) || (value.seq0 as number) < 0) { + malformed(tag, 'seq0 must be a non-negative safe integer') + } + if (typeof value.time0 !== 'number' || !Number.isFinite(value.time0)) { + malformed(tag, 'time0 must be a finite number') + } + const data = value.data + if (!isRecord(data)) malformed(tag, 'data must be an object') + if (tag === 'tool-call-chunks') { + const withName = hasExactKeys(data, ['turn', 'step', 'index', 'id', 'name', 'dt', 'args']) + if (!withName && !hasExactKeys(data, ['turn', 'step', 'index', 'id', 'dt', 'args'])) { + malformed(tag, 'data must be exactly {turn, step, index, id, name?, dt, args}') + } + if (typeof data.id !== 'string' || (withName && typeof data.name !== 'string')) { + malformed(tag, 'id (and name when present) must be strings') + } + validateRunData(tag, data, 'args') + } else { + if (!hasExactKeys(data, ['turn', 'step', 'index', 'dt', 'texts'])) { + malformed(tag, 'data must be exactly {turn, step, index, dt, texts}') + } + validateRunData(tag, data, 'texts') + } + return value as unknown as ChunkRow +} + +/** Expand a validated row back into its exact original events, in order. */ +function expandRow(row: ChunkRow): SessionEvent[] { + const members = row.type === 'tool-call-chunks' ? row.data.args : row.data.texts + const events: SessionEvent[] = [] + let time = row.time0 + for (let k = 0; k < members.length; k++) { + if (k > 0) time += row.data.dt[k - 1] as number + let chunk: StreamChunk + switch (row.type) { + case 'text-chunks': + chunk = { type: 'text-delta', index: row.data.index, text: members[k] as string } + break + case 'reasoning-chunks': + chunk = { type: 'reasoning-delta', index: row.data.index, text: members[k] as string } + break + case 'tool-call-chunks': + chunk = { + type: 'tool-call-delta', + index: row.data.index, + id: row.data.id, + ...Object.hasOwn(row.data, 'name') ? { name: row.data.name as string } : {}, + argumentsDelta: members[k] as string, + } + break + /* v8 ignore next 2 -- validateRow only returns the three row tags */ + default: + return assertNever(row, 'chunk-rows expandRow') + } + events.push({ + type: 'assistant/chunk', + seq: row.seq0 + k, + time, + data: { turn: row.data.turn, step: row.data.step, chunk }, + }) + } + return events +} + +/** + * Decode one parsed JSONL line value into the session event(s) it stores. + * Chunk-row-tagged values validate and expand (a malformed row throws — it is + * corrupt storage, and treating it as an event would silently drop a whole + * run); every other value passes through as a single event, unvalidated, + * exactly as readers treated event lines before packing existed. + * + * @param value - one line's `JSON.parse` result. + * @returns the stored events, in log order. + */ +export function decodeStorageRecord(value: unknown): SessionEvent[] { + if (!isRecord(value)) return [value as SessionEvent] + const tag = value.type + if (tag !== 'text-chunks' && tag !== 'reasoning-chunks' && tag !== 'tool-call-chunks') { + return [value as SessionEvent] + } + return expandRow(validateRow(value, tag)) +} diff --git a/packages/core/session/src/index.ts b/packages/core/session/src/index.ts index 89f1627445..a49bdeb05e 100644 --- a/packages/core/session/src/index.ts +++ b/packages/core/session/src/index.ts @@ -22,6 +22,8 @@ export * from './types.ts' export { isJsonValue, snapshotJsonValue } from './json.ts' export type { JsonValue } from './json.ts' export { interruptedTurnClosers } from './repair.ts' +export { decodeStorageRecord, packChunkRuns } from './chunk-rows.ts' +export type { ChunkRow, StorageRecord } from './chunk-rows.ts' export type { SurfaceFoldReplacement, SurfaceFoldResult, SurfaceNode } from './surface.ts' export { foldSurface, isSurfaceEvent, isSurfaceEligibleType } from './surface.ts' export { isToolPairingBalanced } from './tool-pairing.ts' diff --git a/packages/core/session/tests/chunk-rows.spec.ts b/packages/core/session/tests/chunk-rows.spec.ts new file mode 100644 index 0000000000..9eb9173bca --- /dev/null +++ b/packages/core/session/tests/chunk-rows.spec.ts @@ -0,0 +1,208 @@ +/** + * Chunk-row codec tests: pack/expand round-trip losslessness (example-based and + * property-based), run-boundary rules, whitelist fall-through, and decoder + * validation failures. + */ + +import { describe, expect, it } from 'vitest' +import fc from 'fast-check' +import { CallId } from '@deepseek-ai/dsh-llm' +import type { StreamChunk } from '@deepseek-ai/dsh-llm' +import { decodeStorageRecord, packChunkRuns } from '@deepseek-ai/dsh-session' +import type { ChunkRow, SessionEvent, StorageRecord } from '@deepseek-ai/dsh-session' + +/** Build an `assistant/chunk` event with the exact live-append shape. */ +function chunkEvent(seq: number, time: number, chunk: StreamChunk, turn = 1, step = 1): SessionEvent { + return { type: 'assistant/chunk', seq, time, data: { turn, step, chunk } } +} + +/** Sequential delta events (contiguous seqs, fixed 10ms gaps) of one kind. */ +function deltaRun(kind: 'text-delta' | 'reasoning-delta', count: number, seq0 = 0, index = 0): SessionEvent[] { + return Array.from({ length: count }, (_, k) => + chunkEvent(seq0 + k, 1000 + 10 * k, { type: kind, index, text: `t${k}` })) +} + +/** Decode a packed record list back to a flat event list. */ +function decodeAll(records: readonly StorageRecord[]): SessionEvent[] { + return records.flatMap(record => decodeStorageRecord(JSON.parse(JSON.stringify(record)))) +} + +describe('packChunkRuns', () => { + it('packs a text-delta run into one text-chunks row and round-trips it', () => { + const events = deltaRun('text-delta', 5) + const packed = packChunkRuns(events) + expect(packed).toHaveLength(1) + const row = packed[0] as ChunkRow + expect(row.type).toBe('text-chunks') + expect(row.seq0).toBe(0) + expect(row.time0).toBe(1000) + expect(row.data).toMatchObject({ turn: 1, step: 1, index: 0, dt: [10, 10, 10, 10], texts: ['t0', 't1', 't2', 't3', 't4'] }) + expect(decodeAll(packed)).toStrictEqual(events) + }) + + it('packs reasoning and tool-call runs under their own tags', () => { + const reasoning = deltaRun('reasoning-delta', 3) + const toolCall = [4, 5, 6].map(seq => + chunkEvent(seq, 1000 + seq, { type: 'tool-call-delta', index: 1, id: CallId('c1'), name: 'write', argumentsDelta: `a${seq}` })) + const packed = packChunkRuns([...reasoning, ...toolCall]) + expect(packed.map(r => (r as ChunkRow).type)).toStrictEqual(['reasoning-chunks', 'tool-call-chunks']) + const row = packed[1] as ChunkRow & { type: 'tool-call-chunks' } + expect(row.data).toMatchObject({ id: 'c1', name: 'write', args: ['a4', 'a5', 'a6'] }) + expect(decodeAll(packed)).toStrictEqual([...reasoning, ...toolCall]) + }) + + it('packs a name-less tool-call run and round-trips field absence', () => { + const events = [0, 1, 2].map(seq => + chunkEvent(seq, 1000, { type: 'tool-call-delta', index: 0, id: CallId('c1'), argumentsDelta: `a${seq}` })) + const packed = packChunkRuns(events) + expect(packed).toHaveLength(1) + expect(Object.hasOwn((packed[0] as ChunkRow).data, 'name')).toBe(false) + const decoded = decodeAll(packed) + expect(decoded).toStrictEqual(events) + expect(decoded.every(e => !Object.hasOwn((e.data as { chunk: object }).chunk, 'name'))).toBe(true) + }) + + it('leaves runs shorter than three events verbatim', () => { + const events = deltaRun('text-delta', 2) + expect(packChunkRuns(events)).toStrictEqual(events) + }) + + it('leaves non-delta chunks and non-chunk events verbatim between runs', () => { + const events: SessionEvent[] = [ + chunkEvent(0, 1000, { type: 'block-start', index: 0, blockType: 'text' }), + ...deltaRun('text-delta', 3, 1), + chunkEvent(4, 1040, { type: 'block-end', index: 0, block: { type: 'text', text: 't0t1t2' } }), + { type: 'step/end', seq: 5, time: 1050, data: { turn: 1, step: 1 } }, + ] + const packed = packChunkRuns(events) + expect(packed).toHaveLength(4) + expect((packed[1] as ChunkRow).type).toBe('text-chunks') + expect(decodeAll(packed)).toStrictEqual(events) + }) + + it.each([ + ['a seq gap', deltaRun('text-delta', 3).map((e, k) => ({ ...e, seq: k === 2 ? 9 : e.seq }))], + ['a kind switch', [...deltaRun('text-delta', 2), ...deltaRun('reasoning-delta', 1, 2)]], + ['a block-index switch', [...deltaRun('text-delta', 2), ...deltaRun('text-delta', 1, 2, 7)]], + ['a step switch', deltaRun('text-delta', 3).map((e, k) => k === 2 ? chunkEvent(e.seq, e.time, (e.data as { chunk: StreamChunk }).chunk, 1, 2) : e)], + ])('breaks a run on %s (both halves too short to pack)', (_label, events) => { + expect(packChunkRuns(events as SessionEvent[])).toStrictEqual(events) + }) + + it('breaks a tool-call run on call-id or name change', () => { + const call = (seq: number, id: string, name?: string): SessionEvent => + chunkEvent(seq, 1000, { type: 'tool-call-delta', index: 0, id: CallId(id), ...name !== undefined ? { name } : {}, argumentsDelta: 'a' }) + const idSwitch = [call(0, 'c1', 'w'), call(1, 'c1', 'w'), call(2, 'c2', 'w')] + expect(packChunkRuns(idSwitch)).toStrictEqual(idSwitch) + const namePresence = [call(0, 'c1', 'w'), call(1, 'c1', 'w'), call(2, 'c1')] + expect(packChunkRuns(namePresence)).toStrictEqual(namePresence) + }) + + it('stores an off-whitelist delta verbatim (extra field, bad type, fractional time)', () => { + const extraField = { ...chunkEvent(0, 1000, { type: 'text-delta', index: 0, text: 'x' }), surfaceOp: 'append' } + const badText = chunkEvent(1, 1001, { type: 'text-delta', index: 0, text: 7 as unknown as string }) + const fractionalTime = chunkEvent(2, 1001.5, { type: 'text-delta', index: 0, text: 'y' }) + const events = [extraField, badText, fractionalTime] as SessionEvent[] + expect(packChunkRuns(events)).toStrictEqual(events) + }) + + it('stores a delta with an off-whitelist data envelope verbatim (parsed-fixture shapes)', () => { + const mk = (seq: number, data: unknown): SessionEvent => + ({ type: 'assistant/chunk', seq, time: 1000, data } as SessionEvent) + const events = [ + mk(0, 'not-an-object'), + mk(1, { turn: 1, step: 1, chunk: { type: 'text-delta', index: 0, text: 'a' }, extra: 1 }), + mk(2, { turn: 'x', step: 1, chunk: { type: 'text-delta', index: 0, text: 'a' } }), + mk(3, { turn: 1, step: 1, chunk: 'not-an-object' }), + mk(4, { turn: 1, step: 1, chunk: { type: 'text-delta', index: 'x', text: 'a' } }), + mk(5, { turn: 1, step: 1, chunk: { type: 'tool-call-delta', index: 0, id: 7, argumentsDelta: 'a' } }), + mk(6, { turn: 1, step: 1, chunk: { type: 'tool-call-delta', index: 0, id: 'c', name: 7, argumentsDelta: 'a' } }), + ] + expect(packChunkRuns(events)).toStrictEqual(events) + }) +}) + +describe('decodeStorageRecord', () => { + it('passes non-row values through as single events, unvalidated', () => { + const event = { type: 'turn/start', seq: 0, time: 1, data: { turn: 1 } } + expect(decodeStorageRecord(event)).toStrictEqual([event]) + expect(decodeStorageRecord('junk')).toStrictEqual(['junk']) + expect(decodeStorageRecord(null)).toStrictEqual([null]) + }) + + it('reconstructs timestamps through negative dt gaps (clock stepped back)', () => { + const events = [ + chunkEvent(0, 1000, { type: 'text-delta', index: 0, text: 'a' }), + chunkEvent(1, 990, { type: 'text-delta', index: 0, text: 'b' }), + chunkEvent(2, 995, { type: 'text-delta', index: 0, text: 'c' }), + ] + expect(decodeAll(packChunkRuns(events))).toStrictEqual(events) + }) + + it.each([ + ['a non-object data', { type: 'text-chunks', seq0: 0, time0: 1, data: 'x' }], + ['an envelope with extra keys', { type: 'text-chunks', seq0: 0, time0: 1, data: { turn: 1, step: 1, index: 0, dt: [], texts: ['a'] }, extra: 1 }], + ['a negative seq0', { type: 'text-chunks', seq0: -1, time0: 1, data: { turn: 1, step: 1, index: 0, dt: [], texts: ['a'] } }], + ['a non-finite time0', { type: 'text-chunks', seq0: 0, time0: Infinity, data: { turn: 1, step: 1, index: 0, dt: [], texts: ['a'] } }], + ['a data shape mismatch', { type: 'text-chunks', seq0: 0, time0: 1, data: { turn: 1, step: 1, index: 0, dt: [], args: ['a'] } }], + ['a non-string member', { type: 'text-chunks', seq0: 0, time0: 1, data: { turn: 1, step: 1, index: 0, dt: [], texts: [7] } }], + ['an empty member list', { type: 'text-chunks', seq0: 0, time0: 1, data: { turn: 1, step: 1, index: 0, dt: [], texts: [] } }], + ['a dt arity mismatch', { type: 'text-chunks', seq0: 0, time0: 1, data: { turn: 1, step: 1, index: 0, dt: [1, 2], texts: ['a', 'b'] } }], + ['a non-finite dt gap', { type: 'text-chunks', seq0: 0, time0: 1, data: { turn: 1, step: 1, index: 0, dt: [NaN], texts: ['a', 'b'] } }], + ['a non-numeric turn', { type: 'text-chunks', seq0: 0, time0: 1, data: { turn: 'x', step: 1, index: 0, dt: [], texts: ['a'] } }], + ['a tool-call row without id', { type: 'tool-call-chunks', seq0: 0, time0: 1, data: { turn: 1, step: 1, index: 0, dt: [], args: ['a'] } }], + ['a tool-call row with non-string id', { type: 'tool-call-chunks', seq0: 0, time0: 1, data: { turn: 1, step: 1, index: 0, id: 7, dt: [], args: ['a'] } }], + ['a tool-call row with non-string name', { type: 'tool-call-chunks', seq0: 0, time0: 1, data: { turn: 1, step: 1, index: 0, id: 'c', name: 7, dt: [], args: ['a'] } }], + ])('throws on %s', (_label, row) => { + expect(() => decodeStorageRecord(row)).toThrow(/malformed .* storage row/) + }) +}) + +// --- Property: pack∘decode is the identity over arbitrary event batches --- + +const deltaChunkArb: fc.Arbitrary = fc.oneof( + fc.record({ type: fc.constant<'text-delta'>('text-delta'), index: fc.nat(2), text: fc.string() }), + fc.record({ type: fc.constant<'reasoning-delta'>('reasoning-delta'), index: fc.nat(2), text: fc.string() }), + fc.record({ + type: fc.constant<'tool-call-delta'>('tool-call-delta'), + index: fc.nat(2), + id: fc.constantFrom(CallId('c1'), CallId('c2')), + argumentsDelta: fc.string(), + }), + fc.record({ + type: fc.constant<'tool-call-delta'>('tool-call-delta'), + index: fc.nat(2), + id: fc.constantFrom(CallId('c1'), CallId('c2')), + name: fc.constantFrom('write', 'read'), + argumentsDelta: fc.string(), + }), +) + +const boundaryChunkArb: fc.Arbitrary = fc.oneof( + fc.record({ type: fc.constant<'block-start'>('block-start'), index: fc.nat(2), blockType: fc.constant<'text'>('text') }), + fc.record({ type: fc.constant<'finish'>('finish'), reason: fc.constant({ kind: 'stop' as const }) }), +) + +/** Batches with contiguous seqs, arbitrary gaps in time, mixed chunk kinds and turn/step placement. */ +const batchArb: fc.Arbitrary = fc.array( + fc.record({ + chunk: fc.oneof({ weight: 4, arbitrary: deltaChunkArb }, { weight: 1, arbitrary: boundaryChunkArb }), + gap: fc.integer({ min: -5, max: 200 }), + turn: fc.nat(1), + step: fc.nat(1), + }), + { maxLength: 40 }, + // JSON round-trip normalizes fast-check's null-prototype records into the + // plain objects real log events are (the log is JSON), so equality compares + // values, not prototypes. +).map(entries => JSON.parse(JSON.stringify( + entries.map((entry, k) => chunkEvent(k, 1000 + entry.gap * k, entry.chunk, entry.turn, entry.step)), +)) as SessionEvent[]) + +describe('chunk-row codec properties', () => { + it('JSON-serialized pack∘decode reproduces every batch exactly', () => { + fc.assert(fc.property(batchArb, (events) => { + expect(decodeAll(packChunkRuns(events))).toStrictEqual(events) + })) + }) +}) diff --git a/packages/examples/acp-demo/README.md b/packages/examples/acp-demo/README.md index 17e3da78f3..1b91863fbe 100644 --- a/packages/examples/acp-demo/README.md +++ b/packages/examples/acp-demo/README.md @@ -31,6 +31,7 @@ Because the package wires no logger entry, an ACP leaf has **nothing to get wron | `tools` | `{ mode: 'native' }` | tool-registry presentation config (`native` / `code` / `both`), routed through `dsh-agent-spine-demo` | | `skills` | owner defaults | registry-cache, local-provider, and model-facing skill-tool config, routed through `dsh-agent-spine-demo` | | `persistenceRoot` | `./.sessions` | the JSONL backend's root directory | +| `packChunks` | `false` | write delta-chunk runs as packed storage rows (the JSONL backend's `packChunks`) | The leaf supplies the swappable backends: an LLM adapter (`llm-deepseek` for the real model, `llm-replay` for keyless snapshot replay) and a bash executor. diff --git a/packages/examples/acp-demo/src/index.ts b/packages/examples/acp-demo/src/index.ts index 0439363301..b3a2767d78 100644 --- a/packages/examples/acp-demo/src/index.ts +++ b/packages/examples/acp-demo/src/index.ts @@ -39,6 +39,8 @@ export interface Config { tools?: ToolsConfig /** Directory the JSONL session backend writes under. Defaults to `./.sessions`. */ persistenceRoot?: string + /** Write delta-chunk runs as packed storage rows (the JSONL backend's `packChunks`). Defaults to `false`. */ + packChunks?: boolean /** Skill registry, local-provider, and model-facing consumer config forwarded to agent-spine-demo. */ skills?: agentCore.SkillConfig } @@ -57,6 +59,7 @@ export const Config: z = z.object({ // TODO(single-default-literal): share this schema default and the defensive // apply() fallback through one named constant while retaining both boundaries. persistenceRoot: z.string().default('./.sessions'), + packChunks: z.boolean().default(false), skills: agentCore.SkillConfigSchema, }) /* jscpd:ignore-end */ @@ -76,6 +79,9 @@ export function apply(ctx: Context, config: Config): void { ...config.skills !== undefined ? { skills: config.skills } : {}, }) ctx.plugin(UserInteractionService) - ctx.plugin(SessionPersistenceJsonl, { root: config.persistenceRoot ?? './.sessions' }) + ctx.plugin(SessionPersistenceJsonl, { + root: config.persistenceRoot ?? './.sessions', + ...config.packChunks !== undefined ? { packChunks: config.packChunks } : {}, + }) ctx.plugin(acp, { model: config.model }) } diff --git a/packages/examples/stdio-demo/README.md b/packages/examples/stdio-demo/README.md index 5087a2fca0..7d26a0a121 100644 --- a/packages/examples/stdio-demo/README.md +++ b/packages/examples/stdio-demo/README.md @@ -31,6 +31,7 @@ The leaf `cordis.yml` supplies only the **swappable backends** — an LLM adapte | `tools` | `{ mode: 'native' }` | tool-registry presentation config (`native` / `code` / `both`), routed through `dsh-agent-spine-demo` | | `skills` | owner defaults | registry-cache, local-provider, and model-facing skill-tool config, routed through `dsh-agent-spine-demo` | | `persistenceRoot` | `./.sessions` | the JSONL backend's root directory | +| `packChunks` | `false` | write delta-chunk runs as packed storage rows (the JSONL backend's `packChunks`) | | `welcome` | `ready.` | the stdin-chat banner | | `resumeSessionId` | — | resume a persisted session id instead of starting fresh (sourced from an env var in the leaf) | diff --git a/packages/examples/stdio-demo/src/index.ts b/packages/examples/stdio-demo/src/index.ts index 89f4142af6..21a66ae5dd 100644 --- a/packages/examples/stdio-demo/src/index.ts +++ b/packages/examples/stdio-demo/src/index.ts @@ -44,6 +44,8 @@ export interface Config { tools?: ToolsConfig /** Directory the JSONL session backend writes under. Defaults to `./.sessions`. */ persistenceRoot?: string + /** Write delta-chunk runs as packed storage rows (the JSONL backend's `packChunks`). Defaults to `false`. */ + packChunks?: boolean /** stdin-chat banner printed once on start. Defaults to `'ready.'`. */ welcome?: string /** Skill registry, local-provider, and model-facing consumer config forwarded to agent-spine-demo. */ @@ -67,6 +69,7 @@ export const Config: z = z.object({ // TODO(single-default-literal): share these schema defaults and defensive // apply() fallbacks through named constants while retaining both boundaries. persistenceRoot: z.string().default('./.sessions'), + packChunks: z.boolean().default(false), welcome: z.string().default('ready.'), skills: agentCore.SkillConfigSchema, resumeSessionId: z.string(), @@ -93,7 +96,10 @@ export function apply(ctx: Context, config: Config): void { }], ...config.skills !== undefined ? { skills: config.skills } : {}, }) - ctx.plugin(SessionPersistenceJsonl, { root: config.persistenceRoot ?? './.sessions' }) + ctx.plugin(SessionPersistenceJsonl, { + root: config.persistenceRoot ?? './.sessions', + ...config.packChunks !== undefined ? { packChunks: config.packChunks } : {}, + }) ctx.plugin(UserInteractionService) ctx.plugin(toolAskUser) ctx.plugin(uiStdio, { welcome: config.welcome ?? 'ready.', agent: 'main' }) diff --git a/packages/session-persistence/session-persistence-jsonl/README.md b/packages/session-persistence/session-persistence-jsonl/README.md index 343fb4a70a..df5337d53a 100644 --- a/packages/session-persistence/session-persistence-jsonl/README.md +++ b/packages/session-persistence/session-persistence-jsonl/README.md @@ -7,10 +7,11 @@ The JSONL durable session-persistence backend — a concrete `SessionPersistence ``` / cwd-/ # per-project bucket (or _no-cwd/ when no cwd) - .jsonl # header line + one SessionEvent per line (verbatim) + .jsonl # header line + one storage record per line ``` -- The first `.jsonl` line is the immutable `SessionHeader` tagged `{ type: 'session', version, id, cwd?, createdAt, parentSession?, seedLength? }`; every subsequent line is one `SessionEvent` JSON, **verbatim including `assistant/chunk`** so `seq` stays contiguous (`events[i].seq === i`). +- The first `.jsonl` line is the immutable `SessionHeader` tagged `{ type: 'session', version, id, cwd?, createdAt, parentSession?, seedLength? }`; every subsequent line is one storage record. `assistant/chunk` events are never dropped, and `seq` stays contiguous across the decoded log. +- A storage record is a `SessionEvent` JSON verbatim, or — written only under `packChunks` — a **packed chunk row** (`text-chunks` / `reasoning-chunks` / `tool-call-chunks`; bare slash-less tags like the header's `session`, so row tags cannot be confused with event types): one line holding a run of ≥3 consecutive same-block `assistant/chunk` delta events, `seq0`/`time0` plus per-member `dt` gaps reconstructing every member's `seq`/`time` exactly. The lossless codec lives in `@deepseek-ai/dsh-session` (`packChunkRuns`/`decodeStorageRecord`) and whitelists exact shapes — anything unrecognized stores verbatim. Reading is layout-blind: `load` always decodes rows, so packed, unpacked, and mixed files load identically. - Session ids are unvalidated branded strings, so they are percent-encoded to a single safe path segment before use (no traversal, no collision). ## Config @@ -18,6 +19,7 @@ The JSONL durable session-persistence backend — a concrete `SessionPersistence | Key | Type | Notes | |---|---|---| | `root` | `string` (required) | Root directory for all session files. **No default** — a `process.cwd()` default would scatter files as the process's cwd changes (bash calls, subprocesses). | +| `packChunks` | `boolean` (default `false`) | Write delta-chunk runs as packed rows (~60% smaller logs measured on a real coding session). Off, the written layout is byte-identical to the pre-packing format; reading packed rows works regardless of this switch. Off by default while the snapshot goldens stay one-event-per-line — recording with packing on rewrites every fixture `session.jsonl`. | ## Durability and crash semantics diff --git a/packages/session-persistence/session-persistence-jsonl/src/format.ts b/packages/session-persistence/session-persistence-jsonl/src/format.ts index 39bdecf751..f5b5515ab9 100644 --- a/packages/session-persistence/session-persistence-jsonl/src/format.ts +++ b/packages/session-persistence/session-persistence-jsonl/src/format.ts @@ -10,7 +10,8 @@ import { createHash } from 'node:crypto' import { join } from 'node:path' -import type { SessionEvent, SessionHeader, SessionId } from '@deepseek-ai/dsh-session' +import { decodeStorageRecord, packChunkRuns } from '@deepseek-ai/dsh-session' +import type { SessionEvent, SessionHeader, SessionId, StorageRecord } from '@deepseek-ai/dsh-session' /** * The first line of a session's `.jsonl` file: the immutable @@ -126,17 +127,26 @@ export function logPath(root: string, cwd: string | undefined, id: SessionId): s } /** - * Serialize one event as a JSONL line (no trailing newline). - * @param event - the event to serialize verbatim. - * @returns the event's single-line JSON text; the writer adds the newline. + * Serialize an event batch as JSONL lines (no trailing newline). With + * `packChunks` on, delta-chunk runs pack into `text-chunks` / + * `reasoning-chunks` / `tool-call-chunks` storage rows; off writes one event + * per line, byte-identical to the pre-packing layout. Reading is layout-blind + * either way ({@link scanLog} always decodes rows), so the switch only shapes + * NEW bytes. + * @param events - the batch to serialize, in log order. + * @param packChunks - whether to pack delta runs into storage rows. + * @returns the batch's JSONL text; the writer adds the final newline. */ -export function eventLine(event: SessionEvent): string { - return JSON.stringify(event) +export function eventLines(events: readonly SessionEvent[], packChunks: boolean): string { + const records: readonly StorageRecord[] = packChunks ? packChunkRuns(events) : events + return records.map(record => JSON.stringify(record)).join('\n') } /** * Parse a JSONL log buffer into its preserved event prefix (the header is line - * 0). Fully written events in an interrupted final turn remain part of the + * 0). Event lines pass through verbatim; packed chunk rows expand back into + * their events, so callers see one contiguous event list regardless of layout. + * Fully written events in an interrupted final turn remain part of the * prefix. The first unparsable record or seq gap after the last `turn/end` * marks a tolerated torn tail; the same hole in the committed region rejects. * @@ -175,46 +185,60 @@ export function scanLog(buffer: Buffer): { meta: SessionHeader; events: SessionE } const headerLine = parsedHeader - // Parse every complete record first so the last valid `turn/end` determines - // whether an earlier hole is committed corruption or an uncommitted tail. - interface Parsed { ok: boolean; event?: SessionEvent; endByte: number } + // Parse and decode every complete line first so the last valid `turn/end` + // determines whether an earlier hole is committed corruption or an + // uncommitted tail. One line yields one event, or a whole run for a packed + // chunk row; a row-tagged line that fails row validation is a hole, exactly + // like unparsable JSON. + interface Parsed { ok: boolean; events?: SessionEvent[]; endByte: number } const parsed: Parsed[] = eventEntries.map((entry) => { try { - return { ok: true, event: JSON.parse(entry.text) as SessionEvent, endByte: entry.endByte } + return { ok: true, events: decodeStorageRecord(JSON.parse(entry.text)), endByte: entry.endByte } } catch { return { ok: false, endByte: entry.endByte } } }) - // The last index (into eventEntries) that is a valid `turn/end` — the last - // fully-committed boundary (the loop flushes only at turn/end). + // The last index (into eventEntries) that ends in a valid `turn/end` — the + // last fully-committed boundary (the loop flushes only at turn/end). A packed + // row never stores a turn/end, so only single-event lines can match. let lastTurnEnd = -1 for (let i = parsed.length - 1; i >= 0; i--) { const p = parsed[i] - if (p?.ok && p.event?.type === 'turn/end') { lastTurnEnd = i; break } + if (p?.ok && p.events?.some(e => e.type === 'turn/end')) { lastTurnEnd = i; break } } // Preserve the contiguous prefix, including a complete interrupted turn; // holes through the last committed boundary throw, while later holes stop. + // Contiguity is a cursor over seqs (not the line index): a packed row + // advances the cursor by its whole run. const preserved: SessionEvent[] = [] - for (let i = 0; i < parsed.length; i++) { + let lastPreservedLine = -1 + scan: for (let i = 0; i < parsed.length; i++) { const p = parsed[i] - if (!p?.ok || p.event === undefined) { + if (!p?.ok || p.events === undefined) { if (i <= lastTurnEnd) throw new Error(`corrupt session log: unparsable committed event at line ${i + 1}`) break // torn tail fragment after the last turn/end — stop, tolerate } - if (p.event.seq !== i) { - if (i <= lastTurnEnd) throw new Error(`corrupt session log: seq gap in committed region at line ${i + 1} (expected ${i}, got ${p.event.seq})`) - break // gap after the last turn/end — torn tail, stop + for (const event of p.events) { + if (event.seq !== preserved.length) { + if (i <= lastTurnEnd) { + throw new Error(`corrupt session log: seq gap in committed region at line ${i + 1} (expected ${preserved.length}, got ${event.seq})`) + } + break scan // gap after the last turn/end — torn tail, stop + } + preserved.push(event) } - preserved.push(p.event) + lastPreservedLine = i } - // committedBytes = end of the last PRESERVED line (header if none): the next - // append truncates any torn bytes past this point before writing the - // synthetic closers + new events. - const lastPreserved = parsed[preserved.length - 1] - const committedBytes = preserved.length > 0 && lastPreserved ? lastPreserved.endByte : headerEntry.endByte + // committedBytes = end of the last FULLY preserved line (header if none): the + // next append truncates any torn bytes past this point before writing the + // synthetic closers + new events. A line is preserved whole or not at all — + // a mid-row seq gap discards the whole row, keeping the truncation offset on + // a line boundary. + const lastPreserved = parsed[lastPreservedLine] + const committedBytes = lastPreserved !== undefined ? lastPreserved.endByte : headerEntry.endByte return { meta: fromHeaderLine(headerLine), events: preserved, committedBytes } } diff --git a/packages/session-persistence/session-persistence-jsonl/src/index.ts b/packages/session-persistence/session-persistence-jsonl/src/index.ts index 1d13ff424e..9fa142efe1 100644 --- a/packages/session-persistence/session-persistence-jsonl/src/index.ts +++ b/packages/session-persistence/session-persistence-jsonl/src/index.ts @@ -16,10 +16,10 @@ import { } from '@deepseek-ai/dsh-session-persistence' import type { SessionEvent, SessionId, SessionHeader } from '@deepseek-ai/dsh-session' import { - encodeSegment, eventLine, logPath, parseHeaderMeta, scanLog, sessionDir, toHeaderLine, + encodeSegment, eventLines, logPath, parseHeaderMeta, scanLog, sessionDir, toHeaderLine, } from './format.ts' -/** Plugin config: where the JSONL backend keeps its session logs (`root` is required — no default). */ +/** Plugin config: where the JSONL backend keeps its session logs, and the packed-row write switch. */ export interface Config { /** * Root directory for all session files. Required (no default): a default of @@ -27,6 +27,15 @@ export interface Config { * (bash calls, subprocesses). Sessions group under per-cwd subdirectories. */ root: string + /** + * Write runs of consecutive `assistant/chunk` delta events as packed + * `text-chunks`/`reasoning-chunks`/`tool-call-chunks` rows (lossless, + * ~60% smaller logs measured on a real session). Off by default while + * snapshot fixtures stay in the one-event-per-line layout: recording with + * packing on rewrites every golden `session.jsonl`. READING packed rows is + * unconditional — a log's layout never depends on this switch. + */ + packChunks?: boolean } /** Whether a filesystem error means absence; every non-ENOENT failure must surface. */ @@ -44,6 +53,7 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi static Config: z = z.object({ root: z.string().required(), + packChunks: z.boolean().default(false), }) /** @@ -54,12 +64,16 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi override readonly name = 'session-persistence-jsonl' private root: string + private packChunks: boolean private coordinator: PersistenceCoordinator constructor(ctx: Context, public config: Config) { super(ctx) // Resolve once so later process.cwd() changes cannot split one backend across roots. this.root = resolve(config.root) + // schemastery (static Config) applied the default before construction; + // the cast records that runtime fact for exactOptionalPropertyTypes. + this.packChunks = (config as Required).packChunks this.coordinator = new PersistenceCoordinator(this.ctx, this) } @@ -168,7 +182,7 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi throw new Error(`refusing to materialize "${meta.id}": a log already exists on disk (load/resume it instead)`) } const header = JSON.stringify(toHeaderLine(meta)) - const body = events.map(eventLine).join('\n') + const body = eventLines(events, this.packChunks) const content = header + '\n' + body + '\n' const tmp = `${finalPath}.${randomBytes(6).toString('hex')}.tmp` @@ -223,7 +237,7 @@ export class SessionPersistenceJsonl extends SessionPersistence implements Persi try { const { size: before } = await handle.stat() try { - await handle.writeFile(events.map(eventLine).join('\n') + '\n') + await handle.writeFile(eventLines(events, this.packChunks) + '\n') await handle.sync() } catch (error) { // Roll back whatever bytes landed so a retry starts from a clean EOF. diff --git a/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts b/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts index 75fcb48732..9c2b0fa4be 100644 --- a/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts +++ b/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts @@ -6,7 +6,7 @@ import { join } from 'node:path' import SessionStore, { SessionId } from '@deepseek-ai/dsh-session' import type { Session, SessionEvent, SessionHeader } from '@deepseek-ai/dsh-session' import SessionPersistenceJsonl from '@deepseek-ai/dsh-session-persistence-jsonl' -import { encodeSegment, logPath, scanLog, sessionDir } from '../src/format.ts' +import { encodeSegment, eventLines, logPath, scanLog, sessionDir } from '../src/format.ts' import { runPersistenceContract, meta, oneTurnLog, appendLog } from '../../session-persistence/tests/contract.ts' import { runCoordinatorContract, type CoordinatorFixture } from '../../session-persistence/tests/coordinator-contract.ts' @@ -414,6 +414,119 @@ describe('SessionPersistenceJsonl: scanLog unit', () => { }) }) +describe('SessionPersistenceJsonl: packed chunk rows (packChunks: true)', () => { + let ctx: Context + beforeEach(async () => { + root = await freshRoot() + ctx = new Context() + await ctx.plugin(SessionStore) + await ctx.plugin(SessionPersistenceJsonl, { root, packChunks: true }) + }) + afterEach(async () => { await ctx.fiber.dispose() }) + + /** A one-turn log whose step streams a five-member text-delta run. */ + function chunkRunLog(): SessionEvent[] { + const deltas: SessionEvent[] = Array.from({ length: 5 }, (_, k) => ({ + type: 'assistant/chunk', + seq: 2 + k, + time: 3 + k, + data: { turn: 1, step: 1, chunk: { type: 'text-delta', index: 0, text: `t${k}` } }, + })) + return [ + { type: 'turn/start', seq: 0, time: 1, data: { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } } }, + { type: 'step/start', seq: 1, time: 2, data: { turn: 1, step: 1 } }, + ...deltas, + { type: 'assistant/message', seq: 7, time: 8, data: { turn: 1, step: 1, content: [{ type: 'text', text: 't0t1t2t3t4' }] }, surfaceOp: 'append', sourceEventSeqs: [2, 3, 4, 5, 6] }, + { type: 'step/end', seq: 8, time: 9, data: { turn: 1, step: 1 } }, + { type: 'turn/end', seq: 9, time: 10, data: { turn: 1, reason: { kind: 'completed' } } }, + ] + } + + it('writes a delta run as one text-chunks row and loads back identical events', async () => { + const m = meta('packed', '/work') + const log = chunkRunLog() + await ctx.sessionPersistence.create(m) + await ctx.sessionPersistence.append(m.id, log) + + const raw = (await readFile(logPath(root, '/work', m.id), 'utf8')).split('\n').filter(Boolean) + const tags = raw.slice(1).map(line => (JSON.parse(line) as { type: string }).type) + expect(tags).toEqual(['turn/start', 'step/start', 'text-chunks', 'assistant/message', 'step/end', 'turn/end']) + + const loaded = await ctx.sessionPersistence.load(m.id) + expect(loaded.events).toEqual(log) + }) + + it('loads a mixed file: verbatim lines from an unpacked writer, then packed appends', async () => { + const m = meta('mixed', '/work') + const log = chunkRunLog() + // First turn written line-per-event by an unpacked-config writer (an old + // file, hand-planted so this packed-config backend adopts it on load). + await mkdir(sessionDir(root, '/work'), { recursive: true }) + await writeFile(logPath(root, '/work', m.id), [ + JSON.stringify({ type: 'session', version: 0, id: 'mixed', createdAt: 1000, cwd: '/work' }), + ...log.map(e => JSON.stringify(e)), + ].join('\n') + '\n') + // Adopt the stored log (cursor = stored length), then append a second turn + // through THIS packed-config backend. + expect((await ctx.sessionPersistence.load(m.id)).events).toEqual(log) + const secondTurn: SessionEvent[] = JSON.parse(JSON.stringify(log)) as SessionEvent[] + for (const [k, e] of secondTurn.entries()) { + ;(e as { seq: number }).seq = 10 + k + ;(e.data as { turn: number }).turn = 2 + } + await ctx.sessionPersistence.append(m.id, secondTurn) + + const loaded = await ctx.sessionPersistence.load(m.id) + expect(loaded.events).toEqual([...log, ...secondTurn]) + // The packed append really packed: the file's tail carries a text-chunks row. + const tags = (await readFile(logPath(root, '/work', m.id), 'utf8')).split('\n').filter(Boolean) + .map(line => (JSON.parse(line) as { type: string }).type) + expect(tags.filter(t => t === 'text-chunks')).toHaveLength(1) + expect(tags.filter(t => t === 'assistant/chunk')).toHaveLength(5) + }) + + it('scanLog: a packed row advances the seq cursor by its whole run', () => { + const logText = [ + JSON.stringify({ type: 'session', version: 0, id: 'rows', createdAt: 1 }), + JSON.stringify({ type: 'turn/start', seq: 0, time: 1, data: { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } } }), + JSON.stringify({ type: 'text-chunks', seq0: 1, time0: 2, data: { turn: 1, step: 1, index: 0, dt: [1, 1], texts: ['a', 'b', 'c'] } }), + JSON.stringify({ type: 'turn/end', seq: 4, time: 5, data: { turn: 1, reason: { kind: 'completed' } } }), + ].join('\n') + '\n' + const { events } = scanLog(Buffer.from(logText)) + expect(events.map(e => e.seq)).toEqual([0, 1, 2, 3, 4]) + expect(events[2]).toEqual({ type: 'assistant/chunk', seq: 2, time: 3, data: { turn: 1, step: 1, chunk: { type: 'text-delta', index: 0, text: 'b' } } }) + }) + + it('scanLog: a malformed packed row in the committed region rejects like corrupt JSON', () => { + const logText = [ + JSON.stringify({ type: 'session', version: 0, id: 'bad-row', createdAt: 1 }), + // dt arity mismatch — row validation throws, so the line is a committed hole. + JSON.stringify({ type: 'text-chunks', seq0: 0, time0: 1, data: { turn: 1, step: 1, index: 0, dt: [], texts: ['a', 'b'] } }), + JSON.stringify({ type: 'turn/end', seq: 2, time: 3, data: { turn: 1, reason: { kind: 'completed' } } }), + ].join('\n') + '\n' + expect(() => scanLog(Buffer.from(logText))).toThrow(/unparsable committed event/) + }) + + it('scanLog: a packed row with a mid-run seq gap after the last turn/end drops the whole row', () => { + const logText = [ + JSON.stringify({ type: 'session', version: 0, id: 'row-gap', createdAt: 1 }), + JSON.stringify({ type: 'turn/start', seq: 0, time: 1, data: { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } } }), + // seq0 skips 1 — the run's first member is already a gap; no turn/end follows. + JSON.stringify({ type: 'text-chunks', seq0: 2, time0: 2, data: { turn: 1, step: 1, index: 0, dt: [1, 1], texts: ['a', 'b', 'c'] } }), + ].join('\n') + '\n' + const scanned = scanLog(Buffer.from(logText)) + expect(scanned.events.map(e => e.seq)).toEqual([0]) + // committedBytes stays on the line boundary BEFORE the dropped row. + const headerAndTurn = logText.split('\n').slice(0, 2).join('\n') + '\n' + expect(scanned.committedBytes).toBe(Buffer.byteLength(headerAndTurn, 'utf8')) + }) + + it('eventLines(packChunks: false) is byte-identical to the pre-packing layout', () => { + const log = chunkRunLog() + expect(eventLines(log, false)).toBe(log.map(e => JSON.stringify(e)).join('\n')) + }) +}) + describe('SessionPersistenceJsonl: edge cases', () => { let ctx: Context beforeEach(async () => { diff --git a/packages/support/acp-snapshot/src/normalize.ts b/packages/support/acp-snapshot/src/normalize.ts index e3f32792ab..4fe45d3710 100644 --- a/packages/support/acp-snapshot/src/normalize.ts +++ b/packages/support/acp-snapshot/src/normalize.ts @@ -81,8 +81,10 @@ export function normalizeStdout(rawStdout: string, ctx: NormalizeContext): strin * Normalize a session JSONL log into a stable golden: the header line's * volatile fields (`createdAt`, `id`, `cwd`) and every event's `time` are * zeroed/scrubbed, all volatile strings scrubbed, and `seq` is LEFT INTACT - * (deterministic by contract). Output is JSONL in the same shape as the input — - * one compact record per line. + * (deterministic by contract). A packed chunk row's timing (`time0`, the `dt` + * gaps) zeroes just like an event `time`; its `seq0` stays, like `seq`. + * Output is JSONL in the same shape as the input — one compact record per + * line. * * @param rawLog The raw session `.jsonl` content. * @param ctx The run's volatile values to scrub. @@ -95,6 +97,13 @@ export function normalizeSessionLog(rawLog: string, ctx: NormalizeContext): stri // Header line: { type: 'session', createdAt, id, cwd, … }. if (record.type === 'session') { if ('createdAt' in record) record.createdAt = 0 + } else if ('time0' in record) { + // Packed chunk row: zero the anchor timestamp and every member gap. + record.time0 = 0 + const data = record.data + if (data !== null && typeof data === 'object' && Array.isArray((data as { dt?: unknown }).dt)) { + (data as { dt: unknown[] }).dt = (data as { dt: unknown[] }).dt.map(() => 0) + } } else if ('time' in record) { // Event line: zero the epoch-ms timestamp; keep seq (deterministic). record.time = 0 diff --git a/packages/support/acp-snapshot/tests/normalize.spec.ts b/packages/support/acp-snapshot/tests/normalize.spec.ts index d6c95045f6..e921b2f5b9 100644 --- a/packages/support/acp-snapshot/tests/normalize.spec.ts +++ b/packages/support/acp-snapshot/tests/normalize.spec.ts @@ -109,6 +109,27 @@ describe('normalizeSessionLog', () => { expect(out).toContain('"decision":"block"') // the decision is the behavior — kept }) + it('zeroes a packed chunk row\'s time0 and dt gaps but keeps seq0 and payload', () => { + const row = JSON.stringify({ + type: 'text-chunks', seq0: 7, time0: 999, + data: { turn: 1, step: 1, index: 0, dt: [212, 27, 0], texts: ['a', 'b', 'c', 'd'] }, + }) + const out = normalizeSessionLog(`${header({})}\n${row}\n`, ctx) + expect(out).toContain('"time0":0') + expect(out).toContain('"dt":[0,0,0]') + expect(out).toContain('"seq0":7') // seq0 is deterministic, like seq — NOT scrubbed + expect(out).toContain('"texts":["a","b","c","d"]') + expect(out).not.toContain('999') + expect(out).not.toContain('212') + }) + + it('zeroes time0 even when a malformed row carries no dt array', () => { + const row = JSON.stringify({ type: 'text-chunks', seq0: 1, time0: 999, data: 'not-an-object' }) + const out = normalizeSessionLog(`${header({})}\n${row}\n`, ctx) + expect(out).toContain('"time0":0') + expect(out).not.toContain('999') + }) + it('leaves a non-hook event durationMs untouched (only hook/result is scrubbed)', () => { const ev = JSON.stringify({ type: 'tool/result', seq: 2, time: 5, data: { durationMs: 88 } }) const out = normalizeSessionLog(`${header({})}\n${ev}\n`, ctx) diff --git a/packages/support/llm-replay/src/index.ts b/packages/support/llm-replay/src/index.ts index 2e509973e5..0836445d8d 100644 --- a/packages/support/llm-replay/src/index.ts +++ b/packages/support/llm-replay/src/index.ts @@ -9,6 +9,7 @@ import { existsSync, readFileSync } from 'node:fs' import { delimiter as pathDelimiter } from 'node:path' import type { Context } from 'cordis' +import { decodeStorageRecord } from '@deepseek-ai/dsh-session' import type { SessionEvent } from '@deepseek-ai/dsh-session' import type { GenerateOptions, StreamChunk } from '@deepseek-ai/dsh-llm' import { LlmError, assertNever } from '@deepseek-ai/dsh-llm' @@ -68,7 +69,9 @@ export interface SessionScript { /** * Parse a session `.jsonl` buffer into its event list. Line 0 is the session * header (a `{type:'session',…}` record), every subsequent non-empty line is a - * {@link SessionEvent}. The header is skipped; malformed lines fail loud. + * {@link SessionEvent} or a packed chunk row (expanded back into its events, so + * a fixture recorded with `packChunks` on derives the same script). The header + * is skipped; malformed lines fail loud. * @param text - the raw `.jsonl` file contents. * @returns every event after the header, in log order. */ @@ -77,8 +80,7 @@ export function parseSessionLog(text: string): SessionEvent[] { const events: SessionEvent[] = [] // The JSONL backend guarantees line 0 is the session header. for (let i = 1; i < lines.length; i++) { - const parsed: unknown = JSON.parse(lines[i] as string) - events.push(parsed as SessionEvent) + events.push(...decodeStorageRecord(JSON.parse(lines[i] as string))) } return events } diff --git a/packages/support/llm-replay/tests/llm-replay.spec.ts b/packages/support/llm-replay/tests/llm-replay.spec.ts index ac52ec11c9..9e48896dc5 100644 --- a/packages/support/llm-replay/tests/llm-replay.spec.ts +++ b/packages/support/llm-replay/tests/llm-replay.spec.ts @@ -90,6 +90,19 @@ describe('parseSessionLog', () => { const ev = chunkEvent(1, 1, 1, TEXT_CHUNKS[0] as StreamChunk) expect(parseSessionLog(`${header}\n\n${JSON.stringify(ev)}\n\n`)).toEqual([ev]) }) + + it('expands a packed chunk row into its events (a fixture recorded with packChunks on)', () => { + const header = JSON.stringify({ type: 'session', version: 0, id: 's1', createdAt: 0 }) + const row = JSON.stringify({ + type: 'text-chunks', seq0: 1, time0: 0, + data: { turn: 1, step: 1, index: 0, dt: [0, 0], texts: ['a', 'b', 'c'] }, + }) + expect(parseSessionLog(`${header}\n${row}\n`)).toEqual([ + chunkEvent(1, 1, 1, { type: 'text-delta', index: 0, text: 'a' }), + chunkEvent(2, 1, 1, { type: 'text-delta', index: 0, text: 'b' }), + chunkEvent(3, 1, 1, { type: 'text-delta', index: 0, text: 'c' }), + ]) + }) }) describe('deriveReplayScript', () => { From 8f5c592b9f8390d0309c8c93f245d650943702c6 Mon Sep 17 00:00:00 2001 From: kingwl Date: Wed, 15 Jul 2026 22:44:32 +0800 Subject: [PATCH 2/9] fix(session): break chunk runs on time gaps that cannot subtract exactly Two safe-integer timestamps can differ by more than 2^53-1 (e.g. MIN_SAFE_INTEGER to MAX_SAFE_INTEGER-1), so the dt subtraction rounds and the packed row decodes to a timestamp one off the original -- violating the codec's lossless contract. Unreachable from a real clock (the gap needs ~285k years) but reachable from hand-written fixtures, and the decoder accepts hand-written rows. continues() now refuses to extend a run across such a gap (the check is exact both ways: an in-range true gap subtracts without rounding and passes; an out-of-range one rounds to an out-of-range value and fails), splitting the run instead -- whitelist philosophy, compression lost, data never. The decoder tightens to match the encoder's image: time0 and dt must be safe integers, and reconstructed member seqs/times must stay in safe range, so float reconstruction is exact wherever validation passes. The round-trip property now draws times from the full safe-integer range (it previously generated only small gaps, which is how this escaped); the bot's counterexample is pinned as an example test. Found by ds-review-bot on #338. --- packages/core/session/src/chunk-rows.ts | 37 +++++++++++++++---- .../core/session/tests/chunk-rows.spec.ts | 34 +++++++++++++++-- 2 files changed, 60 insertions(+), 11 deletions(-) diff --git a/packages/core/session/src/chunk-rows.ts b/packages/core/session/src/chunk-rows.ts index c42532946c..56de67b405 100644 --- a/packages/core/session/src/chunk-rows.ts +++ b/packages/core/session/src/chunk-rows.ts @@ -135,6 +135,12 @@ function indexOf(event: DeltaEvent): number { /** Whether `next` extends a run ending in `prev` (same kind already checked by the caller). */ function continues(prev: DeltaEvent, next: DeltaEvent, kind: DeltaKind): boolean { if (next.seq !== prev.seq + 1) return false + // Two safe-integer times can sit further apart than a double subtracts + // exactly (2^53-1 and its negation differ by ~2^54); a rounded gap would + // decode to a different timestamp. The check is exact in both directions: a + // true gap within safe range subtracts without rounding and passes, while a + // true gap beyond it rounds to a value that is itself beyond and fails. + if (!Number.isSafeInteger(next.time - prev.time)) return false if (next.data.turn !== prev.data.turn || next.data.step !== prev.data.step) return false if (indexOf(next) !== indexOf(prev)) return false if (kind !== 'tool-call-delta') return true @@ -219,8 +225,8 @@ function malformed(tag: string, why: string): never { throw new Error(`malformed ${tag} storage row: ${why}`) } -/** Validate the shared run-data fields and the payload/dt arity. */ -function validateRunData(tag: string, data: Record, payloadKey: 'texts' | 'args'): void { +/** Validate the shared run-data fields and the payload/dt arity; returns the member payload. */ +function validateRunData(tag: string, data: Record, payloadKey: 'texts' | 'args'): string[] { if (typeof data.turn !== 'number' || typeof data.step !== 'number' || typeof data.index !== 'number') { malformed(tag, 'turn/step/index must be numbers') } @@ -229,12 +235,13 @@ function validateRunData(tag: string, data: Record, payloadKey: malformed(tag, `${payloadKey} must be a non-empty string array`) } const dt = data.dt - if (!Array.isArray(dt) || dt.some(gap => typeof gap !== 'number' || !Number.isFinite(gap))) { - malformed(tag, 'dt must be an array of finite numbers') + if (!Array.isArray(dt) || dt.some(gap => !Number.isSafeInteger(gap))) { + malformed(tag, 'dt must be an array of safe integers') } if (dt.length !== payload.length - 1) { malformed(tag, `dt length ${dt.length} does not match ${payload.length} members`) } + return payload as string[] } /** Validate a row-tagged parsed value's envelope and data, throwing on any malformation. */ @@ -245,11 +252,12 @@ function validateRow(value: Record, tag: ChunkRow['type']): Chu if (!Number.isSafeInteger(value.seq0) || (value.seq0 as number) < 0) { malformed(tag, 'seq0 must be a non-negative safe integer') } - if (typeof value.time0 !== 'number' || !Number.isFinite(value.time0)) { - malformed(tag, 'time0 must be a finite number') + if (!Number.isSafeInteger(value.time0)) { + malformed(tag, 'time0 must be a safe integer') } const data = value.data if (!isRecord(data)) malformed(tag, 'data must be an object') + let payload: string[] if (tag === 'tool-call-chunks') { const withName = hasExactKeys(data, ['turn', 'step', 'index', 'id', 'name', 'dt', 'args']) if (!withName && !hasExactKeys(data, ['turn', 'step', 'index', 'id', 'dt', 'args'])) { @@ -258,12 +266,25 @@ function validateRow(value: Record, tag: ChunkRow['type']): Chu if (typeof data.id !== 'string' || (withName && typeof data.name !== 'string')) { malformed(tag, 'id (and name when present) must be strings') } - validateRunData(tag, data, 'args') + payload = validateRunData(tag, data, 'args') } else { if (!hasExactKeys(data, ['turn', 'step', 'index', 'dt', 'texts'])) { malformed(tag, 'data must be exactly {turn, step, index, dt, texts}') } - validateRunData(tag, data, 'texts') + payload = validateRunData(tag, data, 'texts') + } + // Reconstruction bounds. The encoder only packs runs whose member seqs and + // times are all safe integers, so a running value that leaves safe range is + // outside any encoder's image: float arithmetic would round it to a + // different number than exact arithmetic, a silent corruption. Within safe + // range every step is exact, so the first departure is always caught. + if (!Number.isSafeInteger((value.seq0 as number) + payload.length - 1)) { + malformed(tag, 'member seqs must stay safe integers') + } + let time = value.time0 as number + for (const gap of data.dt as number[]) { + time += gap + if (!Number.isSafeInteger(time)) malformed(tag, 'member times must stay safe integers') } return value as unknown as ChunkRow } diff --git a/packages/core/session/tests/chunk-rows.spec.ts b/packages/core/session/tests/chunk-rows.spec.ts index 9eb9173bca..28611e5e41 100644 --- a/packages/core/session/tests/chunk-rows.spec.ts +++ b/packages/core/session/tests/chunk-rows.spec.ts @@ -106,6 +106,22 @@ describe('packChunkRuns', () => { expect(packChunkRuns(events)).toStrictEqual(events) }) + it('breaks a run on a time gap beyond safe-integer range (subtraction would round)', () => { + // Both endpoints are safe integers, but their true difference (~2^54) + // exceeds exact double range: b - a rounds, so a + (b - a) !== b and a + // packed row would decode to a different timestamp. + const a = Number.MIN_SAFE_INTEGER + const b = Number.MAX_SAFE_INTEGER - 1 + expect(a + (b - a)).not.toBe(b) // the rounding this guard exists for + const events = [ + chunkEvent(0, a, { type: 'text-delta', index: 0, text: 'x' }), + chunkEvent(1, b, { type: 'text-delta', index: 0, text: 'y' }), + chunkEvent(2, b + 1, { type: 'text-delta', index: 0, text: 'z' }), + ] + expect(packChunkRuns(events)).toStrictEqual(events) // split at the gap; halves too short + expect(decodeAll(packChunkRuns(events))).toStrictEqual(events) + }) + it('stores a delta with an off-whitelist data envelope verbatim (parsed-fixture shapes)', () => { const mk = (seq: number, data: unknown): SessionEvent => ({ type: 'assistant/chunk', seq, time: 1000, data } as SessionEvent) @@ -144,11 +160,15 @@ describe('decodeStorageRecord', () => { ['an envelope with extra keys', { type: 'text-chunks', seq0: 0, time0: 1, data: { turn: 1, step: 1, index: 0, dt: [], texts: ['a'] }, extra: 1 }], ['a negative seq0', { type: 'text-chunks', seq0: -1, time0: 1, data: { turn: 1, step: 1, index: 0, dt: [], texts: ['a'] } }], ['a non-finite time0', { type: 'text-chunks', seq0: 0, time0: Infinity, data: { turn: 1, step: 1, index: 0, dt: [], texts: ['a'] } }], + ['a fractional time0', { type: 'text-chunks', seq0: 0, time0: 1.5, data: { turn: 1, step: 1, index: 0, dt: [], texts: ['a'] } }], ['a data shape mismatch', { type: 'text-chunks', seq0: 0, time0: 1, data: { turn: 1, step: 1, index: 0, dt: [], args: ['a'] } }], ['a non-string member', { type: 'text-chunks', seq0: 0, time0: 1, data: { turn: 1, step: 1, index: 0, dt: [], texts: [7] } }], ['an empty member list', { type: 'text-chunks', seq0: 0, time0: 1, data: { turn: 1, step: 1, index: 0, dt: [], texts: [] } }], ['a dt arity mismatch', { type: 'text-chunks', seq0: 0, time0: 1, data: { turn: 1, step: 1, index: 0, dt: [1, 2], texts: ['a', 'b'] } }], ['a non-finite dt gap', { type: 'text-chunks', seq0: 0, time0: 1, data: { turn: 1, step: 1, index: 0, dt: [NaN], texts: ['a', 'b'] } }], + ['a fractional dt gap', { type: 'text-chunks', seq0: 0, time0: 1, data: { turn: 1, step: 1, index: 0, dt: [0.5], texts: ['a', 'b'] } }], + ['a member seq leaving safe range', { type: 'text-chunks', seq0: Number.MAX_SAFE_INTEGER, time0: 1, data: { turn: 1, step: 1, index: 0, dt: [0, 0], texts: ['a', 'b', 'c'] } }], + ['a member time leaving safe range', { type: 'text-chunks', seq0: 0, time0: Number.MAX_SAFE_INTEGER, data: { turn: 1, step: 1, index: 0, dt: [1], texts: ['a', 'b'] } }], ['a non-numeric turn', { type: 'text-chunks', seq0: 0, time0: 1, data: { turn: 'x', step: 1, index: 0, dt: [], texts: ['a'] } }], ['a tool-call row without id', { type: 'tool-call-chunks', seq0: 0, time0: 1, data: { turn: 1, step: 1, index: 0, dt: [], args: ['a'] } }], ['a tool-call row with non-string id', { type: 'tool-call-chunks', seq0: 0, time0: 1, data: { turn: 1, step: 1, index: 0, id: 7, dt: [], args: ['a'] } }], @@ -183,11 +203,19 @@ const boundaryChunkArb: fc.Arbitrary = fc.oneof( fc.record({ type: fc.constant<'finish'>('finish'), reason: fc.constant({ kind: 'stop' as const }) }), ) -/** Batches with contiguous seqs, arbitrary gaps in time, mixed chunk kinds and turn/step placement. */ +/** + * Batches with contiguous seqs, arbitrary timestamps, mixed chunk kinds and + * turn/step placement. Times draw from the FULL safe-integer range (not just + * realistic clocks) so the property exercises the gap-overflow guard: two safe + * endpoints can differ by more than a double subtracts exactly. + */ const batchArb: fc.Arbitrary = fc.array( fc.record({ chunk: fc.oneof({ weight: 4, arbitrary: deltaChunkArb }, { weight: 1, arbitrary: boundaryChunkArb }), - gap: fc.integer({ min: -5, max: 200 }), + time: fc.oneof( + { weight: 4, arbitrary: fc.integer({ min: 995, max: 9000 }) }, + { weight: 1, arbitrary: fc.integer({ min: Number.MIN_SAFE_INTEGER, max: Number.MAX_SAFE_INTEGER }) }, + ), turn: fc.nat(1), step: fc.nat(1), }), @@ -196,7 +224,7 @@ const batchArb: fc.Arbitrary = fc.array( // plain objects real log events are (the log is JSON), so equality compares // values, not prototypes. ).map(entries => JSON.parse(JSON.stringify( - entries.map((entry, k) => chunkEvent(k, 1000 + entry.gap * k, entry.chunk, entry.turn, entry.step)), + entries.map((entry, k) => chunkEvent(k, entry.time, entry.chunk, entry.turn, entry.step)), )) as SessionEvent[]) describe('chunk-row codec properties', () => { From f9f8dc57fe5d6a1db830865aa3b679261c3f4802 Mon Sep 17 00:00:00 2001 From: kingwl Date: Mon, 20 Jul 2026 15:28:08 +0800 Subject: [PATCH 3/9] docs: regenerate website API source anchors after merge --- website/zh-CN/api/harness/events.md | 8 ++++---- website/zh-CN/api/harness/sessions.md | 18 +++++++++--------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/website/zh-CN/api/harness/events.md b/website/zh-CN/api/harness/events.md index 9cad9215fb..bd6efa8f8c 100644 --- a/website/zh-CN/api/harness/events.md +++ b/website/zh-CN/api/harness/events.md @@ -571,7 +571,7 @@ Creation announcement during session publication. A synchronous throw vetoes and - `session` — the session just entered and announced. -[Source](https://github.com/deepseek-harness/deepseek-harness/blob/master/packages/core/session/src/index.ts#L47) +[Source](https://github.com/deepseek-harness/deepseek-harness/blob/master/packages/core/session/src/index.ts#L49) ### session/disposed @@ -594,7 +594,7 @@ Emitted once when an announced session leaves the store, including publication r - `session` — the session that is no longer live in the store. -[Source](https://github.com/deepseek-harness/deepseek-harness/blob/master/packages/core/session/src/index.ts#L57) +[Source](https://github.com/deepseek-harness/deepseek-harness/blob/master/packages/core/session/src/index.ts#L59) ### session/event @@ -620,7 +620,7 @@ Post-commit, fire-and-forget append feed. The listener snapshot resolves before - `session` — the session whose log grew. - `event` — the appended event, exactly as recorded. -[Source](https://github.com/deepseek-harness/deepseek-harness/blob/master/packages/core/session/src/index.ts#L69) +[Source](https://github.com/deepseek-harness/deepseek-harness/blob/master/packages/core/session/src/index.ts#L71) ### session/flush @@ -643,7 +643,7 @@ Awaited parallel durability checkpoint: every listener runs and the caller await - `session` — the session whose buffered events must reach durable storage. -[Source](https://github.com/deepseek-harness/deepseek-harness/blob/master/packages/core/session/src/index.ts#L79) +[Source](https://github.com/deepseek-harness/deepseek-harness/blob/master/packages/core/session/src/index.ts#L81) ## subagent/* diff --git a/website/zh-CN/api/harness/sessions.md b/website/zh-CN/api/harness/sessions.md index f59001009d..5ac6bf2fa2 100644 --- a/website/zh-CN/api/harness/sessions.md +++ b/website/zh-CN/api/harness/sessions.md @@ -7,7 +7,7 @@ In-memory session store (`ctx.sessions`). Persistence is intentionally not implemented here — persistence plugins subscribe to `session/event` and flush on `session/flush` / dispose. -[Source](https://github.com/deepseek-harness/deepseek-harness/blob/master/packages/core/session/src/index.ts#L577) +[Source](https://github.com/deepseek-harness/deepseek-harness/blob/master/packages/core/session/src/index.ts#L579) ### ctx.sessions.create(id?, options?) @@ -44,7 +44,7 @@ For an agent whose session must be torn down IN ORDER with its loop (so the loop **Returns** the live session, already entered and announced. -[Source](https://github.com/deepseek-harness/deepseek-harness/blob/master/packages/core/session/src/index.ts#L606) +[Source](https://github.com/deepseek-harness/deepseek-harness/blob/master/packages/core/session/src/index.ts#L608) ### ctx.sessions.prepare(id?, options?) @@ -75,7 +75,7 @@ Build a session WITHOUT entering it into the store — validate the id/cwd and c **Returns** the constructed session, NOT yet in the store. -[Source](https://github.com/deepseek-harness/deepseek-harness/blob/master/packages/core/session/src/index.ts#L635) +[Source](https://github.com/deepseek-harness/deepseek-harness/blob/master/packages/core/session/src/index.ts#L637) ### ctx.sessions.enter(session) @@ -112,7 +112,7 @@ Re-checks the id for a duplicate: `prepare` and `enter` are public cross-package **Returns** the detach disposer (publication hooks + store removal). When called from a synchronous `session/created` listener, removal and disposal wait until that creation dispatch unwinds. -[Source](https://github.com/deepseek-harness/deepseek-harness/blob/master/packages/core/session/src/index.ts#L679) +[Source](https://github.com/deepseek-harness/deepseek-harness/blob/master/packages/core/session/src/index.ts#L681) ### ctx.sessions.announce(session) @@ -131,7 +131,7 @@ Emit `session/created` exactly once for an entered session (with the carrier ent - `session` — the entered session to announce to listeners. -[Source](https://github.com/deepseek-harness/deepseek-harness/blob/master/packages/core/session/src/index.ts#L734) +[Source](https://github.com/deepseek-harness/deepseek-harness/blob/master/packages/core/session/src/index.ts#L736) ### ctx.sessions.flush(session) @@ -156,7 +156,7 @@ Dispatch the awaited `session/flush` durability checkpoint for `session`, with t **Returns** resolves when every flush listener has settled; after all settle, rejects with the first registered listener failure if any listener failed. -[Source](https://github.com/deepseek-harness/deepseek-harness/blob/master/packages/core/session/src/index.ts#L786) +[Source](https://github.com/deepseek-harness/deepseek-harness/blob/master/packages/core/session/src/index.ts#L788) ### ctx.sessions.get(id) @@ -175,7 +175,7 @@ Look up a live session. **Returns** the session, or undefined when no live session has that id. -[Source](https://github.com/deepseek-harness/deepseek-harness/blob/master/packages/core/session/src/index.ts#L818) +[Source](https://github.com/deepseek-harness/deepseek-harness/blob/master/packages/core/session/src/index.ts#L820) ### ctx.sessions.list() @@ -191,7 +191,7 @@ All live sessions, in creation order. **Returns** a fresh array; mutating it does not affect the store. -[Source](https://github.com/deepseek-harness/deepseek-harness/blob/master/packages/core/session/src/index.ts#L826) +[Source](https://github.com/deepseek-harness/deepseek-harness/blob/master/packages/core/session/src/index.ts#L828) ### ctx.sessions.fork(source, boundary?, childSessionId?) @@ -220,4 +220,4 @@ Create a live child session from a turn-enclosed prefix of a live source. `bound **Returns** The created live child session. -[Source](https://github.com/deepseek-harness/deepseek-harness/blob/master/packages/core/session/src/index.ts#L843) +[Source](https://github.com/deepseek-harness/deepseek-harness/blob/master/packages/core/session/src/index.ts#L845) From 568d9a70d681e8d775c428ff6468cca7c7bc2456 Mon Sep 17 00:00:00 2001 From: kingwl Date: Mon, 20 Jul 2026 20:05:54 +0800 Subject: [PATCH 4/9] test(session-persistence-jsonl): add required delegationDepth to packed-row fixtures Master made delegationDepth mandatory on the on-disk header; the packed-chunk tests' handwritten header lines predate that and were rejected at load. --- .../session-persistence-jsonl/tests/jsonl.spec.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts b/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts index c7428e0836..b1cc2fed2c 100644 --- a/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts +++ b/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts @@ -595,7 +595,7 @@ describe('SessionPersistenceJsonl: packed chunk rows (packChunks: true)', () => // file, hand-planted so this packed-config backend adopts it on load). await mkdir(sessionDir(root, '/work'), { recursive: true }) await writeFile(logPath(root, '/work', m.id), [ - JSON.stringify({ type: 'session', version: 0, id: 'mixed', createdAt: 1000, cwd: '/work' }), + JSON.stringify({ type: 'session', version: 0, id: 'mixed', createdAt: 1000, cwd: '/work', delegationDepth: 0 }), ...log.map(e => JSON.stringify(e)), ].join('\n') + '\n') // Adopt the stored log (cursor = stored length), then append a second turn @@ -619,7 +619,7 @@ describe('SessionPersistenceJsonl: packed chunk rows (packChunks: true)', () => it('scanLog: a packed row advances the seq cursor by its whole run', () => { const logText = [ - JSON.stringify({ type: 'session', version: 0, id: 'rows', createdAt: 1 }), + JSON.stringify({ type: 'session', version: 0, id: 'rows', createdAt: 1, delegationDepth: 0 }), JSON.stringify({ type: 'turn/start', seq: 0, time: 1, data: { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } } }), JSON.stringify({ type: 'text-chunks', seq0: 1, time0: 2, data: { turn: 1, step: 1, index: 0, dt: [1, 1], texts: ['a', 'b', 'c'] } }), JSON.stringify({ type: 'turn/end', seq: 4, time: 5, data: { turn: 1, reason: { kind: 'completed' } } }), @@ -631,7 +631,7 @@ describe('SessionPersistenceJsonl: packed chunk rows (packChunks: true)', () => it('scanLog: a malformed packed row in the committed region rejects like corrupt JSON', () => { const logText = [ - JSON.stringify({ type: 'session', version: 0, id: 'bad-row', createdAt: 1 }), + JSON.stringify({ type: 'session', version: 0, id: 'bad-row', createdAt: 1, delegationDepth: 0 }), // dt arity mismatch — row validation throws, so the line is a committed hole. JSON.stringify({ type: 'text-chunks', seq0: 0, time0: 1, data: { turn: 1, step: 1, index: 0, dt: [], texts: ['a', 'b'] } }), JSON.stringify({ type: 'turn/end', seq: 2, time: 3, data: { turn: 1, reason: { kind: 'completed' } } }), @@ -641,7 +641,7 @@ describe('SessionPersistenceJsonl: packed chunk rows (packChunks: true)', () => it('scanLog: a packed row with a mid-run seq gap after the last turn/end drops the whole row', () => { const logText = [ - JSON.stringify({ type: 'session', version: 0, id: 'row-gap', createdAt: 1 }), + JSON.stringify({ type: 'session', version: 0, id: 'row-gap', createdAt: 1, delegationDepth: 0 }), JSON.stringify({ type: 'turn/start', seq: 0, time: 1, data: { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } } }), // seq0 skips 1 — the run's first member is already a gap; no turn/end follows. JSON.stringify({ type: 'text-chunks', seq0: 2, time0: 2, data: { turn: 1, step: 1, index: 0, dt: [1, 1], texts: ['a', 'b', 'c'] } }), From 3869f2ce17a8eb64512ecc2e0260298866355361 Mon Sep 17 00:00:00 2001 From: kingwl Date: Mon, 20 Jul 2026 20:19:32 +0800 Subject: [PATCH 5/9] chore(acp-demo): fence the persistence passthrough clone for jscpd The acp/stdio front doors intentionally duplicate their small persistence passthrough blocks (same rationale as their Config schemas, already fenced). --- packages/examples/acp-demo/src/index.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/examples/acp-demo/src/index.ts b/packages/examples/acp-demo/src/index.ts index cf95101e14..c1b9989a3f 100644 --- a/packages/examples/acp-demo/src/index.ts +++ b/packages/examples/acp-demo/src/index.ts @@ -98,10 +98,14 @@ export const Config: z = z.object({ export function apply(ctx: Context, config: Config): void { ctx.plugin(agentCore, agentCore.pickSpineConfig(config)) ctx.plugin(UserInteractionService) + // Same rationale as the Config schema above: each front door forwards its own + // persistence passthroughs rather than sharing a facade with stdio-demo. + /* jscpd:ignore-start */ ctx.plugin(SessionPersistenceJsonl, { root: config.persistenceRoot ?? DEFAULT_PERSISTENCE_ROOT, ...config.packChunks !== undefined ? { packChunks: config.packChunks } : {}, ...(config.persistenceCompression === undefined ? {} : { compression: config.persistenceCompression }), }) + /* jscpd:ignore-end */ ctx.plugin(acp, { provider: config.provider, model: config.model }) } From f4ad7310791c289153b03d297c235bcf0cf16b73 Mon Sep 17 00:00:00 2001 From: kingwl Date: Wed, 22 Jul 2026 16:46:13 +0800 Subject: [PATCH 6/9] test(snapshot): add the packed-chunks opt-in replay scenario MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An overlay pair switches the acp demo's packChunks on; the authored fixture (text-turn's recording refreshed through that overlay) commits a session.jsonl whose reasoning run persists as one packed reasoning-chunks row while the short text run stays verbatim. Keyless replay proves the packed fixture derives the same model script (fixture reading is layout-blind) and the re-persisted log packs identically — the transcript surface of the packChunks switch is now pinned without touching any existing golden. --- .../packed-chunks.cordis.snapshot.yml | 45 +++++++++++++++++++ examples/acp-agent/packed-chunks.cordis.yml | 23 ++++++++++ examples/acp-agent/tests/acp.snapshot.ts | 8 ++++ .../tests/snapshots/packed-chunks/input.json | 7 +++ .../snapshots/packed-chunks/session.jsonl | 18 ++++++++ .../packed-chunks/stdout.expected.jsonl | 27 +++++++++++ 6 files changed, 128 insertions(+) create mode 100644 examples/acp-agent/packed-chunks.cordis.snapshot.yml create mode 100644 examples/acp-agent/packed-chunks.cordis.yml create mode 100644 examples/acp-agent/tests/snapshots/packed-chunks/input.json create mode 100644 examples/acp-agent/tests/snapshots/packed-chunks/session.jsonl create mode 100644 examples/acp-agent/tests/snapshots/packed-chunks/stdout.expected.jsonl diff --git a/examples/acp-agent/packed-chunks.cordis.snapshot.yml b/examples/acp-agent/packed-chunks.cordis.snapshot.yml new file mode 100644 index 0000000000..11ca2bbe71 --- /dev/null +++ b/examples/acp-agent/packed-chunks.cordis.snapshot.yml @@ -0,0 +1,45 @@ +# Keyless replay counterpart of packed-chunks.cordis.yml. Patches do not +# compose across includes, so this applies the packChunks config and the +# DeepSeek-to-replay swap directly to `cordis.yml`. +- id: base + name: '@cordisjs/plugin-include' + config: + path: ./cordis.yml + patches: + - id: llm-deepseek + name: '@deepseek-ai/dsh-llm-deepseek' + disabled: true + - id: sandbox + name: '@deepseek-ai/dsh-sandbox-local' + config: + runnerCommand: + - bash + - -c + - while [ "$1" != "--" ]; do shift; done; shift; exec "$@" + - passthrough-runner + runnerFailureSignatures: + - 'passthrough-runner: profile rejected' + - id: acp-agent + name: '@deepseek-ai/dsh-acp-demo' + config: + provider: deepseek + model: deepseek-v4-flash + persistenceRoot: !!js process.env.DSH_SNAPSHOT_SESSIONS_ROOT ?? './.sessions' + persistenceCompression: 'none' + packChunks: true + workspaceContext: + maxBytes: 65536 + persona: | + You are a coding assistant powered by the {{model}} model. Your working directory is {{cwd}}. Your bash tool runs under a file sandbox — a `[sandbox: file access denied …]` result is policy, not a command bug. + + Verify your work by running the code or tests. Keep answers brief and factual. + - insert: + - id: llm-replay + name: '@deepseek-ai/dsh-llm-replay' + config: + providers: + - id: deepseek + name: DeepSeek + models: + - id: deepseek-v4-flash + - id: deepseek-v4-pro diff --git a/examples/acp-agent/packed-chunks.cordis.yml b/examples/acp-agent/packed-chunks.cordis.yml new file mode 100644 index 0000000000..c44a4764e8 --- /dev/null +++ b/examples/acp-agent/packed-chunks.cordis.yml @@ -0,0 +1,23 @@ +# The packed-chunk-rows overlay: the base tree with the JSONL backend's +# `packChunks` switched on, so delta-chunk runs persist as packed storage rows. +# A config patch replaces the whole app config, so unchanged base fields are +# restated below. +- id: base + name: '@cordisjs/plugin-include' + config: + path: ./cordis.yml + patches: + - id: acp-agent + name: '@deepseek-ai/dsh-acp-demo' + config: + provider: deepseek + model: deepseek-v4-flash + persistenceRoot: !!js process.env.DSH_SNAPSHOT_SESSIONS_ROOT ?? './.sessions' + persistenceCompression: !!js "process.env.DSH_SNAPSHOT === undefined ? 'zstd' : 'none'" + packChunks: true + workspaceContext: + maxBytes: 65536 + persona: | + You are a coding assistant powered by the {{model}} model. Your working directory is {{cwd}}. Your bash tool runs under a file sandbox — a `[sandbox: file access denied …]` result is policy, not a command bug. + + Verify your work by running the code or tests. Keep answers brief and factual. diff --git a/examples/acp-agent/tests/acp.snapshot.ts b/examples/acp-agent/tests/acp.snapshot.ts index 65df474f97..7b3120ff01 100644 --- a/examples/acp-agent/tests/acp.snapshot.ts +++ b/examples/acp-agent/tests/acp.snapshot.ts @@ -31,6 +31,7 @@ const WORKSPACE_CONTEXT_CONFIG = fileURLToPath(new URL('../workspace-context.cor const ADVANCED_CONFIG = fileURLToPath(new URL('../advanced.cordis.yml', import.meta.url)) const FS_CONFIG = fileURLToPath(new URL('../fs.cordis.yml', import.meta.url)) const DEPTH_TWO_CONFIG = fileURLToPath(new URL('../depth-two.cordis.yml', import.meta.url)) +const PACKED_CHUNKS_CONFIG = fileURLToPath(new URL('../packed-chunks.cordis.yml', import.meta.url)) const LSP_CONFIG = fileURLToPath(new URL('./lsp.cordis.yml', import.meta.url)) function snapshotModeFromEnv(value: string | undefined): SnapshotSuiteOptions['mode'] { @@ -57,6 +58,13 @@ const SCENARIOS: Scenario[] = [ // Its prompt and tool-schema sidecars pin the composed header. { name: 'text-turn', hasModelTurn: true, recorded: true, pinsHeader: true }, { name: 'tool-call-turn', hasModelTurn: true, recorded: true }, + // Authored from text-turn's recording: the same single text turn persisted + // through the packChunks overlay. The committed session.jsonl carries a + // packed `reasoning-chunks` row (the reasoning stream is the ≥3-delta run; + // the two text deltas stay verbatim), so replay proves a packed fixture + // derives the same model script (reading is layout-blind) and the + // re-persisted log packs identically. + { name: 'packed-chunks', hasModelTurn: true, recorded: false, configPath: PACKED_CHUNKS_CONFIG }, // The fs overlay only adds the spill stack (the sandboxed filesystem tools // live in the base tree), so these scenarios share the default header class. { diff --git a/examples/acp-agent/tests/snapshots/packed-chunks/input.json b/examples/acp-agent/tests/snapshots/packed-chunks/input.json new file mode 100644 index 0000000000..5fe0259a4e --- /dev/null +++ b/examples/acp-agent/tests/snapshots/packed-chunks/input.json @@ -0,0 +1,7 @@ +{ + "steps": [ + { "op": "initialize" }, + { "op": "newSession" }, + { "op": "prompt", "text": "Reply with exactly the word: PONG. Do not use any tools." } + ] +} diff --git a/examples/acp-agent/tests/snapshots/packed-chunks/session.jsonl b/examples/acp-agent/tests/snapshots/packed-chunks/session.jsonl new file mode 100644 index 0000000000..4f6b5bbc0c --- /dev/null +++ b/examples/acp-agent/tests/snapshots/packed-chunks/session.jsonl @@ -0,0 +1,18 @@ +{"type":"session","version":0,"id":"539aa64c-7f37-40ff-abd8-ed45b717be1b","createdAt":1783600629539,"cwd":"/var/folders/bn/vj1dvck95yd5jh3x4wskflxm0000gn/T/acp-snap-cwd-ka5r8w","delegationDepth":0} +{"type":"turn/start","seq":0,"time":1783600629541,"data":{"turn":1,"trigger":{"kind":"message","source":{"kind":"user"}}}} +{"type":"user/message","seq":1,"time":1783600629541,"data":{"content":[{"type":"text","text":"Reply with exactly the word: PONG. Do not use any tools."}],"source":{"kind":"user"}},"surfaceOp":"append"} +{"type":"session/title","seq":2,"time":1783600629541,"data":{"title":"Reply with exactly the word:","messageSeqs":[1],"source":{"kind":"fallback"}}} +{"type":"step/start","seq":3,"time":1783600629542,"data":{"turn":1,"step":1}} +{"type":"request/header","seq":4,"time":1783600629542,"data":{"header":{"config":{"provider":"deepseek","model":"deepseek-v4-flash"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}} +{"type":"assistant/chunk","seq":5,"time":1783600630819,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"reasoning"}}} +{"type":"reasoning-chunks","seq0":6,"time0":1784709770168,"data":{"turn":1,"step":1,"index":0,"dt":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0],"texts":["The"," user"," wants"," me"," to"," reply"," with"," exactly"," the"," word"," \"","P","ONG","\""," and"," not"," use"," any"," tools","."]}} +{"type":"assistant/chunk","seq":26,"time":1783600630822,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":1,"blockType":"text"}}} +{"type":"assistant/chunk","seq":27,"time":1783600630852,"data":{"turn":1,"step":1,"chunk":{"type":"text-delta","index":1,"text":"P"}}} +{"type":"assistant/chunk","seq":28,"time":1783600630852,"data":{"turn":1,"step":1,"chunk":{"type":"text-delta","index":1,"text":"ONG"}}} +{"type":"assistant/chunk","seq":29,"time":1783600630852,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":0,"block":{"type":"reasoning","text":"The user wants me to reply with exactly the word \"PONG\" and not use any tools."}}}} +{"type":"assistant/chunk","seq":30,"time":1783600630852,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":1,"block":{"type":"text","text":"PONG"}}}} +{"type":"assistant/chunk","seq":31,"time":1783600630885,"data":{"turn":1,"step":1,"chunk":{"type":"usage","usage":{"inputTokens":3091,"outputTokens":23,"cacheReadTokens":0,"reasoningTokens":20}}}} +{"type":"assistant/chunk","seq":32,"time":1783600630886,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"stop"}}}} +{"type":"assistant/message","seq":33,"time":1784709770169,"data":{"turn":1,"step":1,"content":[{"type":"reasoning","text":"The user wants me to reply with exactly the word \"PONG\" and not use any tools."},{"type":"text","text":"PONG"}],"provenance":{"provider":"deepseek","model":"deepseek-v4-flash"},"usage":{"inputTokens":3091,"outputTokens":23,"cacheReadTokens":0,"reasoningTokens":20}},"sourceEventSeqs":[5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32],"surfaceOp":"append"} +{"type":"step/end","seq":34,"time":1784709770170,"data":{"turn":1,"step":1}} +{"type":"turn/end","seq":35,"time":1784709770170,"data":{"turn":1,"reason":{"kind":"completed"}}} diff --git a/examples/acp-agent/tests/snapshots/packed-chunks/stdout.expected.jsonl b/examples/acp-agent/tests/snapshots/packed-chunks/stdout.expected.jsonl new file mode 100644 index 0000000000..bba9f955f3 --- /dev/null +++ b/examples/acp-agent/tests/snapshots/packed-chunks/stdout.expected.jsonl @@ -0,0 +1,27 @@ +{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentInfo":{"name":"deepseek-harness-acp","version":"0.0.1"},"agentCapabilities":{"loadSession":true,"promptCapabilities":{"image":false,"audio":false,"embeddedContext":false}},"authMethods":[]}} +{"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}","configOptions":[{"id":"model","name":"Model","description":"Sets this session's provider and model.","category":"model","type":"select","currentValue":"[\"deepseek\",\"deepseek-v4-flash\"]","options":[{"value":"[\"deepseek\",\"deepseek-v4-flash\"]","name":"deepseek-v4-flash"},{"value":"[\"deepseek\",\"deepseek-v4-pro\"]","name":"deepseek-v4-pro"}]},{"id":"permission","name":"Permissions","description":"The session permission preset: each choice bundles a sandbox mode and an approval policy.","category":"mode","type":"select","currentValue":"danger-full-access","options":[{"value":"workspace-write","name":"workspace-write","description":"Write inside the workspace and permitted temporary directories; wider retries require approval."},{"value":"danger-full-access","name":"danger-full-access","description":"Full file access without approval prompts."}]}]}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"available_commands_update","availableCommands":[{"name":"goal","description":"set or view the goal for a long-running task","input":{"hint":"[|clear|edit |pause|resume]"}}]}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"session_info_update","title":"Reply with exactly the word:","updatedAt":"{{updatedAt}}"}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":"The"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" user"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" wants"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" me"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" to"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" reply"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" with"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" exactly"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" the"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" word"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" \""}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":"P"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":"ONG"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":"\""}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" and"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" not"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" use"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" any"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" tools"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":"."}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"P"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"ONG"}}}} +{"jsonrpc":"2.0","id":3,"result":{"stopReason":"end_turn"}} From 4529a22e850cde927713b0d1fd0f7da29aedc903 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 20:56:27 +0800 Subject: [PATCH 7/9] fix(snapshot): preserve packed chunk timing on refresh --- .../2026-07-08-shared-acp-snapshot-package.md | 2 +- packages/support/acp-snapshot/README.md | 2 +- packages/support/acp-snapshot/src/suite.ts | 24 +++++++++++- .../support/acp-snapshot/tests/suite.spec.ts | 37 +++++++++++++++++++ 4 files changed, 61 insertions(+), 4 deletions(-) diff --git a/.agents/notes/implemented/testing/2026-07-08-shared-acp-snapshot-package.md b/.agents/notes/implemented/testing/2026-07-08-shared-acp-snapshot-package.md index aadeaeea30..7b148adf78 100644 --- a/.agents/notes/implemented/testing/2026-07-08-shared-acp-snapshot-package.md +++ b/.agents/notes/implemented/testing/2026-07-08-shared-acp-snapshot-package.md @@ -18,7 +18,7 @@ The machinery lives in [`packages/support/acp-snapshot`](../../../../packages/su **`src/normalize.ts`** — the pure normalizers, hook-free by policy: when a future event carries a new volatile field (an approval duration, say), the shared normalizer learns it in the same change, keeping one home for what "normalized" means rather than per-suite scrub extensions. -**`src/suite.ts`** — the `Scenario` type and `defineAcpSnapshotSuite(options)`, registering the per-scenario compares, record/refresh fixture write-back, the header pin with its live uniformity guard, and the fixture guard block (no orphan scenario dirs, required files present, exactly one pin per class, every JSONL a `scrubSystemPrompts` fixed point, non-pinning fixtures also `scrubRequestHeaders` fixed points). A scenario directory's `session.jsonl` plus contiguous `session..jsonl` siblings are its ordered primary/child inventory, so the scenario table declares policy without duplicating a child count. The pinned-header contract ([pinned-header Agent Note](2026-07-06-pin-request-header-content-in-one-scenario.md)) is per-suite: each header class flags exactly one `pinsHeader` scenario, whose `system-prompt.expected.md` and JSONL tool list split the composed header into reviewable artifacts; the uniformity guard compares both against every live header in that class. A pinning scenario declares any legitimate changed-header count, and its Markdown artifact records every full changed prompt. The pure helpers (`sessionFixtureNames`, `fixtureContext`, `normalizedHeaders`, `normalizedSystemPrompts`, `formatSystemPromptSnapshot`, `headerChangeCount`) are exported from the module for direct unit coverage. +**`src/suite.ts`** — the `Scenario` type and `defineAcpSnapshotSuite(options)`, registering the per-scenario compares, record/refresh fixture write-back, the header pin with its live uniformity guard, and the fixture guard block (no orphan scenario dirs, required files present, exactly one pin per class, every JSONL a `scrubSystemPrompts` fixed point, non-pinning fixtures also `scrubRequestHeaders` fixed points). Refresh retains existing volatile event times and packed-run anchors, plus member gaps when the run arity is unchanged; fresh chunk-fragment arrays remain authoritative because their boundaries are replay behavior. A scenario directory's `session.jsonl` plus contiguous `session..jsonl` siblings are its ordered primary/child inventory, so the scenario table declares policy without duplicating a child count. The pinned-header contract ([pinned-header Agent Note](2026-07-06-pin-request-header-content-in-one-scenario.md)) is per-suite: each header class flags exactly one `pinsHeader` scenario, whose `system-prompt.expected.md` and JSONL tool list split the composed header into reviewable artifacts; the uniformity guard compares both against every live header in that class. A pinning scenario declares any legitimate changed-header count, and its Markdown artifact records every full changed prompt. The pure helpers (`sessionFixtureNames`, `fixtureContext`, `normalizedHeaders`, `normalizedSystemPrompts`, `formatSystemPromptSnapshot`, `headerChangeCount`) are exported from the module for direct unit coverage. ## Alternatives considered diff --git a/packages/support/acp-snapshot/README.md b/packages/support/acp-snapshot/README.md index 6e37534e73..3759f1c102 100644 --- a/packages/support/acp-snapshot/README.md +++ b/packages/support/acp-snapshot/README.md @@ -7,7 +7,7 @@ Four layers, importable separately: - **`launchAcpTestAgent` (launcher)** — boots a source agent under tsx or a built `lib` agent under plain Node from a temp cwd, connects the SDK client over a raw-byte stdout tee, collects session updates and stderr, surfaces asynchronous spawn failures through startup, fails closed on unhandled permission requests, and owns graceful or signalled shutdown. Shutdown waits for process exit, inherited stdio closure, and ACP parser exhaustion before resolving or propagating a child error, so captures are complete and callers can remove owned paths after either outcome. When Windows accepts forced termination but publishes its exit marker asynchronously, shutdown gives that marker a bounded grace before treating fallback refusal as a second failure. Snapshot and ordinary e2e suites share this process boundary; a test supplies only agent paths, cwd, environment overrides, and any permission policy. - **`runScenario` (harness)** — drives ACP JSON-RPC stdio from a deterministic `input.json` script through the launcher, tees raw stdout for the expected-output and purity checks, and harvests every persisted raw JSONL session log (parent and subagent children, primary-first) after graceful stdin EOF. `AgentUnderTest` supplies absolute `binScript`, optional `libBinScript`, `configPath`, and `tsconfigPath` paths because the subprocess cwd is outside the repo. Startup failures preserve captured agent stderr in the rejected diagnostic. - **Normalizers** — pure functions turning the two captured surfaces into stable text: `normalizeStdout` (JSON-RPC ids → first-seen sequence; UUIDs/cwd → tokens; cwd-rooted separators selected as canonical `/` or host-native; `session_info_update.updatedAt` → `{{updatedAt}}`; doubles as the stdout-purity check), `normalizeSessionLog` (times zeroed, `seq` kept, the same cwd-path policy), `scrubSystemPrompts` (prompt text → `{{system}}`), `scrubToolSchemas` (schema bulk → `{{tools}}`), and `scrubRequestHeaders` (all header bulk → `{{system}}`/`{{tools}}`/`{{messagePrefix}}` outside each pin, structure kept — [pinned-header Agent Note](../../../.agents/notes/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md)). -- **`defineAcpSnapshotSuite` (factory)** — registers the whole describe/it tree for a scenario table: per-scenario expected-output and re-persisted-log comparisons, record/refresh fixture write-back, rejection of structured `UNKNOWN_TOOL` results, the per-header-class pin (`system-prompt.expected.md` plus `tool-schemas.expected.json`) with its live uniformity guard, and the fixture guard block (no orphan scenario dirs, required files present, exactly one pin per class, every JSONL prompt/schema-scrubbed, non-pinning fixtures fully header-scrubbed). Refresh preserves existing volatile fields by event position and gives a newly inserted `session/title` its preceding event's time, so feature-driven insertions do not churn the remainder of a fixture. Each scenario directory's `session.jsonl` plus contiguous `session..jsonl` siblings are the ordered primary/child inventory; the scenario table does not duplicate their count. Must be called at vitest collection time. +- **`defineAcpSnapshotSuite` (factory)** — registers the whole describe/it tree for a scenario table: per-scenario expected-output and re-persisted-log comparisons, record/refresh fixture write-back, rejection of structured `UNKNOWN_TOOL` results, the per-header-class pin (`system-prompt.expected.md` plus `tool-schemas.expected.json`) with its live uniformity guard, and the fixture guard block (no orphan scenario dirs, required files present, exactly one pin per class, every JSONL prompt/schema-scrubbed, non-pinning fixtures fully header-scrubbed). Refresh preserves existing volatile fields by record position, including each packed run's anchor and, when its arity is unchanged, member gaps, without replacing fresh chunk-fragment arrays; a newly inserted `session/title` receives its preceding event's time so feature-driven insertions do not churn the remainder of a fixture. Each scenario directory's `session.jsonl` plus contiguous `session..jsonl` siblings are the ordered primary/child inventory; the scenario table does not duplicate their count. Must be called at vitest collection time. A consuming `*.snapshot.ts` is the scenario table plus one factory call: diff --git a/packages/support/acp-snapshot/src/suite.ts b/packages/support/acp-snapshot/src/suite.ts index ab6008913c..7cca816a3f 100644 --- a/packages/support/acp-snapshot/src/suite.ts +++ b/packages/support/acp-snapshot/src/suite.ts @@ -451,6 +451,25 @@ function preserveFixtureVolatiles(record: Record, existing: Rec return } if ('time' in record && 'time' in existing) record.time = existing.time + if ( + (record.type === 'text-chunks' || record.type === 'reasoning-chunks' || record.type === 'tool-call-chunks') + && 'time0' in record && 'time0' in existing + ) { + record.time0 = existing.time0 + const data = record.data + const existingData = existing.data + if (data !== null && typeof data === 'object' && existingData !== null && typeof existingData === 'object') { + const gaps = (data as { dt?: unknown }).dt + const existingGaps = (existingData as { dt?: unknown }).dt + // Equal arity means every preserved gap still belongs to the same fresh + // chunk position. Payload arrays remain fresh because their boundaries + // are meaningful replay behavior, not volatile timing. + if (Array.isArray(gaps) && Array.isArray(existingGaps) && gaps.length === existingGaps.length) { + const preservedGaps = existingGaps as unknown[] + (data as { dt: unknown[] }).dt = [...preservedGaps] + } + } + } if (record.type !== 'hook/result') return const data = record.data const existingData = existing.data @@ -466,8 +485,9 @@ function preserveFixtureVolatiles(record: Record, existing: Rec /** * Rewrite a fresh replay-produced log so repeated refreshes do not churn * volatile fixture fields. Meaningful event payloads come from `fresh`; the - * existing fixture lends session ids, cwd, creation times, event times, and - * hook durations where the record shape still matches. + * existing fixture lends session ids, cwd, creation times, event times, + * packed-run anchors and same-arity gaps, and hook durations where the record + * shape still matches. * * @param fresh The newly harvested session JSONL. * @param existing The committed fixture JSONL being refreshed. diff --git a/packages/support/acp-snapshot/tests/suite.spec.ts b/packages/support/acp-snapshot/tests/suite.spec.ts index b80b5a50d2..341728cd60 100644 --- a/packages/support/acp-snapshot/tests/suite.spec.ts +++ b/packages/support/acp-snapshot/tests/suite.spec.ts @@ -451,6 +451,43 @@ describe('refreshFixtureReplacements', () => { }) describe('stabilizeRefreshLog', () => { + it('preserves packed member times without flattening fresh chunk boundaries', () => { + const fresh = [ + '{"type":"session","id":"same","createdAt":200}', + '{"type":"text-chunks","seq0":2,"time0":200,"data":{"turn":1,"step":1,"index":0,"dt":[5,7],"texts":["new",""," split"]}}', + '', + ].join('\n') + const existing = [ + '{"type":"session","id":"same","createdAt":100}', + '{"type":"text-chunks","seq0":2,"time0":100,"data":{"turn":1,"step":1,"index":0,"dt":[1,2],"texts":["old","chunk","shape"]}}', + '', + ].join('\n') + + expect(stabilizeRefreshLog(fresh, existing, [])).toBe([ + '{"type":"session","id":"same","createdAt":100}', + '{"type":"text-chunks","seq0":2,"time0":100,"data":{"turn":1,"step":1,"index":0,"dt":[1,2],"texts":["new",""," split"]}}', + '', + ].join('\n')) + }) + + it.each([ + ['fresh data is null', null, { dt: [1, 2] }], + ['existing data is null', { dt: [5, 7] }, null], + ['fresh gaps are not an array', { dt: 'fresh' }, { dt: [1, 2] }], + ['existing gaps are not an array', { dt: [5, 7] }, { dt: 'existing' }], + ['the chunk arity changed', { dt: [5, 7, 9] }, { dt: [1, 2] }], + ])('keeps fresh packed gaps when %s', (_case, freshData, existingData) => { + const freshRow = { type: 'reasoning-chunks', seq0: 2, time0: 200, data: freshData } + const existingRow = { type: 'reasoning-chunks', seq0: 2, time0: 100, data: existingData } + const output = stabilizeRefreshLog( + `${JSON.stringify({ type: 'session', id: 'same', createdAt: 200 })}\n${JSON.stringify(freshRow)}\n`, + `${JSON.stringify({ type: 'session', id: 'same', createdAt: 100 })}\n${JSON.stringify(existingRow)}\n`, + [], + ).trim().split('\n').map(line => JSON.parse(line) as Record) + + expect(output[1]).toStrictEqual({ ...freshRow, time0: 100 }) + }) + it('aligns volatile times across a newly inserted log event', () => { const fresh = [ '{"type":"session","id":"same","createdAt":200}', From 479e590029ce2dbc6ebb5eb015fa869ef59cacf2 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 21:28:38 +0800 Subject: [PATCH 8/9] fix(snapshot): align packed refresh by logical event --- .../2026-07-08-shared-acp-snapshot-package.md | 2 +- .../snapshots/packed-chunks/session.jsonl | 22 +++--- packages/support/acp-snapshot/README.md | 2 +- packages/support/acp-snapshot/src/suite.ts | 70 ++++++++++++------- .../support/acp-snapshot/tests/suite.spec.ts | 53 +++++++++++--- 5 files changed, 102 insertions(+), 47 deletions(-) diff --git a/.agents/notes/implemented/testing/2026-07-08-shared-acp-snapshot-package.md b/.agents/notes/implemented/testing/2026-07-08-shared-acp-snapshot-package.md index 7b148adf78..8d6032d1cc 100644 --- a/.agents/notes/implemented/testing/2026-07-08-shared-acp-snapshot-package.md +++ b/.agents/notes/implemented/testing/2026-07-08-shared-acp-snapshot-package.md @@ -18,7 +18,7 @@ The machinery lives in [`packages/support/acp-snapshot`](../../../../packages/su **`src/normalize.ts`** — the pure normalizers, hook-free by policy: when a future event carries a new volatile field (an approval duration, say), the shared normalizer learns it in the same change, keeping one home for what "normalized" means rather than per-suite scrub extensions. -**`src/suite.ts`** — the `Scenario` type and `defineAcpSnapshotSuite(options)`, registering the per-scenario compares, record/refresh fixture write-back, the header pin with its live uniformity guard, and the fixture guard block (no orphan scenario dirs, required files present, exactly one pin per class, every JSONL a `scrubSystemPrompts` fixed point, non-pinning fixtures also `scrubRequestHeaders` fixed points). Refresh retains existing volatile event times and packed-run anchors, plus member gaps when the run arity is unchanged; fresh chunk-fragment arrays remain authoritative because their boundaries are replay behavior. A scenario directory's `session.jsonl` plus contiguous `session..jsonl` siblings are its ordered primary/child inventory, so the scenario table declares policy without duplicating a child count. The pinned-header contract ([pinned-header Agent Note](2026-07-06-pin-request-header-content-in-one-scenario.md)) is per-suite: each header class flags exactly one `pinsHeader` scenario, whose `system-prompt.expected.md` and JSONL tool list split the composed header into reviewable artifacts; the uniformity guard compares both against every live header in that class. A pinning scenario declares any legitimate changed-header count, and its Markdown artifact records every full changed prompt. The pure helpers (`sessionFixtureNames`, `fixtureContext`, `normalizedHeaders`, `normalizedSystemPrompts`, `formatSystemPromptSnapshot`, `headerChangeCount`) are exported from the module for direct unit coverage. +**`src/suite.ts`** — the `Scenario` type and `defineAcpSnapshotSuite(options)`, registering the per-scenario compares, record/refresh fixture write-back, the header pin with its live uniformity guard, and the fixture guard block (no orphan scenario dirs, required files present, exactly one pin per class, every JSONL a `scrubSystemPrompts` fixed point, non-pinning fixtures also `scrubRequestHeaders` fixed points). Refresh expands packed timing envelopes before aligning existing volatile event times, so switching between packed and unpacked layouts cannot shift later records; fresh chunk-fragment arrays remain authoritative because their boundaries are replay behavior. A scenario directory's `session.jsonl` plus contiguous `session..jsonl` siblings are its ordered primary/child inventory, so the scenario table declares policy without duplicating a child count. The pinned-header contract ([pinned-header Agent Note](2026-07-06-pin-request-header-content-in-one-scenario.md)) is per-suite: each header class flags exactly one `pinsHeader` scenario, whose `system-prompt.expected.md` and JSONL tool list split the composed header into reviewable artifacts; the uniformity guard compares both against every live header in that class. A pinning scenario declares any legitimate changed-header count, and its Markdown artifact records every full changed prompt. The pure helpers (`sessionFixtureNames`, `fixtureContext`, `normalizedHeaders`, `normalizedSystemPrompts`, `formatSystemPromptSnapshot`, `headerChangeCount`) are exported from the module for direct unit coverage. ## Alternatives considered diff --git a/examples/acp-agent/tests/snapshots/packed-chunks/session.jsonl b/examples/acp-agent/tests/snapshots/packed-chunks/session.jsonl index 4f6b5bbc0c..6c4e1d2a49 100644 --- a/examples/acp-agent/tests/snapshots/packed-chunks/session.jsonl +++ b/examples/acp-agent/tests/snapshots/packed-chunks/session.jsonl @@ -5,14 +5,14 @@ {"type":"step/start","seq":3,"time":1783600629542,"data":{"turn":1,"step":1}} {"type":"request/header","seq":4,"time":1783600629542,"data":{"header":{"config":{"provider":"deepseek","model":"deepseek-v4-flash"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}} {"type":"assistant/chunk","seq":5,"time":1783600630819,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"reasoning"}}} -{"type":"reasoning-chunks","seq0":6,"time0":1784709770168,"data":{"turn":1,"step":1,"index":0,"dt":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0],"texts":["The"," user"," wants"," me"," to"," reply"," with"," exactly"," the"," word"," \"","P","ONG","\""," and"," not"," use"," any"," tools","."]}} -{"type":"assistant/chunk","seq":26,"time":1783600630822,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":1,"blockType":"text"}}} -{"type":"assistant/chunk","seq":27,"time":1783600630852,"data":{"turn":1,"step":1,"chunk":{"type":"text-delta","index":1,"text":"P"}}} -{"type":"assistant/chunk","seq":28,"time":1783600630852,"data":{"turn":1,"step":1,"chunk":{"type":"text-delta","index":1,"text":"ONG"}}} -{"type":"assistant/chunk","seq":29,"time":1783600630852,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":0,"block":{"type":"reasoning","text":"The user wants me to reply with exactly the word \"PONG\" and not use any tools."}}}} -{"type":"assistant/chunk","seq":30,"time":1783600630852,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":1,"block":{"type":"text","text":"PONG"}}}} -{"type":"assistant/chunk","seq":31,"time":1783600630885,"data":{"turn":1,"step":1,"chunk":{"type":"usage","usage":{"inputTokens":3091,"outputTokens":23,"cacheReadTokens":0,"reasoningTokens":20}}}} -{"type":"assistant/chunk","seq":32,"time":1783600630886,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"stop"}}}} -{"type":"assistant/message","seq":33,"time":1784709770169,"data":{"turn":1,"step":1,"content":[{"type":"reasoning","text":"The user wants me to reply with exactly the word \"PONG\" and not use any tools."},{"type":"text","text":"PONG"}],"provenance":{"provider":"deepseek","model":"deepseek-v4-flash"},"usage":{"inputTokens":3091,"outputTokens":23,"cacheReadTokens":0,"reasoningTokens":20}},"sourceEventSeqs":[5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32],"surfaceOp":"append"} -{"type":"step/end","seq":34,"time":1784709770170,"data":{"turn":1,"step":1}} -{"type":"turn/end","seq":35,"time":1784709770170,"data":{"turn":1,"reason":{"kind":"completed"}}} +{"type":"reasoning-chunks","seq0":6,"time0":1783600630820,"data":{"turn":1,"step":1,"index":0,"dt":[2,30,0,0,0,33,1,40,0,0,0,0,0,18,0,36,0,0,0],"texts":["The"," user"," wants"," me"," to"," reply"," with"," exactly"," the"," word"," \"","P","ONG","\""," and"," not"," use"," any"," tools","."]}} +{"type":"assistant/chunk","seq":26,"time":1783600630980,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":1,"blockType":"text"}}} +{"type":"assistant/chunk","seq":27,"time":1783600630980,"data":{"turn":1,"step":1,"chunk":{"type":"text-delta","index":1,"text":"P"}}} +{"type":"assistant/chunk","seq":28,"time":1783600631006,"data":{"turn":1,"step":1,"chunk":{"type":"text-delta","index":1,"text":"ONG"}}} +{"type":"assistant/chunk","seq":29,"time":1783600631008,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":0,"block":{"type":"reasoning","text":"The user wants me to reply with exactly the word \"PONG\" and not use any tools."}}}} +{"type":"assistant/chunk","seq":30,"time":1783600631009,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":1,"block":{"type":"text","text":"PONG"}}}} +{"type":"assistant/chunk","seq":31,"time":1783600631009,"data":{"turn":1,"step":1,"chunk":{"type":"usage","usage":{"inputTokens":3091,"outputTokens":23,"cacheReadTokens":0,"reasoningTokens":20}}}} +{"type":"assistant/chunk","seq":32,"time":1783600631009,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"stop"}}}} +{"type":"assistant/message","seq":33,"time":1783600631011,"data":{"turn":1,"step":1,"content":[{"type":"reasoning","text":"The user wants me to reply with exactly the word \"PONG\" and not use any tools."},{"type":"text","text":"PONG"}],"provenance":{"provider":"deepseek","model":"deepseek-v4-flash"},"usage":{"inputTokens":3091,"outputTokens":23,"cacheReadTokens":0,"reasoningTokens":20}},"sourceEventSeqs":[5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32],"surfaceOp":"append"} +{"type":"step/end","seq":34,"time":1783600631011,"data":{"turn":1,"step":1}} +{"type":"turn/end","seq":35,"time":1783600631011,"data":{"turn":1,"reason":{"kind":"completed"}}} diff --git a/packages/support/acp-snapshot/README.md b/packages/support/acp-snapshot/README.md index 3759f1c102..37718b97e1 100644 --- a/packages/support/acp-snapshot/README.md +++ b/packages/support/acp-snapshot/README.md @@ -7,7 +7,7 @@ Four layers, importable separately: - **`launchAcpTestAgent` (launcher)** — boots a source agent under tsx or a built `lib` agent under plain Node from a temp cwd, connects the SDK client over a raw-byte stdout tee, collects session updates and stderr, surfaces asynchronous spawn failures through startup, fails closed on unhandled permission requests, and owns graceful or signalled shutdown. Shutdown waits for process exit, inherited stdio closure, and ACP parser exhaustion before resolving or propagating a child error, so captures are complete and callers can remove owned paths after either outcome. When Windows accepts forced termination but publishes its exit marker asynchronously, shutdown gives that marker a bounded grace before treating fallback refusal as a second failure. Snapshot and ordinary e2e suites share this process boundary; a test supplies only agent paths, cwd, environment overrides, and any permission policy. - **`runScenario` (harness)** — drives ACP JSON-RPC stdio from a deterministic `input.json` script through the launcher, tees raw stdout for the expected-output and purity checks, and harvests every persisted raw JSONL session log (parent and subagent children, primary-first) after graceful stdin EOF. `AgentUnderTest` supplies absolute `binScript`, optional `libBinScript`, `configPath`, and `tsconfigPath` paths because the subprocess cwd is outside the repo. Startup failures preserve captured agent stderr in the rejected diagnostic. - **Normalizers** — pure functions turning the two captured surfaces into stable text: `normalizeStdout` (JSON-RPC ids → first-seen sequence; UUIDs/cwd → tokens; cwd-rooted separators selected as canonical `/` or host-native; `session_info_update.updatedAt` → `{{updatedAt}}`; doubles as the stdout-purity check), `normalizeSessionLog` (times zeroed, `seq` kept, the same cwd-path policy), `scrubSystemPrompts` (prompt text → `{{system}}`), `scrubToolSchemas` (schema bulk → `{{tools}}`), and `scrubRequestHeaders` (all header bulk → `{{system}}`/`{{tools}}`/`{{messagePrefix}}` outside each pin, structure kept — [pinned-header Agent Note](../../../.agents/notes/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md)). -- **`defineAcpSnapshotSuite` (factory)** — registers the whole describe/it tree for a scenario table: per-scenario expected-output and re-persisted-log comparisons, record/refresh fixture write-back, rejection of structured `UNKNOWN_TOOL` results, the per-header-class pin (`system-prompt.expected.md` plus `tool-schemas.expected.json`) with its live uniformity guard, and the fixture guard block (no orphan scenario dirs, required files present, exactly one pin per class, every JSONL prompt/schema-scrubbed, non-pinning fixtures fully header-scrubbed). Refresh preserves existing volatile fields by record position, including each packed run's anchor and, when its arity is unchanged, member gaps, without replacing fresh chunk-fragment arrays; a newly inserted `session/title` receives its preceding event's time so feature-driven insertions do not churn the remainder of a fixture. Each scenario directory's `session.jsonl` plus contiguous `session..jsonl` siblings are the ordered primary/child inventory; the scenario table does not duplicate their count. Must be called at vitest collection time. +- **`defineAcpSnapshotSuite` (factory)** — registers the whole describe/it tree for a scenario table: per-scenario expected-output and re-persisted-log comparisons, record/refresh fixture write-back, rejection of structured `UNKNOWN_TOOL` results, the per-header-class pin (`system-prompt.expected.md` plus `tool-schemas.expected.json`) with its live uniformity guard, and the fixture guard block (no orphan scenario dirs, required files present, exactly one pin per class, every JSONL prompt/schema-scrubbed, non-pinning fixtures fully header-scrubbed). Refresh expands packed timing envelopes before aligning existing volatile event times, so switching between packed and unpacked layouts cannot shift later records; fresh chunk-fragment arrays remain authoritative. A newly inserted `session/title` receives its preceding event's time so feature-driven insertions do not churn the remainder of a fixture. Each scenario directory's `session.jsonl` plus contiguous `session..jsonl` siblings are the ordered primary/child inventory; the scenario table does not duplicate their count. Must be called at vitest collection time. A consuming `*.snapshot.ts` is the scenario table plus one factory call: diff --git a/packages/support/acp-snapshot/src/suite.ts b/packages/support/acp-snapshot/src/suite.ts index 7cca816a3f..e6780624de 100644 --- a/packages/support/acp-snapshot/src/suite.ts +++ b/packages/support/acp-snapshot/src/suite.ts @@ -42,6 +42,8 @@ const WINDOWS_STDOUT_SNAPSHOT = 'stdout.expected.windows.jsonl' /** Stable session-log token standing in for the sidecar's initial schemas. */ const TOOLS_TOKEN = '{{tools}}' +const PACKED_CHUNK_ROW_TYPES = new Set(['text-chunks', 'reasoning-chunks', 'tool-call-chunks']) + /** A snapshot scenario and how its fixtures are produced. */ export interface Scenario { name: string @@ -398,6 +400,23 @@ function parseJsonlRecords(text: string): Record[] { .map(line => JSON.parse(line) as Record) } +/** One packed row's member times, or `undefined` for an ordinary record. */ +function packedTimes(record: Record): number[] | undefined { + if (!PACKED_CHUNK_ROW_TYPES.has(record.type as string)) return undefined + const row = record as unknown as { time0: number; data: { dt: number[] } } + const times = [row.time0] + for (const gap of row.data.dt) times.push((times[times.length - 1] as number) + gap) + return times +} + +/** Expand packed timing envelopes so refresh alignment follows logical events, not physical lines. */ +function logicalRecords(records: Record[]): Record[] { + return records.flatMap((record) => { + const times = packedTimes(record) + return times === undefined ? [record] : times.map(time => ({ type: 'assistant/chunk', time })) + }) +} + /** * Find tool calls whose structured result reports `UNKNOWN_TOOL`. * @@ -451,25 +470,6 @@ function preserveFixtureVolatiles(record: Record, existing: Rec return } if ('time' in record && 'time' in existing) record.time = existing.time - if ( - (record.type === 'text-chunks' || record.type === 'reasoning-chunks' || record.type === 'tool-call-chunks') - && 'time0' in record && 'time0' in existing - ) { - record.time0 = existing.time0 - const data = record.data - const existingData = existing.data - if (data !== null && typeof data === 'object' && existingData !== null && typeof existingData === 'object') { - const gaps = (data as { dt?: unknown }).dt - const existingGaps = (existingData as { dt?: unknown }).dt - // Equal arity means every preserved gap still belongs to the same fresh - // chunk position. Payload arrays remain fresh because their boundaries - // are meaningful replay behavior, not volatile timing. - if (Array.isArray(gaps) && Array.isArray(existingGaps) && gaps.length === existingGaps.length) { - const preservedGaps = existingGaps as unknown[] - (data as { dt: unknown[] }).dt = [...preservedGaps] - } - } - } if (record.type !== 'hook/result') return const data = record.data const existingData = existing.data @@ -482,12 +482,32 @@ function preserveFixtureVolatiles(record: Record, existing: Rec } } +/** Carry logical member times into a fresh packed row while leaving its fragment arrays untouched. */ +function preservePackedMemberTimes( + record: Record, + existingMembers: Record[], +): void { + if (!PACKED_CHUNK_ROW_TYPES.has(record.type as string)) return + const row = record as unknown as { time0: number; data: { dt: number[] } } + const firstTime = existingMembers[0]?.time + if (!Number.isSafeInteger(firstTime)) return + row.time0 = firstTime as number + if (existingMembers.length !== row.data.dt.length + 1) return + const times = existingMembers.map(member => Number.isSafeInteger(member.time) ? member.time as number : undefined) + if (times.some(time => time === undefined)) return + const memberTimes = times as number[] + const gaps = memberTimes.slice(1).map((time, index) => time - (memberTimes[index] as number)) + if (gaps.some(gap => !Number.isSafeInteger(gap))) return + row.data.dt = gaps +} + /** * Rewrite a fresh replay-produced log so repeated refreshes do not churn * volatile fixture fields. Meaningful event payloads come from `fresh`; the - * existing fixture lends session ids, cwd, creation times, event times, - * packed-run anchors and same-arity gaps, and hook durations where the record - * shape still matches. + * existing fixture lends session ids, cwd, creation times, logical event + * times, and hook durations where the record shape still matches. Packed + * timing envelopes expand for alignment, so packing does not shift later + * records; fresh fragment arrays remain authoritative. * * @param fresh The newly harvested session JSONL. * @param existing The committed fixture JSONL being refreshed. @@ -497,21 +517,23 @@ function preserveFixtureVolatiles(record: Record, existing: Rec export function stabilizeRefreshLog(fresh: string, existing: string, replacements: FixtureReplacement[]): string { let stable = fresh for (const { from, to } of replacements) stable = stable.split(from).join(to) - const existingRecords = parseJsonlRecords(existing) + const existingRecords = logicalRecords(parseJsonlRecords(existing)) const records = parseJsonlRecords(stable) let existingIndex = 0 let previousEventTime: unknown for (let i = 0; i < records.length; i++) { const record = records[i] as Record const existingRecord = existingRecords[existingIndex] + const memberCount = packedTimes(record)?.length ?? 1 const insertedTitle = record.type === 'session/title' && existingRecord?.type !== 'session/title' if (insertedTitle) { /* v8 ignore next -- a title is turn-enclosed, so a preceding event time exists in every valid fixture. */ if (typeof previousEventTime !== 'number') throw new Error('acp-snapshot: inserted title has no preceding event time') record.time = previousEventTime } else { + preservePackedMemberTimes(record, existingRecords.slice(existingIndex, existingIndex + memberCount)) preserveFixtureVolatiles(record, existingRecord) - existingIndex += 1 + existingIndex += memberCount } if (typeof record.time === 'number') previousEventTime = record.time } diff --git a/packages/support/acp-snapshot/tests/suite.spec.ts b/packages/support/acp-snapshot/tests/suite.spec.ts index 341728cd60..5e781e3a30 100644 --- a/packages/support/acp-snapshot/tests/suite.spec.ts +++ b/packages/support/acp-snapshot/tests/suite.spec.ts @@ -451,6 +451,30 @@ describe('refreshFixtureReplacements', () => { }) describe('stabilizeRefreshLog', () => { + it('preserves unpacked member times when refresh first packs a chunk run', () => { + const fresh = [ + '{"type":"session","id":"same","createdAt":200}', + '{"type":"reasoning-chunks","seq0":2,"time0":200,"data":{"turn":1,"step":1,"index":0,"dt":[5,7],"texts":["new",""," split"]}}', + '{"type":"assistant/message","seq":5,"time":220,"data":{}}', + '', + ].join('\n') + const existing = [ + '{"type":"session","id":"same","createdAt":100}', + '{"type":"assistant/chunk","seq":2,"time":100,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"old"}}}', + '{"type":"assistant/chunk","seq":3,"time":101,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"chunk"}}}', + '{"type":"assistant/chunk","seq":4,"time":103,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"shape"}}}', + '{"type":"assistant/message","seq":5,"time":104,"data":{}}', + '', + ].join('\n') + + expect(stabilizeRefreshLog(fresh, existing, [])).toBe([ + '{"type":"session","id":"same","createdAt":100}', + '{"type":"reasoning-chunks","seq0":2,"time0":100,"data":{"turn":1,"step":1,"index":0,"dt":[1,2],"texts":["new",""," split"]}}', + '{"type":"assistant/message","seq":5,"time":104,"data":{}}', + '', + ].join('\n')) + }) + it('preserves packed member times without flattening fresh chunk boundaries', () => { const fresh = [ '{"type":"session","id":"same","createdAt":200}', @@ -471,21 +495,30 @@ describe('stabilizeRefreshLog', () => { }) it.each([ - ['fresh data is null', null, { dt: [1, 2] }], - ['existing data is null', { dt: [5, 7] }, null], - ['fresh gaps are not an array', { dt: 'fresh' }, { dt: [1, 2] }], - ['existing gaps are not an array', { dt: [5, 7] }, { dt: 'existing' }], - ['the chunk arity changed', { dt: [5, 7, 9] }, { dt: [1, 2] }], - ])('keeps fresh packed gaps when %s', (_case, freshData, existingData) => { - const freshRow = { type: 'reasoning-chunks', seq0: 2, time0: 200, data: freshData } - const existingRow = { type: 'reasoning-chunks', seq0: 2, time0: 100, data: existingData } + ['the old run is absent', [], 200], + ['the old run is shorter', [100, 101], 100], + ['a later old time is invalid', [100, 'invalid', 103], 100], + ['an old gap is not exactly representable', [Number.MIN_SAFE_INTEGER, Number.MAX_SAFE_INTEGER - 1, Number.MAX_SAFE_INTEGER - 1], Number.MIN_SAFE_INTEGER], + ])('keeps fresh packed gaps when %s', (_case, existingTimes, expectedTime0) => { + const freshRow = { + type: 'reasoning-chunks', + seq0: 2, + time0: 200, + data: { turn: 1, step: 1, index: 0, dt: [5, 7], texts: ['new', '', ' split'] }, + } + const existingRows = existingTimes.map((time, index) => ({ + type: 'assistant/chunk', + seq: index + 2, + time, + data: {}, + })) const output = stabilizeRefreshLog( `${JSON.stringify({ type: 'session', id: 'same', createdAt: 200 })}\n${JSON.stringify(freshRow)}\n`, - `${JSON.stringify({ type: 'session', id: 'same', createdAt: 100 })}\n${JSON.stringify(existingRow)}\n`, + `${JSON.stringify({ type: 'session', id: 'same', createdAt: 100 })}\n${existingRows.map(row => JSON.stringify(row)).join('\n')}\n`, [], ).trim().split('\n').map(line => JSON.parse(line) as Record) - expect(output[1]).toStrictEqual({ ...freshRow, time0: 100 }) + expect(output[1]).toStrictEqual({ ...freshRow, time0: expectedTime0 }) }) it('aligns volatile times across a newly inserted log event', () => { From 53c3ab135de6dc3ac85134e6568466778032fcdc Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 22:14:16 +0800 Subject: [PATCH 9/9] test(snapshot): cover all packed chunk row kinds --- .../testing/2026-06-19-acp-snapshot-tests.md | 2 + examples/acp-agent/tests/acp.snapshot.ts | 36 +++++++-- .../tests/snapshots/packed-chunks/input.json | 2 +- .../snapshots/packed-chunks/session.jsonl | 50 +++++++----- .../packed-chunks/stdout.expected.jsonl | 80 +++++++++++++++---- .../packed-chunks/workspace/hooks.json | 12 +++ 6 files changed, 140 insertions(+), 42 deletions(-) create mode 100644 examples/acp-agent/tests/snapshots/packed-chunks/workspace/hooks.json diff --git a/.agents/notes/implemented/testing/2026-06-19-acp-snapshot-tests.md b/.agents/notes/implemented/testing/2026-06-19-acp-snapshot-tests.md index f400701fcd..2529bb124e 100644 --- a/.agents/notes/implemented/testing/2026-06-19-acp-snapshot-tests.md +++ b/.agents/notes/implemented/testing/2026-06-19-acp-snapshot-tests.md @@ -18,6 +18,8 @@ A snapshot test boots the real ACP example, drives its stdio protocol from a det Each scenario's `session.jsonl` is harvested from a real run. `assistant/chunk` events reproduce the model streams; tool, message, and boundary events capture the harness behavior. One ordinary session artifact therefore serves as both replay source and behavioral expected output. +When a scenario pins an alternative physical storage layout, its fixture is mechanically derived from a real unpacked counterpart. The scenario test requires every intended storage-row kind and exact event-for-event equality after decoding before the ordinary replay and log comparison proves that the assembled process consumes and reproduces that layout. + ### Replay derives the model script from the log `llm-replay` short-circuits the provider-agnostic `llm/stream` waterfall. `deriveReplayScript()` groups recorded chunks by `(turn, step)` and serves one group per model call. The loop makes one stream call per step, so the grouping is exact and includes error finish chunks without special handling. diff --git a/examples/acp-agent/tests/acp.snapshot.ts b/examples/acp-agent/tests/acp.snapshot.ts index 7b3120ff01..682b9e39a5 100644 --- a/examples/acp-agent/tests/acp.snapshot.ts +++ b/examples/acp-agent/tests/acp.snapshot.ts @@ -1,6 +1,9 @@ import { fileURLToPath } from 'node:url' +import { readFileSync } from 'node:fs' import { dirname, join } from 'node:path' +import { expect, it } from 'vitest' import { defineAcpSnapshotSuite, type Scenario, type SnapshotSuiteOptions } from '@deepseek-ai/dsh-acp-snapshot' +import { decodeStorageRecord } from '@deepseek-ai/dsh-session' /** * The acp-agent example's snapshot suite: the scenario table for @@ -33,6 +36,15 @@ const FS_CONFIG = fileURLToPath(new URL('../fs.cordis.yml', import.meta.url)) const DEPTH_TWO_CONFIG = fileURLToPath(new URL('../depth-two.cordis.yml', import.meta.url)) const PACKED_CHUNKS_CONFIG = fileURLToPath(new URL('../packed-chunks.cordis.yml', import.meta.url)) const LSP_CONFIG = fileURLToPath(new URL('./lsp.cordis.yml', import.meta.url)) +const SNAPSHOTS_DIR = join(dirname(fileURLToPath(import.meta.url)), 'snapshots') +const PACKED_CHUNKS_SOURCE = 'hook-cc-pretool-deny' + +function fixtureRecords(name: string): unknown[] { + return readFileSync(join(SNAPSHOTS_DIR, name, 'session.jsonl'), 'utf8') + .trimEnd() + .split('\n') + .map(line => JSON.parse(line) as unknown) +} function snapshotModeFromEnv(value: string | undefined): SnapshotSuiteOptions['mode'] { switch (value) { @@ -58,12 +70,9 @@ const SCENARIOS: Scenario[] = [ // Its prompt and tool-schema sidecars pin the composed header. { name: 'text-turn', hasModelTurn: true, recorded: true, pinsHeader: true }, { name: 'tool-call-turn', hasModelTurn: true, recorded: true }, - // Authored from text-turn's recording: the same single text turn persisted - // through the packChunks overlay. The committed session.jsonl carries a - // packed `reasoning-chunks` row (the reasoning stream is the ≥3-delta run; - // the two text deltas stay verbatim), so replay proves a packed fixture - // derives the same model script (reading is layout-blind) and the - // re-persisted log packs identically. + // Authored from the real PACKED_CHUNKS_SOURCE recording under the same app + // composition. The contract below pins decoded equality and all three row + // kinds; replay additionally proves the assembled app re-packs identically. { name: 'packed-chunks', hasModelTurn: true, recorded: false, configPath: PACKED_CHUNKS_CONFIG }, // The fs overlay only adds the spill stack (the sandboxed filesystem tools // live in the base tree), so these scenarios share the default header class. @@ -208,7 +217,20 @@ const SCENARIOS: Scenario[] = [ defineAcpSnapshotSuite({ agent: AGENT, - snapshotsDir: join(dirname(fileURLToPath(import.meta.url)), 'snapshots'), + snapshotsDir: SNAPSHOTS_DIR, scenarios: SCENARIOS, mode: snapshotModeFromEnv(process.env.DSH_SNAPSHOT), }) + +it('packed ACP fixture retains every chunk row kind without changing the logical session', () => { + const source = fixtureRecords(PACKED_CHUNKS_SOURCE) + const packed = fixtureRecords('packed-chunks') + const rowTypes = packed.flatMap((record) => { + if (record === null || typeof record !== 'object') return [] + const type = (record as { type?: unknown }).type + return type === 'text-chunks' || type === 'reasoning-chunks' || type === 'tool-call-chunks' ? [type] : [] + }) + + expect([...new Set(rowTypes)].sort()).toStrictEqual(['reasoning-chunks', 'text-chunks', 'tool-call-chunks']) + expect([packed[0], ...packed.slice(1).flatMap(record => decodeStorageRecord(record))]).toStrictEqual(source) +}) diff --git a/examples/acp-agent/tests/snapshots/packed-chunks/input.json b/examples/acp-agent/tests/snapshots/packed-chunks/input.json index 5fe0259a4e..3d44990f9b 100644 --- a/examples/acp-agent/tests/snapshots/packed-chunks/input.json +++ b/examples/acp-agent/tests/snapshots/packed-chunks/input.json @@ -2,6 +2,6 @@ "steps": [ { "op": "initialize" }, { "op": "newSession" }, - { "op": "prompt", "text": "Reply with exactly the word: PONG. Do not use any tools." } + { "op": "prompt", "text": "Use the bash tool to run exactly: echo HELLO. Report the tool result you got back verbatim, then stop." } ] } diff --git a/examples/acp-agent/tests/snapshots/packed-chunks/session.jsonl b/examples/acp-agent/tests/snapshots/packed-chunks/session.jsonl index 6c4e1d2a49..9928cbe82d 100644 --- a/examples/acp-agent/tests/snapshots/packed-chunks/session.jsonl +++ b/examples/acp-agent/tests/snapshots/packed-chunks/session.jsonl @@ -1,18 +1,32 @@ -{"type":"session","version":0,"id":"539aa64c-7f37-40ff-abd8-ed45b717be1b","createdAt":1783600629539,"cwd":"/var/folders/bn/vj1dvck95yd5jh3x4wskflxm0000gn/T/acp-snap-cwd-ka5r8w","delegationDepth":0} -{"type":"turn/start","seq":0,"time":1783600629541,"data":{"turn":1,"trigger":{"kind":"message","source":{"kind":"user"}}}} -{"type":"user/message","seq":1,"time":1783600629541,"data":{"content":[{"type":"text","text":"Reply with exactly the word: PONG. Do not use any tools."}],"source":{"kind":"user"}},"surfaceOp":"append"} -{"type":"session/title","seq":2,"time":1783600629541,"data":{"title":"Reply with exactly the word:","messageSeqs":[1],"source":{"kind":"fallback"}}} -{"type":"step/start","seq":3,"time":1783600629542,"data":{"turn":1,"step":1}} -{"type":"request/header","seq":4,"time":1783600629542,"data":{"header":{"config":{"provider":"deepseek","model":"deepseek-v4-flash"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}} -{"type":"assistant/chunk","seq":5,"time":1783600630819,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"reasoning"}}} -{"type":"reasoning-chunks","seq0":6,"time0":1783600630820,"data":{"turn":1,"step":1,"index":0,"dt":[2,30,0,0,0,33,1,40,0,0,0,0,0,18,0,36,0,0,0],"texts":["The"," user"," wants"," me"," to"," reply"," with"," exactly"," the"," word"," \"","P","ONG","\""," and"," not"," use"," any"," tools","."]}} -{"type":"assistant/chunk","seq":26,"time":1783600630980,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":1,"blockType":"text"}}} -{"type":"assistant/chunk","seq":27,"time":1783600630980,"data":{"turn":1,"step":1,"chunk":{"type":"text-delta","index":1,"text":"P"}}} -{"type":"assistant/chunk","seq":28,"time":1783600631006,"data":{"turn":1,"step":1,"chunk":{"type":"text-delta","index":1,"text":"ONG"}}} -{"type":"assistant/chunk","seq":29,"time":1783600631008,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":0,"block":{"type":"reasoning","text":"The user wants me to reply with exactly the word \"PONG\" and not use any tools."}}}} -{"type":"assistant/chunk","seq":30,"time":1783600631009,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":1,"block":{"type":"text","text":"PONG"}}}} -{"type":"assistant/chunk","seq":31,"time":1783600631009,"data":{"turn":1,"step":1,"chunk":{"type":"usage","usage":{"inputTokens":3091,"outputTokens":23,"cacheReadTokens":0,"reasoningTokens":20}}}} -{"type":"assistant/chunk","seq":32,"time":1783600631009,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"stop"}}}} -{"type":"assistant/message","seq":33,"time":1783600631011,"data":{"turn":1,"step":1,"content":[{"type":"reasoning","text":"The user wants me to reply with exactly the word \"PONG\" and not use any tools."},{"type":"text","text":"PONG"}],"provenance":{"provider":"deepseek","model":"deepseek-v4-flash"},"usage":{"inputTokens":3091,"outputTokens":23,"cacheReadTokens":0,"reasoningTokens":20}},"sourceEventSeqs":[5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32],"surfaceOp":"append"} -{"type":"step/end","seq":34,"time":1783600631011,"data":{"turn":1,"step":1}} -{"type":"turn/end","seq":35,"time":1783600631011,"data":{"turn":1,"reason":{"kind":"completed"}}} +{"type":"session","version":0,"id":"ff1c1e99-3bd4-4ef8-a954-80d607d628ba","createdAt":1783352165190,"cwd":"/tmp/acp-snap-cwd-wDnkVo","delegationDepth":0} +{"type":"turn/start","seq":0,"time":1783352165195,"data":{"turn":1,"trigger":{"kind":"message","source":{"kind":"user"}}}} +{"type":"user/message","seq":1,"time":1783352165196,"data":{"content":[{"type":"text","text":"Use the bash tool to run exactly: echo HELLO. Report the tool result you got back verbatim, then stop."}],"source":{"kind":"user"}},"surfaceOp":"append"} +{"type":"session/title","seq":2,"time":1783352165196,"data":{"title":"Use the bash tool to","messageSeqs":[1],"source":{"kind":"fallback"}}} +{"type":"step/start","seq":3,"time":1783352165198,"data":{"turn":1,"step":1}} +{"type":"request/header","seq":4,"time":1783352165199,"data":{"header":{"config":{"provider":"deepseek","model":"deepseek-v4-flash"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}} +{"type":"assistant/chunk","seq":5,"time":1783352165899,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"reasoning"}}} +{"type":"reasoning-chunks","seq0":6,"time0":1783352165899,"data":{"turn":1,"step":1,"index":0,"dt":[149,27,0,0,1,0,0,28,0,1,0,0,28,0,27,0],"texts":["The"," user"," wants"," me"," to"," run"," a"," simple"," bash"," command"," and"," report"," the"," result"," verb","atim","."]}} +{"type":"assistant/chunk","seq":23,"time":1783352166218,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":1,"blockType":"tool-call"}}} +{"type":"tool-call-chunks","seq0":24,"time0":1783352166218,"data":{"turn":1,"step":1,"index":1,"dt":[32,0,0,28,1,0,0,29,0,1,0,27,1,28,0,0,0,29,0,28,0,0,0,31],"id":"call_00_JliP571Bh0QQ8QExbSPk0080","name":"bash","args":["","{","\"","command","\"",": ","\"","echo"," HE","LL","O","\"",", ","\"","description","\"",": ","\"","Run"," echo"," HE","LL","O","\"","}"]}} +{"type":"assistant/chunk","seq":49,"time":1783352166512,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":0,"block":{"type":"reasoning","text":"The user wants me to run a simple bash command and report the result verbatim."}}}} +{"type":"assistant/chunk","seq":50,"time":1783352166512,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":1,"block":{"type":"tool-call","id":"call_00_JliP571Bh0QQ8QExbSPk0080","name":"bash","arguments":"{\"command\": \"echo HELLO\", \"description\": \"Run echo HELLO\"}"}}}} +{"type":"assistant/chunk","seq":51,"time":1783352166512,"data":{"turn":1,"step":1,"chunk":{"type":"usage","usage":{"inputTokens":2878,"outputTokens":83,"cacheReadTokens":0,"reasoningTokens":17}}}} +{"type":"assistant/chunk","seq":52,"time":1783352166512,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}} +{"type":"assistant/message","seq":53,"time":1783352166514,"data":{"turn":1,"step":1,"content":[{"type":"reasoning","text":"The user wants me to run a simple bash command and report the result verbatim."},{"type":"tool-call","id":"call_00_JliP571Bh0QQ8QExbSPk0080","name":"bash","arguments":"{\"command\": \"echo HELLO\", \"description\": \"Run echo HELLO\"}"}],"provenance":{"provider":"deepseek","model":"deepseek-v4-flash"},"usage":{"inputTokens":2878,"outputTokens":83,"cacheReadTokens":0,"reasoningTokens":17}},"sourceEventSeqs":[5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52],"surfaceOp":"append"} +{"type":"tool/call","seq":54,"time":1783352166514,"data":{"turn":1,"step":1,"callId":"call_00_JliP571Bh0QQ8QExbSPk0080","name":"bash","arguments":"{\"command\": \"echo HELLO\", \"description\": \"Run echo HELLO\"}"}} +{"type":"hook/invoked","seq":55,"time":1783352166515,"data":{"turn":1,"point":"PreToolUse","dialect":"claude","handlerId":"claude:PreToolUse:1","matcher":"bash"}} +{"type":"hook/result","seq":56,"time":1783352166528,"data":{"turn":1,"point":"PreToolUse","handlerId":"claude:PreToolUse:1","decision":"block","exitCode":2,"stderrSummary":"bash is disabled by policy in this session","durationMs":12.367074000000684}} +{"type":"tool/result","seq":57,"time":1783352166528,"data":{"turn":1,"step":1,"callId":"call_00_JliP571Bh0QQ8QExbSPk0080","content":[{"type":"text","text":"Error: bash is disabled by policy in this session"}],"isError":true},"sourceEventSeqs":[54],"surfaceOp":"append"} +{"type":"step/end","seq":58,"time":1783352166529,"data":{"turn":1,"step":1}} +{"type":"step/start","seq":59,"time":1783352166529,"data":{"turn":1,"step":2}} +{"type":"assistant/chunk","seq":60,"time":1783352167307,"data":{"turn":1,"step":2,"chunk":{"type":"block-start","index":0,"blockType":"reasoning"}}} +{"type":"reasoning-chunks","seq0":61,"time0":1783352167308,"data":{"turn":1,"step":2,"index":0,"dt":[132,29,0,0,0,0,1,27,0,28,1,0,31,0,25,0,29,1,1,0],"texts":["The"," bash"," tool"," is"," disabled"," by"," policy","."," I"," need"," to"," report"," this"," error"," verb","atim"," back"," to"," the"," user","."]}} +{"type":"assistant/chunk","seq":82,"time":1783352167613,"data":{"turn":1,"step":2,"chunk":{"type":"block-start","index":1,"blockType":"text"}}} +{"type":"text-chunks","seq0":83,"time0":1783352167613,"data":{"turn":1,"step":2,"index":1,"dt":[30,29,29,0,1,28,0,1,0,0,0,26,1,0,0,0,28,0,31,0,25,30,1,0,27,1,0,31,1],"texts":["The"," tool"," returned",":\n\n",">"," Error",":"," bash"," is"," disabled"," by"," policy"," in"," this"," session","\n\n","I"," cannot"," run"," the"," command"," because"," the"," bash"," tool"," is"," disabled"," by"," policy","."]}} +{"type":"assistant/chunk","seq":113,"time":1783352167933,"data":{"turn":1,"step":2,"chunk":{"type":"block-end","index":0,"block":{"type":"reasoning","text":"The bash tool is disabled by policy. I need to report this error verbatim back to the user."}}}} +{"type":"assistant/chunk","seq":114,"time":1783352167933,"data":{"turn":1,"step":2,"chunk":{"type":"block-end","index":1,"block":{"type":"text","text":"The tool returned:\n\n> Error: bash is disabled by policy in this session\n\nI cannot run the command because the bash tool is disabled by policy."}}}} +{"type":"assistant/chunk","seq":115,"time":1783352167933,"data":{"turn":1,"step":2,"chunk":{"type":"usage","usage":{"inputTokens":167,"outputTokens":52,"cacheReadTokens":2816,"reasoningTokens":21}}}} +{"type":"assistant/chunk","seq":116,"time":1783352167933,"data":{"turn":1,"step":2,"chunk":{"type":"finish","reason":{"kind":"stop"}}}} +{"type":"assistant/message","seq":117,"time":1783352167934,"data":{"turn":1,"step":2,"content":[{"type":"reasoning","text":"The bash tool is disabled by policy. I need to report this error verbatim back to the user."},{"type":"text","text":"The tool returned:\n\n> Error: bash is disabled by policy in this session\n\nI cannot run the command because the bash tool is disabled by policy."}],"provenance":{"provider":"deepseek","model":"deepseek-v4-flash"},"usage":{"inputTokens":167,"outputTokens":52,"cacheReadTokens":2816,"reasoningTokens":21}},"sourceEventSeqs":[60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116],"surfaceOp":"append"} +{"type":"step/end","seq":118,"time":1783352167934,"data":{"turn":1,"step":2}} +{"type":"turn/end","seq":119,"time":1783352167934,"data":{"turn":1,"reason":{"kind":"completed"}}} diff --git a/examples/acp-agent/tests/snapshots/packed-chunks/stdout.expected.jsonl b/examples/acp-agent/tests/snapshots/packed-chunks/stdout.expected.jsonl index bba9f955f3..b25efc21cd 100644 --- a/examples/acp-agent/tests/snapshots/packed-chunks/stdout.expected.jsonl +++ b/examples/acp-agent/tests/snapshots/packed-chunks/stdout.expected.jsonl @@ -1,27 +1,75 @@ {"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentInfo":{"name":"deepseek-harness-acp","version":"0.0.1"},"agentCapabilities":{"loadSession":true,"promptCapabilities":{"image":false,"audio":false,"embeddedContext":false}},"authMethods":[]}} {"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}","configOptions":[{"id":"model","name":"Model","description":"Sets this session's provider and model.","category":"model","type":"select","currentValue":"[\"deepseek\",\"deepseek-v4-flash\"]","options":[{"value":"[\"deepseek\",\"deepseek-v4-flash\"]","name":"deepseek-v4-flash"},{"value":"[\"deepseek\",\"deepseek-v4-pro\"]","name":"deepseek-v4-pro"}]},{"id":"permission","name":"Permissions","description":"The session permission preset: each choice bundles a sandbox mode and an approval policy.","category":"mode","type":"select","currentValue":"danger-full-access","options":[{"value":"workspace-write","name":"workspace-write","description":"Write inside the workspace and permitted temporary directories; wider retries require approval."},{"value":"danger-full-access","name":"danger-full-access","description":"Full file access without approval prompts."}]}]}} {"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"available_commands_update","availableCommands":[{"name":"goal","description":"set or view the goal for a long-running task","input":{"hint":"[|clear|edit |pause|resume]"}}]}}} -{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"session_info_update","title":"Reply with exactly the word:","updatedAt":"{{updatedAt}}"}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"session_info_update","title":"Use the bash tool to","updatedAt":"{{updatedAt}}"}}} {"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":"The"}}}} {"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" user"}}}} {"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" wants"}}}} {"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" me"}}}} {"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" to"}}}} -{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" reply"}}}} -{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" with"}}}} -{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" exactly"}}}} -{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" the"}}}} -{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" word"}}}} -{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" \""}}}} -{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":"P"}}}} -{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":"ONG"}}}} -{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":"\""}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" run"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" a"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" simple"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" bash"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" command"}}}} {"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" and"}}}} -{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" not"}}}} -{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" use"}}}} -{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" any"}}}} -{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" tools"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" report"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" the"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" result"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" verb"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":"atim"}}}} {"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":"."}}}} -{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"P"}}}} -{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"ONG"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call","toolCallId":"call_00_JliP571Bh0QQ8QExbSPk0080","title":"echo HELLO","kind":"execute","status":"in_progress","rawInput":"echo HELLO","content":[{"type":"content","content":{"type":"text","text":"Run echo HELLO"}}]}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call_update","toolCallId":"call_00_JliP571Bh0QQ8QExbSPk0080","status":"failed","content":[{"type":"content","content":{"type":"text","text":"```console\nError: bash is disabled by policy in this session\n```"}}]}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":"The"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" bash"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" tool"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" is"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" disabled"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" by"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" policy"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":"."}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" I"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" need"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" to"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" report"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" this"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" error"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" verb"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":"atim"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" back"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" to"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" the"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":" user"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","content":{"type":"text","text":"."}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"The"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":" tool"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":" returned"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":":\n\n"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":">"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":" Error"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":":"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":" bash"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":" is"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":" disabled"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":" by"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":" policy"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":" in"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":" this"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":" session"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"\n\n"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"I"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":" cannot"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":" run"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":" the"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":" command"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":" because"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":" the"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":" bash"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":" tool"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":" is"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":" disabled"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":" by"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":" policy"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"."}}}} {"jsonrpc":"2.0","id":3,"result":{"stopReason":"end_turn"}} diff --git a/examples/acp-agent/tests/snapshots/packed-chunks/workspace/hooks.json b/examples/acp-agent/tests/snapshots/packed-chunks/workspace/hooks.json new file mode 100644 index 0000000000..f509c811c3 --- /dev/null +++ b/examples/acp-agent/tests/snapshots/packed-chunks/workspace/hooks.json @@ -0,0 +1,12 @@ +{ + "hooks": { + "PreToolUse": [ + { + "matcher": "bash", + "hooks": [ + { "type": "command", "command": "echo 'bash is disabled by policy in this session' >&2; exit 2" } + ] + } + ] + } +}