From c5a1c494e78278710a63257f0af7b64f7b6d9ce2 Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Wed, 17 Jun 2026 19:25:29 +0800 Subject: [PATCH 01/17] feature(session): session surface --- docs/adr/0019-session-surface.md | 63 ++++ docs/adr/README.md | 1 + packages/agent-loop/src/agent.ts | 4 +- packages/agent-loop/src/loop.ts | 14 +- packages/invariants/src/index.ts | 30 +- packages/invariants/tests/invariants.spec.ts | 112 ++++++ packages/session-persistence-sqlite/README.md | 2 +- .../session-persistence-sqlite/src/index.ts | 28 +- .../session-persistence-sqlite/src/schema.ts | 38 +- .../tests/sqlite.spec.ts | 130 ++++++- packages/session/README.md | 31 +- packages/session/src/index.ts | 128 +++++-- packages/session/src/repair.ts | 23 +- packages/session/src/surface.ts | 129 +++++++ packages/session/src/types.ts | 36 ++ packages/session/tests/repair.spec.ts | 32 ++ packages/session/tests/surface.spec.ts | 324 ++++++++++++++++++ 17 files changed, 1047 insertions(+), 78 deletions(-) create mode 100644 docs/adr/0019-session-surface.md create mode 100644 packages/session/src/surface.ts create mode 100644 packages/session/tests/surface.spec.ts diff --git a/docs/adr/0019-session-surface.md b/docs/adr/0019-session-surface.md new file mode 100644 index 0000000000..159db911b1 --- /dev/null +++ b/docs/adr/0019-session-surface.md @@ -0,0 +1,63 @@ +# ADR 0019: Session surface — a linked list over the event log for LLM message derivation + +Status: accepted (2026-06-17) + +## Context + +The `Session` event log is the single source of truth ([ADR 0003](0003-event-sourced-sessions.md)), but the only view over it was `deriveMessages()` — a linear scan that filtered and transformed raw events into `Message[]`. This creates problems for session-history-manipulating plugins (compaction, tool-call result pruning, etc.). Without a central mechanism, each plugin would need to wrap `agent/request` to rewrite the message list — a pattern that suffers from listener-ordering fragility, provides no durable record of what was changed, and forces repeated changes to the core `deriveMessages()` whenever a new manipulation is added. A central hub in the `session` package, with a provenance-recording mechanism and enough flexibility for future plugins to manipulate session history through a stable API, lays a solid foundation for plugin development. + +## Decision + +Add a **surface** — a derived, cached linked list of "surface nodes" (the subset of events that produce LLM messages) — maintained by `surfaceOp` markers in the event log. + +### Two new top-level fields on `SessionEvent` + +Every `SessionEvent` gains two optional fields (structural metadata, like `seq`/`time`): + +- **`sourceEventSeqs?: number[]`** — seq numbers of events that are provenance sources (e.g., the `assistant/chunk` seqs that built an `assistant/message`, or the surface nodes shadowed by a compaction marker). Provenance is a core design principle; without it, the replace-range operation cannot be validated on replay. +- **`surfaceOp?: SurfaceOp`** — how this event entered the surface. Absent for non-surface events. + +### SurfaceOp: two operations + +```ts +export type SurfaceOp = + | 'append' // normal tail append + | { op: 'replace'; start: number; end: number } // shadow [start, end] inclusive +``` + +1. **Append** — add a new node to the tail. Used by `user/message`, `assistant/message`, `tool/result`, `context/message`, `steering/message`. The loop passes `surfaceOp: 'append'` on all such appends, and `sourceEventSeqs` where applicable (e.g., `assistant/message` records its `assistant/chunk` sources; `tool/result` records its `tool/call` source). + +2. **Replace** — remove nodes from `start` through `end` (both inclusive) and insert a new node in their place. Both `start` and `end` must be valid surface node seqs in the current surface; `start === end` replaces a single node. The node's `sourceEventSeqs` must contain every shadowed surface node. The shadowed events remain in the log but are no longer on the surface. + +The both-ends-inclusive design was chosen over half-open `[start, endExclusive)` because the surface is a doubly-linked list — both ends are naturally named by node seqs, and single-node replacement (`start === end`) is a common case that reads naturally with inclusive semantics. + +### SurfaceManager: delta-based, not full rebuild + +A `SurfaceManager` class (private to `Session`) maintains the cached linked list. It tracks `_lastProcessedSeq` and processes only the **delta** (new events since the last access) rather than rescanning the entire log. Because the log is append-only, prior events never change — full rebuild is only needed after a wholesale log replacement (e.g., seeding). + +Why delta processing? The naive approach (a dirty flag + full rebuild on every access) would be O(N²) over a session's lifetime — every single-event append triggers a complete scan of all prior events. Delta processing is O(1) when no new events and O(new events) when new events arrive. + +`deriveMessages()` uses the surface when surface markers exist, falling back to the existing linear scan for sessions without markers (backward compatibility). + +### Persistence + +The new fields are serialized as top-level JSON properties. The JSONL backend requires zero changes — `JSON.stringify`/`JSON.parse` preserve everything transparently. The SQLite backend adds two nullable TEXT columns (`source_event_seqs`, `surface_op`) with an `ALTER TABLE` migration (SCHEMA_VERSION 1 → 2). The session format `version` stays at 1 — the new fields are optional and backward-compatible. + +### Crash recovery + +The `repair.ts` module synthesizes `tool/result` closers for orphaned tool calls after a crash. These closers carry `surfaceOp: 'append'` and `sourceEventSeqs` pointing to the orphaned `tool/call` event, so the rehydrated surface is valid. + +### Invariants + +The dev-mode invariants plugin validates: `sourceEventSeqs` references (non-empty, no duplicates, references earlier events, references known seqs) and `surfaceOp` (replace start ≤ end). + +## Consequences + +- **`packages/session`**: New `surface.ts` (`SurfaceManager`), new types (`SurfaceOp`, `SurfaceAppendOpts`), new fields on `SessionEvent`, modified `append()` (third optional `SurfaceAppendOpts` param), refactored `deriveMessages()` (surface path + legacy fallback), surface-aware `repair.ts`. +- **`packages/agent-loop`**: All surface-capable appends pass surface opts. Chunk seqs are collected for `assistant/message` provenance; `tool/call` seqs are captured for `tool/result` provenance. +- **`packages/session-persistence-sqlite`**: Schema migration v1 → v2 (two new nullable TEXT columns). +- **`packages/invariants`**: Surface-related validation rules. +- **`packages/session-persistence-jsonl`**: No changes required. +- **`packages/session-persistence`**: Abstract interface unchanged. + +The surface is the foundation for future compaction: a compaction plugin appends a new event (e.g., `compaction/marker`, added to `SessionEventMap` via declaration merging) with `surfaceOp: { op: 'replace', start, end }` and `sourceEventSeqs` covering the shadowed nodes. Replay preserves the compaction decision deterministically. diff --git a/docs/adr/README.md b/docs/adr/README.md index d9671c6652..8f3180979e 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -30,3 +30,4 @@ Do NOT write an ADR for: a mechanical or local choice (a variable name, a one-fi | [0016](0016-pnpm-over-yarn.md) | pnpm as the package manager instead of Yarn 4 | accepted | | [0017](0017-turn-enclosure-invariant.md) | Every session event is enclosed in a turn | accepted | | [0018](0018-session-persistence.md) | Session persistence as an abstract service over `SessionEvent` | accepted | +| [0019](0019-session-surface.md) | Session surface — a linked list over the event log for LLM message derivation | accepted | diff --git a/packages/agent-loop/src/agent.ts b/packages/agent-loop/src/agent.ts index 64576186c3..0735784616 100644 --- a/packages/agent-loop/src/agent.ts +++ b/packages/agent-loop/src/agent.ts @@ -78,7 +78,7 @@ export class LoopAgent implements Agent { // A turn is open in the LOG (decided from the log, not agent status — // status can be `running` with no turn open): the context/message is // turn-enclosed by that turn, so append it directly. - this.session.append('context/message', { content, source }) + this.session.append('context/message', { content, source }, { surfaceOp: 'append' }) return } // No turn open: wrap the injection in a one-shot turn so every event stays @@ -95,7 +95,7 @@ export class LoopAgent implements Agent { // can't happen for our fixed trigger — no turn was opened and none is owed.) try { this.session.append('turn/start', { turn, trigger: { kind: 'injection', source } }) - this.session.append('context/message', { content, source }) + this.session.append('context/message', { content, source }, { surfaceOp: 'append' }) } finally { // Close the turn if turn/start made it into the log. Contain a throwing // turn/end listener: Session.append pushes before notifying, so a throw diff --git a/packages/agent-loop/src/loop.ts b/packages/agent-loop/src/loop.ts index d2b1ed270b..f1bb879626 100644 --- a/packages/agent-loop/src/loop.ts +++ b/packages/agent-loop/src/loop.ts @@ -276,7 +276,7 @@ async function runTurn(ctx: Context, agent: LoopAgent, handle: LoopHandle, turn: // every event in the log is turn-enclosed. turn/end is now owed, so a throw // while appending these is caught below and the turn is still closed. for (const message of queued) { - session.append('user/message', { content: message.content, source: message.source }) + session.append('user/message', { content: message.content, source: message.source }, { surfaceOp: 'append' }) } ctx.emit('agent/turn-start', agent, turn) @@ -410,7 +410,7 @@ async function runTurn(ctx: Context, agent: LoopAgent, handle: LoopHandle, turn: function drainSteering(ctx: Context, agent: LoopAgent, turn: number): boolean { const messages = agent.inbox.drainSteering() for (const message of messages) { - agent.session.append('steering/message', { turn, content: message.content, source: message.source }) + agent.session.append('steering/message', { turn, content: message.content, source: message.source }, { surfaceOp: 'append' }) ctx.emit('agent/steering', agent, turn, message.content, message.source) } return messages.length > 0 @@ -446,10 +446,12 @@ async function runStep( // --- Model call (streaming-first; raw chunks are the replay record) --- const assembler = new BlockAssembler() + const chunkSeqs: number[] = [] for await (const chunk of ctx.llm.stream(request)) { /* v8 ignore next -- signal.reason always set by agent.abort() which provides a default */ if (signal.aborted) throw new Error(String(signal.reason ?? 'aborted')) - session.append('assistant/chunk', { turn, step, chunk }) + const chunkEvent = session.append('assistant/chunk', { turn, step, chunk }) + chunkSeqs.push(chunkEvent.seq) ctx.emit('agent/stream-chunk', agent, turn, step, chunk) assembler.push(chunk) } @@ -468,7 +470,7 @@ async function runStep( let message: Message = assembler.message() message = await ctx.waterfall('agent/step-result', agent, turn, step, message, () => Promise.resolve(message)) - session.append('assistant/message', { turn, step, content: message.content }) + session.append('assistant/message', { turn, step, content: message.content }, { surfaceOp: 'append', sourceEventSeqs: chunkSeqs }) if (assembler.usage) { session.append('usage', { turn, step, usage: assembler.usage }) } @@ -480,7 +482,7 @@ async function runStep( for (const call of toolCalls) { /* v8 ignore next -- signal.reason always set by agent.abort() which provides a default */ if (signal.aborted) throw new Error(String(signal.reason ?? 'aborted')) - session.append('tool/call', { turn, step, callId: call.id, name: call.name, arguments: call.arguments }) + const callEvent = session.append('tool/call', { turn, step, callId: call.id, name: call.name, arguments: call.arguments }) let parsedArguments: unknown try { parsedArguments = call.arguments ? JSON.parse(call.arguments) : {} @@ -506,7 +508,7 @@ async function runStep( content: result.content, isError: result.isError, ...result.error ? { error: result.error } : {}, - }) + }, { surfaceOp: 'append', sourceEventSeqs: [callEvent.seq] }) // signal CAN flip during the await above (abort() inside a tool); // the analyzer can't see through the await boundary. // signal can flip during the await above (abort() inside a tool); diff --git a/packages/invariants/src/index.ts b/packages/invariants/src/index.ts index ebb4decfd8..ac536eeceb 100644 --- a/packages/invariants/src/index.ts +++ b/packages/invariants/src/index.ts @@ -62,6 +62,8 @@ interface SessionTrace { * `step/end` — a result must arrive in the same step as its call. */ pendingCalls: Set + /** Every seq seen so far — validates `sourceEventSeqs` references. */ + knownSeqs: Set } /** @@ -105,6 +107,30 @@ function checkEvent(trace: SessionTrace, event: SessionEvent): void { } trace.lastSeq = event.seq + // --- Surface invariants --- + if (event.sourceEventSeqs !== undefined) { + if (event.sourceEventSeqs.length === 0) { + throw new InvariantError('sourceEventSeqs must not be empty when present') + } + const unique = new Set(event.sourceEventSeqs) + if (unique.size !== event.sourceEventSeqs.length) { + throw new InvariantError('sourceEventSeqs must not contain duplicates') + } + for (const ref of event.sourceEventSeqs) { + if (ref >= event.seq) { + throw new InvariantError(`sourceEventSeqs must reference earlier events: ${ref} >= current seq ${event.seq}`) + } + if (!trace.knownSeqs.has(ref)) { + throw new InvariantError(`sourceEventSeqs references unknown seq ${ref}`) + } + } + } + if (event.surfaceOp !== undefined && typeof event.surfaceOp !== 'string') { + if (event.surfaceOp.start > event.surfaceOp.end) { + throw new InvariantError(`surface replace: start ${event.surfaceOp.start} must be <= end ${event.surfaceOp.end}`) + } + } + // Boundary/step-scoped events have explicit cases; every OTHER event type — // including plugin-added (merge-extensible) SessionEventMap keys — is caught // by the `default` and must be turn-enclosed (ADR 0017). No assertNever: an @@ -185,6 +211,8 @@ function checkEvent(trace: SessionTrace, event: SessionEvent): void { break } } + // Track every seq seen — used above to validate sourceEventSeqs references. + trace.knownSeqs.add(event.seq) } /** Legal agent status transitions (the only state machine the loop guarantees). */ @@ -216,7 +244,7 @@ export function apply(ctx: Context, config: Config = {}): void { // (re-)apply seeds the baseline, so a reload never produces a false positive. const lastStatus = new WeakMap() - const freshTrace = (): SessionTrace => ({ lastSeq: -1, openTurn: null, openStep: null, pendingCalls: new Set() }) + const freshTrace = (): SessionTrace => ({ lastSeq: -1, openTurn: null, openStep: null, pendingCalls: new Set(), knownSeqs: new Set() }) /** Build (or rebuild) a session's trace by replaying its whole log; freeze it. */ const seedSession = (session: Session): SessionTrace => { diff --git a/packages/invariants/tests/invariants.spec.ts b/packages/invariants/tests/invariants.spec.ts index b96d172c22..69d63aed4e 100644 --- a/packages/invariants/tests/invariants.spec.ts +++ b/packages/invariants/tests/invariants.spec.ts @@ -387,3 +387,115 @@ describe('HMR safety', () => { expect(Object.isFrozen(session.events[0])).toBe(false) }) }) + +describe('surface invariants', () => { + it('accepts well-formed surface metadata', async () => { + const { ctx } = await setup() + const session = ctx.sessions.create() + // Events must be turn-enclosed and step-scoped events need an open step. + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + session.append('step/start', { turn: 1, step: 1 }) + expect(() => { + session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) + session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [1] }) + }).not.toThrow() + }) + + it('accepts replace surface op', async () => { + const { ctx } = await setup() + const session = ctx.sessions.create() + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + session.append('step/start', { turn: 1, step: 1 }) + session.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) + session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 2, end: 2 }, sourceEventSeqs: [2] }) + // no throw — well-formed replace op + }) + + it('rejects empty sourceEventSeqs', async () => { + const { ctx } = await setup() + const session = ctx.sessions.create() + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + expect(() => { + session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [] }) + }).toThrow(InvariantError) + }) + + it('rejects duplicate sourceEventSeqs', async () => { + const { ctx } = await setup() + const session = ctx.sessions.create() + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + session.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) + expect(() => { + session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [1, 1] }) + }).toThrow(/must not contain duplicates/) + }) + + it('rejects sourceEventSeqs referencing the event itself (self-reference)', async () => { + const { ctx } = await setup() + const session = ctx.sessions.create() + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) // seq 0 + // The next event is seq 1. Referencing its own seq fails on "must reference + // earlier events" (the check order is: earlier first, then unknown). + expect(() => { + session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [1] }) + }).toThrow(/must reference earlier/) + }) + + it('accepts sourceEventSeqs referencing a valid earlier event', async () => { + // Positive test: ref < current seq and ref is in knownSeqs → passes. + const { ctx } = await setup() + const session = ctx.sessions.create() + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + session.append('step/start', { turn: 1, step: 1 }) + // seqs so far: 0, 1. The next event at seq 2 references seq 1 → valid. + expect(() => { + session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [1] }) + }).not.toThrow() + }) + + it('rejects sourceEventSeqs referencing a far-future seq', async () => { + const { ctx } = await setup() + const session = ctx.sessions.create() + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + expect(() => { + session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [99] }) + }).toThrow(/must reference earlier/) + }) + + it('rejects sourceEventSeqs referencing unknown seq (gap in event log)', async () => { + // The unknown-seq check fires when a ref passes the "earlier" test but is + // not in knownSeqs — only possible with a gap in seqs. We create a gap by + // directly manipulating the private log array to skip a seq. + const { ctx } = await setup() + const session = ctx.sessions.create() + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + session.append('step/start', { turn: 1, step: 1 }) + // Push a fake event at seq 3 into the internal log, creating a gap at seq 2. + // The invariants plugin replays session.events on every append, so it sees + // this gap during trace reconstruction. + ;(session as unknown as { log: unknown[] }).log.push({ + type: 'assistant/chunk', + seq: 3, + time: Date.now(), + data: { turn: 1, step: 1, chunk: { type: 'text-delta', index: 0, text: 'x' } }, + }) + // Now the log has seqs 0, 1, 3 (gap at 2). Append at what session believes + // is seq 3 (log.length). Reference seq 2: passes earlier (2 < 3) but not + // in knownSeqs ({0, 1, 3} — gap at 2). + expect(() => { + session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [2] }) + }).toThrow(/unknown seq 2/) + }) + + it('rejects replace op with start > end', async () => { + const { ctx } = await setup() + const session = ctx.sessions.create() + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + session.append('step/start', { turn: 1, step: 1 }) + session.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 2 + // start > end is invalid (reversed order). + expect(() => { + session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 2, end: 1 }, sourceEventSeqs: [2] }) + }).toThrow(/must be <= end/) + }) +}) diff --git a/packages/session-persistence-sqlite/README.md b/packages/session-persistence-sqlite/README.md index 3a7a3c8163..dace03c529 100644 --- a/packages/session-persistence-sqlite/README.md +++ b/packages/session-persistence-sqlite/README.md @@ -6,7 +6,7 @@ A SQLite durable session-persistence backend — a second `SessionPersistence` i ## Storage model -Each `SessionEvent` maps 1:1 onto a row in an `events` table `(session_id, seq, type, time, data)` — `data` is the event payload as JSON text, so the row shape is the event verbatim (including `assistant/chunk`, keeping `seq` contiguous). Out-of-log metadata (`SessionMeta`) lives in a `sessions` row, including the mutable `SessionSummary` fields (`updatedAt`, `title`, `firstPrompt`) that `update()` rewrites without touching the event log. A `sessions` row is written only by the first `append` — its existence is the lazy-materialization signal (`has`/`list` report exactly the sessions that have a row), so no separate column is needed. +Each `SessionEvent` maps 1:1 onto a row in an `events` table `(session_id, seq, type, time, data, source_event_seqs, surface_op)` — `data` is the event payload as JSON text, so the row shape is the event verbatim (including `assistant/chunk`, keeping `seq` contiguous). The two `TEXT` columns `source_event_seqs` and `surface_op` are nullable; they store the event's optional surface-metadata fields (see [ADR 0019](../../docs/adr/0019-session-surface.md)). The schema migrates from v1 to v2 via `ALTER TABLE ADD COLUMN` — existing rows get NULL for both columns, which is correct for events written before surface support. Out-of-log metadata (`SessionMeta`) lives in a `sessions` row, including the mutable `SessionSummary` fields (`updatedAt`, `title`, `firstPrompt`) that `update()` rewrites without touching the event log. A `sessions` row is written only by the first `append` — its existence is the lazy-materialization signal (`has`/`list` report exactly the sessions that have a row), so no separate column is needed. The repo targets Node ≥ 24 (the root `engines` field), which includes the stable `node:sqlite` module. The database opens with `foreign_keys = ON` (so `ON DELETE CASCADE` drops a session's events with its row) and `journal_mode = WAL`. The table-layout version is stored in `PRAGMA user_version` and checked on open: a fresh database is stamped with the current `SCHEMA_VERSION`; a database written by a newer, incompatible build (higher `user_version`) is rejected rather than opened against an unknown layout. diff --git a/packages/session-persistence-sqlite/src/index.ts b/packages/session-persistence-sqlite/src/index.ts index bd4b4d9435..8a6cd8cb09 100644 --- a/packages/session-persistence-sqlite/src/index.ts +++ b/packages/session-persistence-sqlite/src/index.ts @@ -33,6 +33,18 @@ import { export { SCHEMA_VERSION } from './schema.ts' +/** + * Serialize an event's surface-metadata fields for SQL binding. Both fields are + * nullable TEXT columns — null when the event has no surface metadata (non-surface + * events, events written before surface support). + */ +function surfaceBindings(event: SessionEvent): [string | null, string | null] { + return [ + event.sourceEventSeqs ? JSON.stringify(event.sourceEventSeqs) : null, + event.surfaceOp !== undefined ? JSON.stringify(event.surfaceOp) : null, + ] +} + /** Plugin configuration. */ export interface Config { /** @@ -180,13 +192,14 @@ export class SessionPersistenceSqlite extends SessionPersistence { // durably closes the interrupted turn before returning, so by the time any // append runs the stored log is balanced and contiguous.) const insertEvent = this.db.prepare( - 'INSERT INTO events (session_id, seq, type, time, data) VALUES (?, ?, ?, ?, ?)', + 'INSERT INTO events (session_id, seq, type, time, data, source_event_seqs, surface_op) VALUES (?, ?, ?, ?, ?, ?, ?)', ) this.db.exec('BEGIN') try { if (!state.materialized) this.writeRow(state.meta) for (const event of events) { - insertEvent.run(id, event.seq, event.type, event.time, JSON.stringify(event.data)) + const [surfaceSeqs, surfaceOp] = surfaceBindings(event) + insertEvent.run(id, event.seq, event.type, event.time, JSON.stringify(event.data), surfaceSeqs, surfaceOp) } // Bump updatedAt on every append (the mutable summary lives in the row). const updatedAt = Date.now() @@ -220,7 +233,7 @@ export class SessionPersistenceSqlite extends SessionPersistence { // discarded (not unloadable); only a parse error / seq gap in the COMMITTED // region (at or before the last turn/end) throws (genuine corruption). const eventRows = this.db - .prepare('SELECT seq, type, time, data FROM events WHERE session_id = ? ORDER BY seq') + .prepare('SELECT seq, type, time, data, source_event_seqs, surface_op FROM events WHERE session_id = ? ORDER BY seq') .all(id) as unknown as EventRow[] const { preserved, tornFrom } = scanRows(eventRows) @@ -248,9 +261,12 @@ export class SessionPersistenceSqlite extends SessionPersistence { this.db.prepare('DELETE FROM events WHERE session_id = ? AND seq >= ?').run(id, tornFrom) } if (closers.length > 0) { - const insertEvent = this.db.prepare('INSERT INTO events (session_id, seq, type, time, data) VALUES (?, ?, ?, ?, ?)') + const insertEvent = this.db.prepare( + 'INSERT INTO events (session_id, seq, type, time, data, source_event_seqs, surface_op) VALUES (?, ?, ?, ?, ?, ?, ?)', + ) for (const event of closers) { - insertEvent.run(id, event.seq, event.type, event.time, JSON.stringify(event.data)) + const [surfaceSeqs, surfaceOp] = surfaceBindings(event) + insertEvent.run(id, event.seq, event.type, event.time, JSON.stringify(event.data), surfaceSeqs, surfaceOp) } } this.db.exec('COMMIT') @@ -497,7 +513,7 @@ export class SessionPersistenceSqlite extends SessionPersistence { /** The preserved events for a session id (torn tail excluded, turn NOT yet closed). */ private eventsFor(id: SessionId): SessionEvent[] { const rows = this.db - .prepare('SELECT seq, type, time, data FROM events WHERE session_id = ? ORDER BY seq') + .prepare('SELECT seq, type, time, data, source_event_seqs, surface_op FROM events WHERE session_id = ? ORDER BY seq') .all(id) as unknown as EventRow[] // Scan on seq+type columns, parsing `data` only for the preserved prefix (a // malformed torn tail must not throw here — same as loadCore). Returns the diff --git a/packages/session-persistence-sqlite/src/schema.ts b/packages/session-persistence-sqlite/src/schema.ts index 1dad51698b..84b357937a 100644 --- a/packages/session-persistence-sqlite/src/schema.ts +++ b/packages/session-persistence-sqlite/src/schema.ts @@ -8,14 +8,14 @@ */ import { DatabaseSync } from 'node:sqlite' -import type { SessionEvent, SessionId, SessionMeta } from '@deepseek-ai/dsh-session' +import type { SessionEvent, SessionId, SessionMeta, SurfaceOp } from '@deepseek-ai/dsh-session' /** * The on-disk schema version. Bumped only on a breaking change to the table * layout; orthogonal to a session's own `version` (which versions the EVENT * vocabulary, stored per session in the `sessions` row). */ -export const SCHEMA_VERSION = 1 +export const SCHEMA_VERSION = 2 /** * A row of the `sessions` table — the out-of-log metadata (`SessionMeta`). The @@ -41,6 +41,10 @@ export interface EventRow { type: string time: number data: string + /** JSON-encoded `number[]` — the event's sourceEventSeqs, or null. */ + source_event_seqs: string | null + /** JSON-encoded `SurfaceOp` — how the event entered the surface, or null. */ + surface_op: string | null } /** @@ -72,6 +76,13 @@ export function openDatabase(path: string): DatabaseSync { // constant (SCHEMA_VERSION is a trusted in-code number, not user input). db.exec(`PRAGMA user_version = ${SCHEMA_VERSION}`) } + if (onDisk === 1) { + // Migrate from v1 to v2: add surface-metadata columns (nullable — existing + // rows get NULL, which is correct for events written before surface existed). + db.exec('ALTER TABLE events ADD COLUMN source_event_seqs TEXT') + db.exec('ALTER TABLE events ADD COLUMN surface_op TEXT') + db.exec(`PRAGMA user_version = ${SCHEMA_VERSION}`) + } db.exec(` CREATE TABLE IF NOT EXISTS sessions ( id TEXT PRIMARY KEY, @@ -86,11 +97,13 @@ export function openDatabase(path: string): DatabaseSync { `) db.exec(` CREATE TABLE IF NOT EXISTS events ( - session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, - seq INTEGER NOT NULL, - type TEXT NOT NULL, - time INTEGER NOT NULL, - data TEXT NOT NULL, + session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, + seq INTEGER NOT NULL, + type TEXT NOT NULL, + time INTEGER NOT NULL, + data TEXT NOT NULL, + source_event_seqs TEXT, + surface_op TEXT, PRIMARY KEY (session_id, seq) ) STRICT `) @@ -113,12 +126,19 @@ export function rowToMeta(row: SessionRow): SessionMeta { /** Reconstruct a {@link SessionEvent} from an `events` row (parses `data`). */ export function rowToEvent(row: EventRow): SessionEvent { - return { - type: row.type, + const event = { + type: row.type as SessionEvent['type'], seq: row.seq, time: row.time, data: JSON.parse(row.data) as SessionEvent['data'], } as SessionEvent + if (row.source_event_seqs !== null) { + event.sourceEventSeqs = JSON.parse(row.source_event_seqs) as number[] + } + if (row.surface_op !== null) { + event.surfaceOp = JSON.parse(row.surface_op) as SurfaceOp + } + return event } /** diff --git a/packages/session-persistence-sqlite/tests/sqlite.spec.ts b/packages/session-persistence-sqlite/tests/sqlite.spec.ts index 44d788ab72..61b166f257 100644 --- a/packages/session-persistence-sqlite/tests/sqlite.spec.ts +++ b/packages/session-persistence-sqlite/tests/sqlite.spec.ts @@ -1,12 +1,13 @@ import { afterEach, describe, expect, it } from 'vitest' import { Context } from 'cordis' +import { DatabaseSync } from 'node:sqlite' import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import SessionStore, { SessionId } from '@deepseek-ai/dsh-session' import type { Session, SessionEvent, SessionMeta } from '@deepseek-ai/dsh-session' import SessionPersistenceSqlite, { SCHEMA_VERSION } from '@deepseek-ai/dsh-session-persistence-sqlite' -import { openDatabase, scanRows, type EventRow } from '../src/schema.ts' +import { openDatabase, rowToEvent, scanRows, type EventRow } from '../src/schema.ts' import { runPersistenceContract, meta, oneTurnLog } from '../../session-persistence/tests/contract.ts' const dirs: string[] = [] @@ -42,7 +43,7 @@ describe('scanRows', () => { // scanRows works off EventRows (data is a JSON string column); build them from // SessionEvents so the unit tests read in terms of the event vocabulary. const rows = (events: SessionEvent[]): EventRow[] => - events.map(e => ({ seq: e.seq, type: e.type, time: e.time, data: JSON.stringify(e.data) })) + events.map(e => ({ seq: e.seq, type: e.type, time: e.time, data: JSON.stringify(e.data), source_event_seqs: null, surface_op: null })) it('preserves the full log when it ends exactly on a turn/end (no torn tail)', () => { const { preserved, tornFrom } = scanRows(rows(oneTurnLog())) @@ -91,8 +92,8 @@ describe('scanRows', () => { it('throws on an unparsable row inside the committed region', () => { const withCorruptCommitted: EventRow[] = [ - { seq: 0, type: 'turn/start', time: 1, data: '{not json' }, // corrupt, sits before a turn/end - { seq: 1, type: 'turn/end', time: 2, data: JSON.stringify({ turn: 1, reason: { kind: 'completed' } }) }, + { seq: 0, type: 'turn/start', time: 1, data: '{not json', source_event_seqs: null, surface_op: null }, // corrupt, sits before a turn/end + { seq: 1, type: 'turn/end', time: 2, data: JSON.stringify({ turn: 1, reason: { kind: 'completed' } }), source_event_seqs: null, surface_op: null }, ] expect(() => scanRows(withCorruptCommitted)).toThrow(/unparsable committed event/) }) @@ -100,7 +101,7 @@ describe('scanRows', () => { it('tolerates an unparsable torn-tail row after the last turn/end', () => { const withCorruptTail: EventRow[] = [ ...rows(oneTurnLog()), - { seq: 6, type: 'turn/start', time: 7, data: '{not json' }, // torn fragment, no committed turn/end after + { seq: 6, type: 'turn/start', time: 7, data: '{not json', source_event_seqs: null, surface_op: null }, // torn fragment, no committed turn/end after ] const { preserved, tornFrom } = scanRows(withCorruptTail) expect(preserved).toEqual(oneTurnLog()) @@ -343,7 +344,7 @@ describe('SessionPersistenceSqlite: durability and crash semantics', () => { }) it('exposes the schema version constant', () => { - expect(SCHEMA_VERSION).toBe(1) + expect(SCHEMA_VERSION).toBe(2) }) }) @@ -749,4 +750,121 @@ describe('SessionPersistenceSqlite: edge cases', () => { await expect(ctx.parallel('session/flush', session)).rejects.toThrow(/id collision/) await ctx.fiber.dispose() }) + + it('migrates a v1 database to v2 (adds surface columns)', async () => { + const path = await freshDbPath() + // Manually create a v1 database with the OLD schema (no surface columns). + const db = new DatabaseSync(path) + db.exec('PRAGMA user_version = 1') + db.exec(` + CREATE TABLE sessions ( + id TEXT PRIMARY KEY, + version INTEGER NOT NULL, + created_at INTEGER NOT NULL, + cwd TEXT, + parent_session TEXT, + updated_at INTEGER NOT NULL, + title TEXT, + first_prompt TEXT + ) STRICT + `) + db.exec(` + CREATE TABLE events ( + session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, + seq INTEGER NOT NULL, + type TEXT NOT NULL, + time INTEGER NOT NULL, + data TEXT NOT NULL, + PRIMARY KEY (session_id, seq) + ) STRICT + `) + db.close() + // Re-open with v2 code: migration adds the surface columns and stamps v2. + const db2 = openDatabase(path) + const version = (db2.prepare('PRAGMA user_version').get() as { user_version: number }).user_version + expect(version).toBe(2) + const info = db2.prepare("PRAGMA table_info('events')").all() as Array<{ name: string }> + const names = info.map(c => c.name) + expect(names).toContain('source_event_seqs') + expect(names).toContain('surface_op') + db2.close() + }) +}) + +describe('surface field round-trip', () => { + it('rowToEvent parses surface fields from EventRow columns', () => { + const row: EventRow = { + seq: 0, type: 'assistant/message', time: 1, + data: JSON.stringify({ turn: 1, step: 1, content: [] }), + source_event_seqs: JSON.stringify([3, 5]), + surface_op: JSON.stringify('append'), + } + const event = rowToEvent(row) + expect(event.sourceEventSeqs).toEqual([3, 5]) + expect(event.surfaceOp).toBe('append') + }) + + it('rowToEvent handles replace surfaceOp object', () => { + const row: EventRow = { + seq: 0, type: 'assistant/message', time: 1, + data: JSON.stringify({ turn: 1, step: 1, content: [] }), + source_event_seqs: JSON.stringify([0, 1]), + surface_op: JSON.stringify({ op: 'replace', start: 0, end: 1 }), + } + const event = rowToEvent(row) + expect(event.sourceEventSeqs).toEqual([0, 1]) + expect(event.surfaceOp).toEqual({ op: 'replace', start: 0, end: 1 }) + }) + + it('scanRows with surface columns reconstructs events with surface fields', () => { + const rows: EventRow[] = [ + { seq: 0, type: 'user/message', time: 1, + data: JSON.stringify({ content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }), + source_event_seqs: null, surface_op: '{"op":"replace","start":0,"end":0}' }, + { seq: 1, type: 'turn/end', time: 2, + data: JSON.stringify({ turn: 1, reason: { kind: 'completed' } }), + source_event_seqs: null, surface_op: null }, + ] + const { preserved } = scanRows(rows) + expect(preserved).toHaveLength(2) + expect(preserved[0]!.surfaceOp).toEqual({ op: 'replace', start: 0, end: 0 }) + expect(preserved[0]!.sourceEventSeqs).toBeUndefined() + expect(preserved[1]!.surfaceOp).toBeUndefined() + }) + + it('append and load round-trips surface fields through SQLite', async () => { + const ctx = new Context() + await ctx.plugin(SessionStore) + const fiber = await ctx.plugin(SessionPersistenceSqlite, { path: ':memory:' }) + const session = ctx.sessions.create('roundtrip-surface') + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) + session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [0] }) + session.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) + await ctx.parallel('session/flush', session) + const loaded = await ctx.sessionPersistence.load(SessionId('roundtrip-surface')) + expect(loaded.events).toHaveLength(4) + const um = loaded.events[1]! + expect(um.surfaceOp).toBe('append') + expect(um.sourceEventSeqs).toBeUndefined() + const am = loaded.events[2]! + expect(am.surfaceOp).toBe('append') + expect(am.sourceEventSeqs).toEqual([0]) + await fiber.dispose() + }) + + it('persists events with surfaceOp but no sourceEventSeqs (covers null branch in surfaceBindings)', async () => { + const ctx = new Context() + await ctx.plugin(SessionStore) + const fiber = await ctx.plugin(SessionPersistenceSqlite, { path: ':memory:' }) + const session = ctx.sessions.create('surface-noseq') + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + session.append('steering/message', { turn: 1, content: [], source: { kind: 'user' } }, { surfaceOp: 'append' }) + session.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) + await ctx.parallel('session/flush', session) + const loaded = await ctx.sessionPersistence.load(SessionId('surface-noseq')) + expect(loaded.events[1]!.surfaceOp).toBe('append') + expect(loaded.events[1]!.sourceEventSeqs).toBeUndefined() + await fiber.dispose() + }) }) diff --git a/packages/session/README.md b/packages/session/README.md index 7cf443fa71..b7787a768c 100644 --- a/packages/session/README.md +++ b/packages/session/README.md @@ -1,6 +1,6 @@ # dsh-session -Event-sourced session log and in-memory store. A `Session` is the append-only source of truth for an agent's whole interaction history — the LLM message history is *derived* from it. +Event-sourced session log and in-memory store. A `Session` is the append-only source of truth for an agent's whole interaction history — the LLM message history is *derived* from it. A **surface** layer (a linked list of message-producing events) is maintained on top of the raw log for efficient derivation and compaction. ## Service: `SessionStore` (ctx key: `sessions`) @@ -24,16 +24,17 @@ Creates and holds event-sourced `Session` instances. Persistence is intentionall Plain class (not a Cordis Service). Create via `ctx.sessions.create()`. -- `session.append(type, data): SessionEvent` — synchronous, never blocks on I/O. **Throws** if `data` is not losslessly JSON-serializable (BigInt, function, symbol, undefined, non-finite number, circular ref, or an exotic object like Map/Set/Date) — the event log is the durable source of truth, so this invariant is enforced at the source (exported as `isJsonValue` for backends to reuse on their replay/fork entry points). -- `session.deriveMessages(): Message[]` — derive the LLM message history from the event log. Raw `assistant/chunk` events are skipped; `context/message` and `steering/message` render as tagged synthetic user messages. +- `session.append(type, data, opts?): SessionEvent` — synchronous, never blocks on I/O. **Throws** if `data` is not losslessly JSON-serializable (BigInt, function, symbol, undefined, non-finite number, circular ref, or an exotic object like Map/Set/Date) — the event log is the durable source of truth, so this invariant is enforced at the source (exported as `isJsonValue` for backends to reuse on their replay/fork entry points). An optional third parameter `opts: SurfaceAppendOpts` carries surface metadata: `surfaceOp` controls how the event enters the surface linked list, and `sourceEventSeqs` records provenance (the seq numbers of events this one derives from). +- `session.deriveMessages(): Message[]` — derive the LLM message history. If any event in the log carries `surfaceOp`, derivation walks the surface linked list (skipping non-surface events). Otherwise, falls back to a linear scan of the raw log (legacy sessions without surface markers). +- `session.surface: SurfaceManager` — the derived surface, lazily rebuilt from `surfaceOp` markers in the log. Processes only new events (delta) on each access — the log is append-only, so prior events never change. - `session.events`, `session.seq`, `session.id` - `session.header: SessionHeader` — immutable creation metadata (`version`, `id`, `createdAt`, optional `cwd`/`parentSession`). Kept out of the event log (a storage concern, not replayable state); a minimal v1 header is synthesized for bare `Session` construction. -### Metadata types (`types.ts`) +### Surface types -- `SessionHeader` — immutable, written once: `{ version, id, createdAt, cwd?, parentSession? }`. -- `SessionSummary` — mutable, updateable without touching the log: `{ updatedAt, title?, firstPrompt? }`. -- `SessionMeta = SessionHeader & SessionSummary` — owned here (beside `SessionId`) because `Session.header` is typed by it; persistence backends re-export these rather than own them (which would force a package cycle). +- `SurfaceOp` — how a surface node entered the linked list: `'append'` (normal tail append) or `{ op: 'replace', start, end }` (replace nodes from `start` through `end` inclusive — both must be valid surface node seqs; `start === end` replaces a single node). Used by compaction to shadow old nodes without deleting them. +- `SurfaceAppendOpts` — `{ surfaceOp?: SurfaceOp; sourceEventSeqs?: number[] }`, the optional third parameter to `session.append()`. +- `SurfaceNode` — `{ seq: number; prev: number | null; next: number | null }`, one node in the surface linked list. ### Session event vocabulary (`types.ts`) @@ -43,11 +44,23 @@ Merge-extensible via `SessionEventMap` — a compaction plugin adds `compaction/ Also defines `TurnTriggerMap` and `TurnEndReasonMap` (merge-extensible sum types for typed turn boundaries — `kind`-tagged instead of strings). +Every `SessionEvent` carries two optional top-level fields (structural metadata): + +- `sourceEventSeqs?: number[]` — seq numbers of provenance sources (e.g., the `assistant/chunk` seqs behind an `assistant/message`, or the shadowed nodes behind a compaction marker). +- `surfaceOp?: SurfaceOp` — how this event entered the surface. Absent for non-surface events (boundaries, chunks, usage, errors). + +### Metadata types (`types.ts`) + +- `SessionHeader` — immutable, written once: `{ version, id, createdAt, cwd?, parentSession? }`. +- `SessionSummary` — mutable, updateable without touching the log: `{ updatedAt, title?, firstPrompt? }`. +- `SessionMeta = SessionHeader & SessionSummary` — owned here (beside `SessionId`) because `Session.header` is typed by it; persistence backends re-export these rather than own them (which would force a package cycle). + ### Extension points - Persistence plugins: subscribe to `session/event` (write-behind) and drain on `session/flush` (awaited) and fiber dispose. A durable backend reads the log and reloads it into a live session; the metadata seam (`SessionHeader`/`SessionSummary`/`SessionMeta`, `session.header`) is what such a backend stores beside the log. -- Replay/fork: `ctx.sessions.create(id, { seed })` seeds a new session with an existing event log. +- Replay/fork: `ctx.sessions.create(id, { seed })` seeds a new session with an existing event log. The surface rebuilds deterministically from `surfaceOp` markers in the seeded events. +- Compaction: a future plugin appends a new event with `surfaceOp: { op: 'replace', start, end }` to shadow old surface nodes. ### What is NOT here (TODO) -- **Session branching/tree** (pi-style entry tree) — defered unless needed beyond seed-based forking. +- **Session branching/tree** (pi-style entry tree) — deferred unless needed beyond seed-based forking. diff --git a/packages/session/src/index.ts b/packages/session/src/index.ts index 34eefb24a3..d0d4d44aed 100644 --- a/packages/session/src/index.ts +++ b/packages/session/src/index.ts @@ -10,12 +10,14 @@ import { Context, Service } from 'cordis' import { isAbsolute } from 'node:path' import type { ContentBlock, Message, MessageSource } from '@deepseek-ai/dsh-llm' import { SessionId } from './types.ts' -import type { CreateSessionOptions, SessionEvent, SessionEventMap, SessionEventType, SessionHeader } from './types.ts' +import type { CreateSessionOptions, SessionEvent, SessionEventMap, SessionEventType, SessionHeader, SurfaceAppendOpts } from './types.ts' import { isJsonValue } from './json.ts' +import { SurfaceManager } from './surface.ts' export * from './types.ts' export { isJsonValue } from './json.ts' export { interruptedTurnClosers } from './repair.ts' +export type { SurfaceNode } from './surface.ts' declare module 'cordis' { interface Context { @@ -65,6 +67,21 @@ export class Session { /** Set by the store so appends are observable; undefined when detached. */ onAppend: ((event: SessionEvent) => void) | undefined + /** + * Derived surface — a cached linked list of message-producing events. + * Lazily rebuilt from `surfaceOp` markers in the log; processes only new + * events (delta) on each access — the log is append-only, so prior events + * never change. + * `append`. Undefined until first accessed (including after fork/seed). + */ + private _surface: SurfaceManager | undefined + + /** The surface linked list over this session's event log. */ + get surface(): SurfaceManager { + if (!this._surface) this._surface = new SurfaceManager(this.log) + return this._surface + } + /** * Immutable creation metadata (format version, cwd, lineage). Supplied by * the store via `ctx.sessions.create()`. When a `Session` is constructed @@ -117,6 +134,11 @@ export class Session { * `onAppend`. The hot path never blocks on I/O — persistence plugins buffer * asynchronously. * + * @param type - The event type (key of {@link SessionEventMap}). + * @param data - The event payload; must be JSON-serializable. + * @param opts - Optional surface metadata: `surfaceOp` controls how the + * event enters the surface linked list; `sourceEventSeqs` records + * provenance (the seq numbers of events this one derives from). * @throws if `data` is not losslessly JSON-serializable (BigInt, function, * symbol, undefined, non-finite number, circular ref, or an exotic object * like Map/Set/Date). The event log is the durable source of truth, so this @@ -125,7 +147,7 @@ export class Session { * throw surfaces at the buggy caller's append site, not asynchronously in a * backend flush. */ - append(type: T, data: SessionEventMap[T]): SessionEvent { + append(type: T, data: SessionEventMap[T], opts?: SurfaceAppendOpts): SessionEvent { if (!isJsonValue(data)) { throw new Error(`session event "${type}" carries non-JSON-serializable data`) } @@ -138,14 +160,29 @@ export class Session { // validated. structuredClone is safe because serializability was just // checked. The returned event carries the SAME snapshot, so a caller reading // back `event.data` sees the logged value, not its own mutable input. - const event = { type, seq: this.log.length, time: Date.now(), data: structuredClone(data) } as SessionEvent + // + // Surface metadata is snapshot separately: sourceEventSeqs (number[] — + // primitives, so array spread is a complete copy) and surfaceOp (a string + // primitive, or cloned if it's a replace object). + const event = { + type, + seq: this.log.length, + time: Date.now(), + data: structuredClone(data), + ...opts?.sourceEventSeqs !== undefined ? { sourceEventSeqs: [...opts.sourceEventSeqs] } : {}, + ...opts?.surfaceOp !== undefined ? { + surfaceOp: typeof opts.surfaceOp === 'string' ? opts.surfaceOp : structuredClone(opts.surfaceOp), + } : {}, + } as SessionEvent this.log.push(event) this.onAppend?.(event) return event } /** - * Derive the LLM message history from the event log. + * Derive the LLM message history from the session surface (when surface + * markers exist) or from a linear scan of the raw event log (legacy sessions + * without surface markers). * * - `user/message` → user message * - `assistant/message` → assistant message (chunks are skipped — they are @@ -163,43 +200,62 @@ export class Session { * negligible next to a model call. */ deriveMessages(): Message[] { + if (this.surface.hasSurface) { + const messages: Message[] = [] + for (const node of this.surface.nodes) { + // Surface nodes are built from this.log — node.seq is always a valid + // index by construction. The non-null assertion expresses that invariant. + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const msg = this._deriveOneMessage(this.log[node.seq]!) + if (msg) messages.push(msg) + } + return messages + } + // Legacy path: linear scan for sessions without surface markers. const messages: Message[] = [] for (const event of this.log) { - // Intentionally non-exhaustive: only message-producing events derive - // history; turn/step boundaries, chunks, usage, and errors are - // trace/replay data. - // eslint-disable-next-line @typescript-eslint/switch-exhaustiveness-check - switch (event.type) { - case 'user/message': { - messages.push({ role: 'user', content: structuredClone(event.data.content) }) - break - } - case 'assistant/message': { - messages.push({ role: 'assistant', content: structuredClone(event.data.content) }) - break - } - case 'tool/result': { - const { callId, content, isError } = event.data - messages.push({ - role: 'user', - content: [{ type: 'tool-result', toolCallId: callId, content: structuredClone(content), isError }], - }) - break - } - case 'context/message': { - const { content, source } = event.data - messages.push({ role: 'user', content: renderTagged('context', structuredClone(content), source) }) - break - } - case 'steering/message': { - const { content, source } = event.data - messages.push({ role: 'user', content: renderTagged('steering', structuredClone(content), source) }) - break - } - } + const msg = this._deriveOneMessage(event) + if (msg) messages.push(msg) } return messages } + + /** + * Derive a single LLM message from one event, or null if the event type + * does not produce a message. Extracted so both the surface path and the + * legacy linear-scan path share the same derivation rules. + */ + private _deriveOneMessage(event: SessionEvent): Message | null { + // Intentionally non-exhaustive: only message-producing events derive + // history; turn/step boundaries, chunks, usage, and errors are + // trace/replay data. + + switch (event.type) { + case 'user/message': { + return { role: 'user', content: structuredClone(event.data.content) } + } + case 'assistant/message': { + return { role: 'assistant', content: structuredClone(event.data.content) } + } + case 'tool/result': { + const { callId, content, isError } = event.data + return { + role: 'user', + content: [{ type: 'tool-result', toolCallId: callId, content: structuredClone(content), isError }], + } + } + case 'context/message': { + const { content, source } = event.data + return { role: 'user', content: renderTagged('context', structuredClone(content), source) } + } + case 'steering/message': { + const { content, source } = event.data + return { role: 'user', content: renderTagged('steering', structuredClone(content), source) } + } + default: + return null + } + } } /** diff --git a/packages/session/src/repair.ts b/packages/session/src/repair.ts index 6a3f60a681..d313af12c3 100644 --- a/packages/session/src/repair.ts +++ b/packages/session/src/repair.ts @@ -61,7 +61,12 @@ export function interruptedTurnClosers(events: readonly SessionEvent[]): Session // call is "pending" until its matching tool/result arrives. Reset at every // turn boundary so a committed earlier turn (already balanced) never leaks a // phantom pending call into the interrupted-turn repair. - const pendingCalls = new Map() + // Track pending tool calls with their callSeq (the seq of the `tool/call` + // event, captured for surface sourceEventSeqs provenance on the synthetic + // result). CallSeq is set from `tool/call` events; the assistant/message + // block scan may register a call first (it appears earlier in the log), and + // the later `tool/call` event fills in the seq. + const pendingCalls = new Map() for (const event of events) { switch (event.type) { case 'turn/start': @@ -87,6 +92,18 @@ export function interruptedTurnClosers(events: readonly SessionEvent[]): Session if (block.type === 'tool-call') pendingCalls.set(block.id, { step: event.data.step }) } break + case 'tool/call': + // Capture the tool/call event seq for surface provenance on the + // synthesized tool/result. The entry may already exist (registered by + // the assistant/message above) or may be new (if the assistant/message + // came from a prior step that was already closed). + { + const entry = pendingCalls.get(event.data.callId) + if (entry) { + entry.callSeq = event.seq + } + } + break case 'tool/result': pendingCalls.delete(event.data.callId) break @@ -112,7 +129,7 @@ export function interruptedTurnClosers(events: readonly SessionEvent[]): Session // crash, so deriveMessages() yields a valid provider transcript on resume (a // dangling assistant tool-call is rejected by every provider). Insertion // order follows the Map (insertion = log order of the assistant messages). - for (const [callId, { step }] of pendingCalls) { + for (const [callId, { step, callSeq }] of pendingCalls) { closers.push({ type: 'tool/result', seq: seq++, @@ -125,6 +142,8 @@ export function interruptedTurnClosers(events: readonly SessionEvent[]): Session isError: true, error: { name: 'InterruptedError', code: 'interrupted' }, }, + surfaceOp: 'append', + ...callSeq !== undefined ? { sourceEventSeqs: [callSeq] } : {}, }) } diff --git a/packages/session/src/surface.ts b/packages/session/src/surface.ts new file mode 100644 index 0000000000..a09dbd001f --- /dev/null +++ b/packages/session/src/surface.ts @@ -0,0 +1,129 @@ +/** + * Surface layer on top of the session event log: a derived, cached linked list + * of events that produce LLM messages. Rebuilt deterministically from + * `surfaceOp` markers in the log — the log is the source of truth; the surface + * is a view. + * + * @module @deepseek-ai/dsh-session/surface + */ + +import type { SessionEvent, SurfaceOp } from './types.ts' + +/** One node in the surface linked list. */ +export interface SurfaceNode { + /** The event seq of this surface node. */ + seq: number + /** The previous surface node's seq, or null if this is the head. */ + prev: number | null + /** The next surface node's seq, or null if this is the tail. */ + next: number | null +} + +/** + * Maintains a cached linked list of surface nodes, rebuilt lazily from + * `surfaceOp` markers in the event log. Because the log is append-only, it + * processes only the delta since the last rebuild — new events are folded + * into the existing surface in O(new events) rather than rescanning the + * whole log. + */ +export class SurfaceManager { + /** Surface nodes in linked-list order (head to tail). Empty until first access. */ + private _nodes: SurfaceNode[] = [] + /** Map from event seq → node for O(1) lookup during replacements. */ + private _nodeBySeq = new Map() + /** The last processed seq. -1 forces a full rebuild on first access. */ + private _lastProcessedSeq = -1 + + constructor(private log: readonly SessionEvent[]) {} + + /** + * Reset to unprocessed state. Call after the log has been replaced + * wholesale (e.g. after Session seed). Not needed for normal appends — + * those are picked up incrementally. + */ + invalidate(): void { + this._lastProcessedSeq = -1 + this._nodes = [] + this._nodeBySeq.clear() + } + + /** The surface nodes in linked-list order (head to tail). */ + get nodes(): readonly SurfaceNode[] { + if (this._lastProcessedSeq < this.log.length - 1) this._processDelta() + return this._nodes + } + + /** Whether any event in the log carries `surfaceOp` markers. */ + get hasSurface(): boolean { + if (this._nodes.length > 0) return true + // Never processed anything — scan the whole log. + if (this._lastProcessedSeq === -1) return this.log.some(e => e.surfaceOp !== undefined) + // Processed up to _lastProcessedSeq without finding surface nodes; check + // only new events. + for (let i = this._lastProcessedSeq + 1; i < this.log.length; i++) { + if (this.log[i]?.surfaceOp !== undefined) return true + } + return false + } + + /** + * Process events from `_lastProcessedSeq + 1` through the end of the log, + * folding new surface markers into the existing linked list. + */ + private _processDelta(): void { + for (let i = this._lastProcessedSeq + 1; i < this.log.length; i++) { + const event = this.log[i] + if (event === undefined || event.surfaceOp === undefined) continue + + if (event.surfaceOp === 'append') { + const tail = this._nodes.length > 0 ? this._nodes[this._nodes.length - 1] : undefined + const node: SurfaceNode = { seq: event.seq, prev: tail?.seq ?? null, next: null } + if (tail) tail.next = event.seq + this._nodes.push(node) + this._nodeBySeq.set(event.seq, node) + } else { + this._replace(this._nodes, this._nodeBySeq, event.seq, event.surfaceOp) + } + } + this._lastProcessedSeq = this.log.length - 1 + } + + /** Apply a replace operation to the in-progress surface. */ + private _replace( + nodes: SurfaceNode[], + nodeBySeq: Map, + newSeq: number, + op: Extract, + ): void { + const startIdx = nodes.findIndex(n => n.seq === op.start) + if (startIdx === -1) { + throw new Error(`surface replace: start seq ${op.start} not found in surface`) + } + const endIdx = nodes.findIndex(n => n.seq === op.end) + if (endIdx === -1) { + throw new Error(`surface replace: end seq ${op.end} not found in surface`) + } + if (startIdx > endIdx) { + throw new Error(`surface replace: start seq ${op.start} (index ${startIdx}) is after end seq ${op.end} (index ${endIdx})`) + } + + // Remove shadowed nodes from `[startIdx, endIdx]` inclusive. + const count = endIdx - startIdx + 1 + const removed = nodes.splice(startIdx, count) + for (const r of removed) nodeBySeq.delete(r.seq) + + // Insert the new node where the removed range was. + const prevNode = startIdx > 0 ? nodes[startIdx - 1] : undefined + const nextNode = startIdx < nodes.length ? nodes[startIdx] : undefined + + const newNode: SurfaceNode = { + seq: newSeq, + prev: prevNode?.seq ?? null, + next: nextNode?.seq ?? null, + } + if (prevNode) prevNode.next = newSeq + if (nextNode) nextNode.prev = newSeq + nodes.splice(startIdx, 0, newNode) + nodeBySeq.set(newSeq, newNode) + } +} diff --git a/packages/session/src/types.ts b/packages/session/src/types.ts index d8ffaf9dc8..9f2cd87643 100644 --- a/packages/session/src/types.ts +++ b/packages/session/src/types.ts @@ -160,6 +160,34 @@ export interface SessionEventMap { export type SessionEventType = keyof SessionEventMap +/** + * How a session event entered the surface linked list. Absent for non-surface + * events (boundaries, chunks, usage, errors). + * + * - `'append'`: added to the tail — normal path for user/assistant/tool/context + * messages. + * - `{ op: 'replace', start, end }`: replaces surface nodes from `start` + * (inclusive) through `end` (inclusive) with this node. Both must exist as + * surface nodes in the current surface. `start === end` replaces a single + * node. The node's {@link SessionEvent.sourceEventSeqs} must include every + * shadowed surface node. Used by compaction and possible other manipulations. + */ +export type SurfaceOp = + | 'append' + | { op: 'replace'; start: number; end: number } + +/** + * Optional surface metadata passed to {@link Session.append}. + * `surfaceOp` controls how the event enters the surface linked list; + * `sourceEventSeqs` records the seq numbers of events that are provenance + * sources of this one (e.g. the `assistant/chunk` seqs behind an + * `assistant/message`, or the shadowed nodes behind a compaction replacement). + */ +export interface SurfaceAppendOpts { + surfaceOp?: SurfaceOp + sourceEventSeqs?: number[] +} + /** * One immutable entry in the session log. * @@ -174,5 +202,13 @@ export type SessionEvent = { /** Unix epoch milliseconds. */ time: number data: SessionEventMap[K] + /** + * Seq numbers of events that are provenance sources of this event + * (e.g. the `assistant/chunk` seqs that built an `assistant/message`, + * or the surface nodes shadowed by a compaction marker). + */ + sourceEventSeqs?: number[] + /** How this event entered the surface; absent for non-surface events. */ + surfaceOp?: SurfaceOp } }[T] diff --git a/packages/session/tests/repair.spec.ts b/packages/session/tests/repair.spec.ts index 893015b218..cf6fb2b51c 100644 --- a/packages/session/tests/repair.spec.ts +++ b/packages/session/tests/repair.spec.ts @@ -122,4 +122,36 @@ describe('interruptedTurnClosers', () => { const result = closers[0]! expect(result.type === 'tool/result' && result.data.callId).toBe('call-b') }) + + it('synthesized tool/result carries surfaceOp and sourceEventSeqs when tool/call was logged', () => { + const events: SessionEvent[] = [ + userTurnStart(1, 0), + { type: 'step/start', seq: 1, time: 1, data: { turn: 1, step: 1 } }, + { type: 'assistant/message', seq: 2, time: 2, data: { turn: 1, step: 1, content: [ + { type: 'tool-call', id: CallId('call-1'), name: 'bash', arguments: '{}' }, + ] } }, + { type: 'tool/call', seq: 3, time: 3, data: { turn: 1, step: 1, callId: CallId('call-1'), name: 'bash', arguments: '{}' } }, + ] + const closers = interruptedTurnClosers(events) + expect(closers.map(e => e.type)).toEqual(['tool/result', 'step/end', 'turn/end']) + const result = closers[0]! + expect(result.surfaceOp).toBe('append') + expect(result.sourceEventSeqs).toEqual([3]) + }) + + it('handles tool/call without a matching assistant/message entry gracefully', () => { + // A tool/call event exists in the log but no assistant/message registered + // the callId in pendingCalls (e.g., a plugin appended it directly, or the + // assistant/message from a prior step didn't have this call). The repair + // should still close the turn — it just won't synthesize a result for this + // call (there's nothing to answer). + const events: SessionEvent[] = [ + userTurnStart(1, 0), + { type: 'step/start', seq: 1, time: 1, data: { turn: 1, step: 1 } }, + { type: 'tool/call', seq: 2, time: 2, data: { turn: 1, step: 1, callId: CallId('orphan'), name: 'bash', arguments: '{}' } }, + ] + const closers = interruptedTurnClosers(events) + // No pending calls → no synthetic tool/result, just step/end + turn/end. + expect(closers.map(e => e.type)).toEqual(['step/end', 'turn/end']) + }) }) diff --git a/packages/session/tests/surface.spec.ts b/packages/session/tests/surface.spec.ts new file mode 100644 index 0000000000..d8f503c5e5 --- /dev/null +++ b/packages/session/tests/surface.spec.ts @@ -0,0 +1,324 @@ +import { describe, expect, it } from 'vitest' +import type { SessionEvent } from '@deepseek-ai/dsh-session' +import { Session, SessionId } from '@deepseek-ai/dsh-session' +import { CallId } from '@deepseek-ai/dsh-llm' + +/** Build a minimal session with turn boundaries and a single user message. */ +function surfaceSession(): Session { + const s = new Session(SessionId('ss')) + s.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + s.append('user/message', { content: [{ type: 'text', text: 'hello' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) + s.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 'hi' }] }, { surfaceOp: 'append' }) + s.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) + return s +} + +describe('SurfaceManager', () => { + it('rebuilds a linked list from surfaceOp: append markers', () => { + const s = surfaceSession() + const nodes = s.surface.nodes + // Only the user/message and assistant/message carry surfaceOp: 'append'. + // The turn boundaries do not have surface markers. + expect(nodes.length).toBe(2) + expect(nodes[0]!.seq).toBe(1) // user/message (turn/start is seq 0) + expect(nodes[0]!.prev).toBeNull() + expect(nodes[0]!.next).toBe(2) // assistant/message (seq 2) + expect(nodes[1]!.seq).toBe(2) + expect(nodes[1]!.prev).toBe(1) + expect(nodes[1]!.next).toBeNull() + }) + + it('hasSurface returns false when no events have surfaceOp', () => { + const s = new Session(SessionId('nosurface')) + s.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + s.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }) + s.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) + expect(s.surface.hasSurface).toBe(false) + }) + + it('hasSurface returns true when any event has surfaceOp', () => { + const s = surfaceSession() + expect(s.surface.hasSurface).toBe(true) + }) + + it('hasSurface detects surface markers that arrive after initial processing', () => { + // Start with no surface markers. Access nodes first to set _lastProcessedSeq + // (via delta processing), keeping _nodes empty. Then append a mix of non-surface + // and surface events, and verify hasSurface detects via the delta-only check. + const s = new Session(SessionId('late')) + s.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + s.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }) + // Access nodes to trigger processing: sets _lastProcessedSeq = 1, _nodes = []. + expect(s.surface.nodes.length).toBe(0) + // Append non-surface events first (exercises the loop-continue branch), then + // a surface event (exercises the return-true branch). + s.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) + s.append('turn/start', { turn: 2, trigger: { kind: 'continuation' } }) + s.append('assistant/message', { turn: 2, step: 1, content: [] }, { surfaceOp: 'append' }) + // hasSurface checks only new seqs [2, 3, 4]; skips 2 and 3 (non-surface), + // finds surfaceOp on seq 4 and returns true. + expect(s.surface.hasSurface).toBe(true) + }) + + it('invalidate resets to full rebuild', () => { + const s = surfaceSession() + expect(s.surface.nodes.length).toBe(2) + // After invalidate, the surface should rebuild from scratch on next access. + ;(s.surface).invalidate() + expect(s.surface.nodes.length).toBe(2) // same result, but rebuilt + }) + + it('empty surface yields empty nodes', () => { + const s = new Session(SessionId('empty')) + // Only turn boundaries, no surface nodes. + s.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + s.append('step/start', { turn: 1, step: 1 }) + s.append('step/end', { turn: 1, step: 1 }) + s.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) + expect(s.surface.nodes.length).toBe(0) + expect(s.surface.hasSurface).toBe(false) + // deriveMessages returns empty array + expect(s.deriveMessages()).toEqual([]) + }) + + it('picks up new events incrementally (delta processing)', () => { + const s = surfaceSession() + expect(s.surface.nodes.length).toBe(2) + // Append another surface node + s.append('tool/result', { turn: 1, step: 1, callId: CallId('c1'), content: [{ type: 'text', text: 'ok' }], isError: false }, { surfaceOp: 'append' }) + expect(s.surface.nodes.length).toBe(3) + expect(s.surface.nodes[2]!.seq).toBe(4) // seq 4: after turn/end at seq 3 + expect(s.surface.nodes[2]!.prev).toBe(2) + expect(s.surface.nodes[1]!.next).toBe(4) + }) + + it('replays identically from a seeded log with surface markers', () => { + const original = surfaceSession() + original.append('tool/result', { turn: 1, step: 1, callId: CallId('c1'), content: [{ type: 'text', text: 'ok' }], isError: false }, { surfaceOp: 'append' }) + const replayed = new Session(SessionId('replay'), [...original.events]) + // Surface rebuilds from the seeded log's markers. + expect(replayed.surface.nodes.map(n => n.seq)).toEqual([1, 2, 4]) + expect(replayed.deriveMessages()).toEqual(original.deriveMessages()) + }) + + it('rebuild with replace operation splices out shadowed nodes', () => { + const s = surfaceSession() + // seq: 0=turn/start, 1=user, 2=assistant, 3=turn/end + // Surface nodes: seq 1 (user), seq 2 (assistant). + // Replace both with a compaction marker. Both 1 and 2 are valid surface seqs. + s.append('assistant/message', + { turn: 2, step: 1, content: [{ type: 'text', text: 'summary' }] }, + { surfaceOp: { op: 'replace', start: 1, end: 2 }, sourceEventSeqs: [1, 2] }, + ) + // Now the surface should have just the compaction node. + expect(s.surface.nodes.length).toBe(1) + expect(s.surface.nodes[0]!.seq).toBe(4) // seq of the compaction marker + expect(s.surface.nodes[0]!.prev).toBeNull() + expect(s.surface.nodes[0]!.next).toBeNull() + }) + + it('replace with both ends at real nodes splices only the range', () => { + const s = new Session(SessionId('range')) + s.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 0 + s.append('user/message', { content: [{ type: 'text', text: 'b' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 1 + s.append('user/message', { content: [{ type: 'text', text: 'c' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 2 + // Replace seq 0 through 1 inclusive: shadow a and b, keep c. + s.append('assistant/message', + { turn: 1, step: 1, content: [{ type: 'text', text: 'summary' }] }, + { surfaceOp: { op: 'replace', start: 0, end: 1 }, sourceEventSeqs: [0, 1] }, + ) // seq 3 + expect(s.surface.nodes.map(n => n.seq)).toEqual([3, 2]) + // Links: 3 ↔ 2 + expect(s.surface.nodes[0]!.prev).toBeNull() + expect(s.surface.nodes[0]!.next).toBe(2) + expect(s.surface.nodes[1]!.prev).toBe(3) + expect(s.surface.nodes[1]!.next).toBeNull() + }) + + it('single-node replacement (start === end)', () => { + const s = new Session(SessionId('single')) + s.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 0 + s.append('user/message', { content: [{ type: 'text', text: 'b' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 1 + // Replace only seq 1 (single node). + s.append('assistant/message', + { turn: 1, step: 1, content: [{ type: 'text', text: 'x' }] }, + { surfaceOp: { op: 'replace', start: 1, end: 1 }, sourceEventSeqs: [1] }, + ) // seq 2 + expect(s.surface.nodes.map(n => n.seq)).toEqual([0, 2]) + expect(s.surface.nodes[0]!.next).toBe(2) + expect(s.surface.nodes[1]!.prev).toBe(0) + }) + + it('throws when replace start is not found', () => { + const s = new Session(SessionId('bad-start')) + s.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 0 + s.append('assistant/message', + { turn: 1, step: 1, content: [{ type: 'text', text: 'y' }] }, + { surfaceOp: { op: 'replace', start: 5, end: 0 }, sourceEventSeqs: [5, 0] }, + ) + expect(() => s.surface.nodes).toThrow(/surface replace: start seq 5 not found/) + }) + + it('throws when replace end is not found', () => { + const s = new Session(SessionId('bad-end')) + s.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 0 + s.append('assistant/message', + { turn: 1, step: 1, content: [{ type: 'text', text: 'y' }] }, + { surfaceOp: { op: 'replace', start: 0, end: 99 }, sourceEventSeqs: [0] }, + ) + expect(() => s.surface.nodes).toThrow(/surface replace: end seq 99 not found/) + }) + + it('throws when start is after end', () => { + const s = new Session(SessionId('reversed')) + s.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 0 + s.append('user/message', { content: [{ type: 'text', text: 'b' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 1 + // start=1, end=0 would be reversed order. + s.append('assistant/message', + { turn: 1, step: 1, content: [{ type: 'text', text: 'y' }] }, + { surfaceOp: { op: 'replace', start: 1, end: 0 }, sourceEventSeqs: [1, 0] }, + ) + expect(() => s.surface.nodes).toThrow(/start seq 1.*after end seq 0/) + }) + + it('sourceEventSeqs is snapshot so caller mutation does not affect logged event', () => { + const s = new Session(SessionId('immutable')) + const sources = [10, 20] + s.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 'h' }] }, { surfaceOp: 'append', sourceEventSeqs: sources }) + // Mutate caller's array after append. + sources.push(30) + sources[0] = 99 + const logged = s.events[0]! + expect(logged.sourceEventSeqs).toEqual([10, 20]) + }) + + it('replace starting at non-head position links to previous node correctly', () => { + const s = new Session(SessionId('mid-replace')) + s.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 0 + s.append('user/message', { content: [{ type: 'text', text: 'b' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 1 + s.append('user/message', { content: [{ type: 'text', text: 'c' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 2 + // Replace the middle node (seq 1) only, keeping seq 0 and seq 2. + s.append('assistant/message', + { turn: 1, step: 1, content: [{ type: 'text', text: 'x' }] }, + { surfaceOp: { op: 'replace', start: 1, end: 1 }, sourceEventSeqs: [1] }, + ) // seq 3 + expect(s.surface.nodes.map(n => n.seq)).toEqual([0, 3, 2]) + // Links: 0 → 3 → 2 + expect(s.surface.nodes[0]!.prev).toBeNull() + expect(s.surface.nodes[0]!.next).toBe(3) + expect(s.surface.nodes[1]!.prev).toBe(0) + expect(s.surface.nodes[1]!.next).toBe(2) + expect(s.surface.nodes[2]!.prev).toBe(3) + expect(s.surface.nodes[2]!.next).toBeNull() + }) + + it('surfaceOp replace object is snapshot so caller mutation is isolated', () => { + const s = new Session(SessionId('immutable-op')) + s.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) + const op = { op: 'replace' as const, start: 0, end: 0 } + s.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 's' }] }, { surfaceOp: op, sourceEventSeqs: [0] }) + // Mutate caller's object after append. + op.start = 99 + const logged = s.events[1]! + expect(logged.surfaceOp).toEqual({ op: 'replace', start: 0, end: 0 }) + }) +}) + +describe('deriveMessages with surface', () => { + it('uses the surface path when surface markers are present', () => { + const s = surfaceSession() + const messages = s.deriveMessages() + expect(messages).toHaveLength(2) + expect(messages[0]!.role).toBe('user') + expect(messages[0]!.content[0]).toMatchObject({ type: 'text', text: 'hello' }) + expect(messages[1]!.role).toBe('assistant') + expect(messages[1]!.content[0]).toMatchObject({ type: 'text', text: 'hi' }) + }) + + it('falls back to linear scan when no surface markers exist', () => { + const s = new Session(SessionId('legacy')) + s.append('user/message', { content: [{ type: 'text', text: 'q' }], source: { kind: 'user' } }) + s.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 'a' }] }) + const messages = s.deriveMessages() + expect(messages).toHaveLength(2) + expect(messages[0]!.role).toBe('user') + expect(messages[1]!.role).toBe('assistant') + }) + + it('surface path skips non-surface events (chunks, boundaries)', () => { + const s = new Session(SessionId('filter')) + s.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + s.append('assistant/chunk', { turn: 1, step: 1, chunk: { type: 'text-delta', index: 0, text: 'h' } }) + s.append('assistant/chunk', { turn: 1, step: 1, chunk: { type: 'text-delta', index: 1, text: 'i' } }) + s.append('user/message', { content: [{ type: 'text', text: 'hello' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) + s.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 'hi' }] }, { surfaceOp: 'append' }) + s.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) + // Chunks and boundaries are NOT in the surface, so only 2 messages. + expect(s.deriveMessages()).toHaveLength(2) + }) + + it('deriveMessages via surface respects replace (shadowed nodes are excluded)', () => { + const s = new Session(SessionId('compacted')) + s.append('user/message', { content: [{ type: 'text', text: 'original' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) + s.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 'compacted' }] }, { surfaceOp: { op: 'replace', start: 0, end: 0 }, sourceEventSeqs: [0] }) + // Only the compaction node is visible. + const messages = s.deriveMessages() + expect(messages).toHaveLength(1) + expect(messages[0]!.content[0]).toMatchObject({ type: 'text', text: 'compacted' }) + }) + + it('context/message and steering/message appear on surface', () => { + const s = new Session(SessionId('ctx')) + s.append('context/message', { content: [{ type: 'text', text: 'file changed' }], source: { kind: 'plugin', plugin: 'watcher' } }, { surfaceOp: 'append' }) + s.append('steering/message', { turn: 1, content: [{ type: 'text', text: 'focus' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) + const messages = s.deriveMessages() + expect(messages).toHaveLength(2) + expect(messages[0]!.content[0]).toMatchObject({ type: 'text', text: '' }) + expect(messages[1]!.content[0]).toMatchObject({ type: 'text', text: '' }) + }) +}) + +describe('Session.append surface opts', () => { + it('records sourceEventSeqs and surfaceOp on the event', () => { + const s = new Session(SessionId('opts')) + const event = s.append('assistant/message', + { turn: 1, step: 1, content: [{ type: 'text', text: 'h' }] }, + { surfaceOp: 'append', sourceEventSeqs: [3, 5, 7] }, + ) + expect(event.sourceEventSeqs).toEqual([3, 5, 7]) + expect(event.surfaceOp).toBe('append') + // The logged event matches the returned event. + expect(s.events[0]!.sourceEventSeqs).toEqual([3, 5, 7]) + expect(s.events[0]!.surfaceOp).toBe('append') + }) + + it('deriveMessages skips surface nodes whose event type is not message-producing', () => { + // A surface node with a type not handled by _deriveOneMessage (e.g., 'usage' + // placed on surface) should be skipped — the null-check in the surface + // derivation path is exercised. + const seed: SessionEvent[] = [ + { type: 'turn/start', seq: 0, time: 1, data: { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } } }, + { type: 'step/start', seq: 1, time: 2, data: { turn: 1, step: 1 } }, + { type: 'usage', seq: 2, time: 3, data: { turn: 1, step: 1, usage: { inputTokens: 0, outputTokens: 0 } }, surfaceOp: 'append' as const }, + { type: 'step/end', seq: 3, time: 4, data: { turn: 1, step: 1 } }, + { type: 'turn/end', seq: 4, time: 5, data: { turn: 1, reason: { kind: 'completed' } } }, + ] + const s = new Session(SessionId('nomessage'), seed) + // The usage event is on the surface but _deriveOneMessage returns null for it. + expect(s.deriveMessages()).toHaveLength(0) + }) + + it('append without surface opts produces an event without surface fields', () => { + const s = new Session(SessionId('noopts')) + s.append('user/message', { content: [{ type: 'text', text: 'x' }], source: { kind: 'user' } }) + expect(s.events[0]!.sourceEventSeqs).toBeUndefined() + expect(s.events[0]!.surfaceOp).toBeUndefined() + }) + + it('surfaceOp primitives are not cloned (they are immutable)', () => { + const s = new Session(SessionId('prim')) + const event = s.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append' }) + // The string 'append' is a primitive — identity-preserving is fine. + expect(event.surfaceOp).toBe('append') + }) +}) From e5d82631942a55916d6199b8cac522a44db51db7 Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Thu, 18 Jun 2026 09:37:19 +0800 Subject: [PATCH 02/17] fix broken cross-links --- docs/rfc/implemented/2026-06-18-session-surface.md | 6 +++--- packages/session-persistence-sqlite/README.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/rfc/implemented/2026-06-18-session-surface.md b/docs/rfc/implemented/2026-06-18-session-surface.md index 159db911b1..528718b968 100644 --- a/docs/rfc/implemented/2026-06-18-session-surface.md +++ b/docs/rfc/implemented/2026-06-18-session-surface.md @@ -1,10 +1,10 @@ -# ADR 0019: Session surface — a linked list over the event log for LLM message derivation +# RFC: Session surface — a linked list over the event log for LLM message derivation -Status: accepted (2026-06-17) +Status: implemented (accepted 2026-06-18) ## Context -The `Session` event log is the single source of truth ([ADR 0003](0003-event-sourced-sessions.md)), but the only view over it was `deriveMessages()` — a linear scan that filtered and transformed raw events into `Message[]`. This creates problems for session-history-manipulating plugins (compaction, tool-call result pruning, etc.). Without a central mechanism, each plugin would need to wrap `agent/request` to rewrite the message list — a pattern that suffers from listener-ordering fragility, provides no durable record of what was changed, and forces repeated changes to the core `deriveMessages()` whenever a new manipulation is added. A central hub in the `session` package, with a provenance-recording mechanism and enough flexibility for future plugins to manipulate session history through a stable API, lays a solid foundation for plugin development. +The `Session` event log is the single source of truth ([event-sourced sessions](../implemented/2026-06-11-event-sourced-sessions.md)), but the only view over it was `deriveMessages()` — a linear scan that filtered and transformed raw events into `Message[]`. This creates problems for session-history-manipulating plugins (compaction, tool-call result pruning, etc.). Without a central mechanism, each plugin would need to wrap `agent/request` to rewrite the message list — a pattern that suffers from listener-ordering fragility, provides no durable record of what was changed, and forces repeated changes to the core `deriveMessages()` whenever a new manipulation is added. A central hub in the `session` package, with a provenance-recording mechanism and enough flexibility for future plugins to manipulate session history through a stable API, lays a solid foundation for plugin development. ## Decision diff --git a/packages/session-persistence-sqlite/README.md b/packages/session-persistence-sqlite/README.md index cec1e9700f..6eb8568e46 100644 --- a/packages/session-persistence-sqlite/README.md +++ b/packages/session-persistence-sqlite/README.md @@ -6,7 +6,7 @@ A SQLite durable session-persistence backend — a second `SessionPersistence` i ## Storage model -Each `SessionEvent` maps 1:1 onto a row in an `events` table `(session_id, seq, type, time, data, source_event_seqs, surface_op)` — `data` is the event payload as JSON text, so the row shape is the event verbatim (including `assistant/chunk`, keeping `seq` contiguous). The two `TEXT` columns `source_event_seqs` and `surface_op` are nullable; they store the event's optional surface-metadata fields (see [ADR 0019](../../docs/adr/0019-session-surface.md)). The schema migrates from v1 to v2 via `ALTER TABLE ADD COLUMN` — existing rows get NULL for both columns, which is correct for events written before surface support. Out-of-log metadata (`SessionMeta`) lives in a `sessions` row, including the mutable `SessionSummary` fields (`updatedAt`, `title`, `firstPrompt`) that `update()` rewrites without touching the event log. A `sessions` row is written only by the first `append` — its existence is the lazy-materialization signal (`has`/`list` report exactly the sessions that have a row), so no separate column is needed. +Each `SessionEvent` maps 1:1 onto a row in an `events` table `(session_id, seq, type, time, data, source_event_seqs, surface_op)` — `data` is the event payload as JSON text, so the row shape is the event verbatim (including `assistant/chunk`, keeping `seq` contiguous). The two `TEXT` columns `source_event_seqs` and `surface_op` are nullable; they store the event's optional surface-metadata fields (see [session surface](../../docs/rfc/implemented/2026-06-18-session-surface.md)). The schema migrates from v1 to v2 via `ALTER TABLE ADD COLUMN` — existing rows get NULL for both columns, which is correct for events written before surface support. Out-of-log metadata (`SessionMeta`) lives in a `sessions` row, including the mutable `SessionSummary` fields (`updatedAt`, `title`, `firstPrompt`) that `update()` rewrites without touching the event log. A `sessions` row is written only by the first `append` — its existence is the lazy-materialization signal (`has`/`list` report exactly the sessions that have a row), so no separate column is needed. The repo targets Node ≥ 24 (the root `engines` field), which includes the stable `node:sqlite` module. The database opens with `foreign_keys = ON` (so `ON DELETE CASCADE` drops a session's events with its row) and `journal_mode = WAL`. The table-layout version is stored in `PRAGMA user_version` and checked on open: a fresh database is stamped with the current `SCHEMA_VERSION`; a database written by a newer, incompatible build (higher `user_version`) is rejected rather than opened against an unknown layout. From 0279cf09d678ec0298715f61329a2e9b968f8e3f Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Thu, 18 Jun 2026 13:26:50 +0800 Subject: [PATCH 03/17] feat(session): enforce SurfaceEvent type --- packages/invariants/src/index.ts | 35 ++++++++++++---- packages/invariants/tests/invariants.spec.ts | 26 +++++++++++- .../session-persistence-sqlite/src/index.ts | 7 ++-- .../session-persistence-sqlite/src/schema.ts | 16 +++---- .../tests/sqlite.spec.ts | 28 ++++++------- packages/session/src/index.ts | 36 ++++++++++++---- packages/session/src/surface.ts | 42 ++++++++++++++++--- packages/session/src/types.ts | 40 ++++++++++++++++-- packages/session/tests/repair.spec.ts | 6 +-- packages/session/tests/surface.spec.ts | 16 +++---- 10 files changed, 189 insertions(+), 63 deletions(-) diff --git a/packages/invariants/src/index.ts b/packages/invariants/src/index.ts index 7655be46d6..97cb0af680 100644 --- a/packages/invariants/src/index.ts +++ b/packages/invariants/src/index.ts @@ -22,7 +22,7 @@ import type { Context } from 'cordis' import { HarnessError } from '@deepseek-ai/dsh-llm' import type { Agent, AgentStatus } from '@deepseek-ai/dsh-agent' -import type { Session, SessionEvent } from '@deepseek-ai/dsh-session' +import type { Session, SessionEvent, SurfaceEventType } from '@deepseek-ai/dsh-session' export const name = 'invariants' export const inject = ['sessions'] @@ -108,15 +108,32 @@ function checkEvent(trace: SessionTrace, event: SessionEvent): void { trace.lastSeq = event.seq // --- Surface invariants --- - if (event.sourceEventSeqs !== undefined) { - if (event.sourceEventSeqs.length === 0) { + // Surface metadata (sourceEventSeqs, surfaceOp) is only valid on + // surface-eligible event types. The compiler enforces this at append() + // call sites; this runtime check catches casts and persisted data. + const SURFACE_TYPES = new Set(['user/message', 'assistant/message', 'tool/result', 'context/message', 'steering/message']) + // Cast to surface-eligible event type so we can access surfaceOp and + // sourceEventSeqs (optional on SessionEvent, mandatory on SurfaceEvent). + // SurfaceEvent's mandatory surfaceOp is too strict here — we need to + // CHECK whether surface metadata is present, not assume it. + const se = event as SessionEvent + if (!SURFACE_TYPES.has(event.type)) { + if (se.sourceEventSeqs !== undefined) { + throw new InvariantError(`${event.type} cannot carry sourceEventSeqs (non-surface event)`) + } + if (se.surfaceOp !== undefined) { + throw new InvariantError(`${event.type} cannot carry surfaceOp (non-surface event)`) + } + } + if (se.sourceEventSeqs !== undefined) { + if (se.sourceEventSeqs.length === 0) { throw new InvariantError('sourceEventSeqs must not be empty when present') } - const unique = new Set(event.sourceEventSeqs) - if (unique.size !== event.sourceEventSeqs.length) { + const unique = new Set(se.sourceEventSeqs) + if (unique.size !== se.sourceEventSeqs.length) { throw new InvariantError('sourceEventSeqs must not contain duplicates') } - for (const ref of event.sourceEventSeqs) { + for (const ref of se.sourceEventSeqs) { if (ref >= event.seq) { throw new InvariantError(`sourceEventSeqs must reference earlier events: ${ref} >= current seq ${event.seq}`) } @@ -125,9 +142,9 @@ function checkEvent(trace: SessionTrace, event: SessionEvent): void { } } } - if (event.surfaceOp !== undefined && typeof event.surfaceOp !== 'string') { - if (event.surfaceOp.start > event.surfaceOp.end) { - throw new InvariantError(`surface replace: start ${event.surfaceOp.start} must be <= end ${event.surfaceOp.end}`) + if (se.surfaceOp !== undefined && typeof se.surfaceOp !== 'string') { + if (se.surfaceOp.start > se.surfaceOp.end) { + throw new InvariantError(`surface replace: start ${se.surfaceOp.start} must be <= end ${se.surfaceOp.end}`) } } diff --git a/packages/invariants/tests/invariants.spec.ts b/packages/invariants/tests/invariants.spec.ts index 8e4c5edebe..28fd27163f 100644 --- a/packages/invariants/tests/invariants.spec.ts +++ b/packages/invariants/tests/invariants.spec.ts @@ -105,7 +105,11 @@ describe('session-log invariants', () => { expect(() => session.append('error', { turn: 1, step: 1, message: 'boom' })) .toThrow(/outside any open turn/) // A PLUGIN-added (merge-extensible) event type is caught by the default too. - expect(() => session.append('compaction/marker' as never, { foo: 'bar' } as never)) + // Cast through `any`: 'compaction/marker' is not in SessionEventType (it's + // merge-extensible), so the typed append() won't accept it. The test verifies + // the runtime default-branch turn-enclosure check. + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-return + expect(() => (session.append as any)('compaction/marker', { foo: 'bar' })) .toThrow(/outside any open turn/) }) @@ -498,4 +502,24 @@ describe('surface invariants', () => { session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 2, end: 1 }, sourceEventSeqs: [2] }) }).toThrow(/must be <= end/) }) + + it('rejects sourceEventSeqs on a non-surface event', async () => { + const { ctx } = await setup() + const session = ctx.sessions.create() + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + // Type system prevents surface metadata on non-surface events; this test + // exercises the runtime guard against casts or persisted-data bypass. + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-return + expect(() => (session.append as any)('turn/end', { turn: 1, reason: { kind: 'completed' } }, { sourceEventSeqs: [0] })) + .toThrow(/cannot carry sourceEventSeqs/) + }) + + it('rejects surfaceOp on a non-surface event', async () => { + const { ctx } = await setup() + const session = ctx.sessions.create() + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-return + expect(() => (session.append as any)('turn/end', { turn: 1, reason: { kind: 'completed' } }, { surfaceOp: 'append' })) + .toThrow(/cannot carry surfaceOp/) + }) }) diff --git a/packages/session-persistence-sqlite/src/index.ts b/packages/session-persistence-sqlite/src/index.ts index 3475f8b4dd..de3e96fda5 100644 --- a/packages/session-persistence-sqlite/src/index.ts +++ b/packages/session-persistence-sqlite/src/index.ts @@ -26,7 +26,7 @@ import { mkdir } from 'node:fs/promises' import { dirname, resolve } from 'node:path' import { SessionPersistence } from '@deepseek-ai/dsh-session-persistence' import { isJsonValue, interruptedTurnClosers } from '@deepseek-ai/dsh-session' -import type { Session, SessionEvent, SessionId, SessionMeta, SessionSummary } from '@deepseek-ai/dsh-session' +import type { Session, SessionEvent, SurfaceEventType, SessionId, SessionMeta, SessionSummary } from '@deepseek-ai/dsh-session' import { openDatabase, rowToMeta, scanRows, type EventRow, type SessionRow, } from './schema.ts' @@ -39,9 +39,10 @@ export { SCHEMA_VERSION } from './schema.ts' * events, events written before surface support). */ function surfaceBindings(event: SessionEvent): [string | null, string | null] { + const se = event as SessionEvent return [ - event.sourceEventSeqs ? JSON.stringify(event.sourceEventSeqs) : null, - event.surfaceOp !== undefined ? JSON.stringify(event.surfaceOp) : null, + se.sourceEventSeqs ? JSON.stringify(se.sourceEventSeqs) : null, + se.surfaceOp !== undefined ? JSON.stringify(se.surfaceOp) : null, ] } diff --git a/packages/session-persistence-sqlite/src/schema.ts b/packages/session-persistence-sqlite/src/schema.ts index 68ff4905ae..974fd77778 100644 --- a/packages/session-persistence-sqlite/src/schema.ts +++ b/packages/session-persistence-sqlite/src/schema.ts @@ -126,19 +126,19 @@ export function rowToMeta(row: SessionRow): SessionMeta { /** Reconstruct a {@link SessionEvent} from an `events` row (parses `data`). */ export function rowToEvent(row: EventRow): SessionEvent { - const event = { + // Surface-metadata fields are conditional on the event type in the type + // system; spread them so each variant gets only the fields it declares. + const surfaceFields = { + ...row.source_event_seqs !== null ? { sourceEventSeqs: JSON.parse(row.source_event_seqs) as number[] } : {}, + ...row.surface_op !== null ? { surfaceOp: JSON.parse(row.surface_op) as SurfaceOp } : {}, + } + return { type: row.type as SessionEvent['type'], seq: row.seq, time: row.time, data: JSON.parse(row.data) as SessionEvent['data'], + ...surfaceFields, } as SessionEvent - if (row.source_event_seqs !== null) { - event.sourceEventSeqs = JSON.parse(row.source_event_seqs) as number[] - } - if (row.surface_op !== null) { - event.surfaceOp = JSON.parse(row.surface_op) as SurfaceOp - } - return event } /** diff --git a/packages/session-persistence-sqlite/tests/sqlite.spec.ts b/packages/session-persistence-sqlite/tests/sqlite.spec.ts index 61b166f257..252e9c4660 100644 --- a/packages/session-persistence-sqlite/tests/sqlite.spec.ts +++ b/packages/session-persistence-sqlite/tests/sqlite.spec.ts @@ -5,7 +5,7 @@ import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import SessionStore, { SessionId } from '@deepseek-ai/dsh-session' -import type { Session, SessionEvent, SessionMeta } from '@deepseek-ai/dsh-session' +import type { Session, SessionEvent, SurfaceEvent, SurfaceEventType, SessionMeta } from '@deepseek-ai/dsh-session' import SessionPersistenceSqlite, { SCHEMA_VERSION } from '@deepseek-ai/dsh-session-persistence-sqlite' import { openDatabase, rowToEvent, scanRows, type EventRow } from '../src/schema.ts' import { runPersistenceContract, meta, oneTurnLog } from '../../session-persistence/tests/contract.ts' @@ -800,8 +800,8 @@ describe('surface field round-trip', () => { surface_op: JSON.stringify('append'), } const event = rowToEvent(row) - expect(event.sourceEventSeqs).toEqual([3, 5]) - expect(event.surfaceOp).toBe('append') + expect((event as SurfaceEvent).sourceEventSeqs).toEqual([3, 5]) + expect((event as SurfaceEvent).surfaceOp).toBe('append') }) it('rowToEvent handles replace surfaceOp object', () => { @@ -812,8 +812,8 @@ describe('surface field round-trip', () => { surface_op: JSON.stringify({ op: 'replace', start: 0, end: 1 }), } const event = rowToEvent(row) - expect(event.sourceEventSeqs).toEqual([0, 1]) - expect(event.surfaceOp).toEqual({ op: 'replace', start: 0, end: 1 }) + expect((event as SurfaceEvent).sourceEventSeqs).toEqual([0, 1]) + expect((event as SurfaceEvent).surfaceOp).toEqual({ op: 'replace', start: 0, end: 1 }) }) it('scanRows with surface columns reconstructs events with surface fields', () => { @@ -827,9 +827,9 @@ describe('surface field round-trip', () => { ] const { preserved } = scanRows(rows) expect(preserved).toHaveLength(2) - expect(preserved[0]!.surfaceOp).toEqual({ op: 'replace', start: 0, end: 0 }) - expect(preserved[0]!.sourceEventSeqs).toBeUndefined() - expect(preserved[1]!.surfaceOp).toBeUndefined() + expect((preserved[0]! as SurfaceEvent).surfaceOp).toEqual({ op: 'replace', start: 0, end: 0 }) + expect((preserved[0]! as SurfaceEvent).sourceEventSeqs).toBeUndefined() + expect((preserved[1] as SessionEvent).surfaceOp).toBeUndefined() }) it('append and load round-trips surface fields through SQLite', async () => { @@ -845,11 +845,11 @@ describe('surface field round-trip', () => { const loaded = await ctx.sessionPersistence.load(SessionId('roundtrip-surface')) expect(loaded.events).toHaveLength(4) const um = loaded.events[1]! - expect(um.surfaceOp).toBe('append') - expect(um.sourceEventSeqs).toBeUndefined() + expect((um as SurfaceEvent).surfaceOp).toBe('append') + expect((um as SurfaceEvent).sourceEventSeqs).toBeUndefined() const am = loaded.events[2]! - expect(am.surfaceOp).toBe('append') - expect(am.sourceEventSeqs).toEqual([0]) + expect((am as SurfaceEvent).surfaceOp).toBe('append') + expect((am as SurfaceEvent).sourceEventSeqs).toEqual([0]) await fiber.dispose() }) @@ -863,8 +863,8 @@ describe('surface field round-trip', () => { session.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) await ctx.parallel('session/flush', session) const loaded = await ctx.sessionPersistence.load(SessionId('surface-noseq')) - expect(loaded.events[1]!.surfaceOp).toBe('append') - expect(loaded.events[1]!.sourceEventSeqs).toBeUndefined() + expect((loaded.events[1]! as SurfaceEvent).surfaceOp).toBe('append') + expect((loaded.events[1]! as SurfaceEvent).sourceEventSeqs).toBeUndefined() await fiber.dispose() }) }) diff --git a/packages/session/src/index.ts b/packages/session/src/index.ts index d0d4d44aed..e04bda5e7d 100644 --- a/packages/session/src/index.ts +++ b/packages/session/src/index.ts @@ -10,7 +10,7 @@ import { Context, Service } from 'cordis' import { isAbsolute } from 'node:path' import type { ContentBlock, Message, MessageSource } from '@deepseek-ai/dsh-llm' import { SessionId } from './types.ts' -import type { CreateSessionOptions, SessionEvent, SessionEventMap, SessionEventType, SessionHeader, SurfaceAppendOpts } from './types.ts' +import type { CreateSessionOptions, SessionEvent, SessionEventMap, SessionEventType, SessionHeader, SurfaceAppendOpts, SurfaceEventType } from './types.ts' import { isJsonValue } from './json.ts' import { SurfaceManager } from './surface.ts' @@ -18,6 +18,7 @@ export * from './types.ts' export { isJsonValue } from './json.ts' export { interruptedTurnClosers } from './repair.ts' export type { SurfaceNode } from './surface.ts' +export { isSurfaceEvent } from './surface.ts' declare module 'cordis' { interface Context { @@ -138,7 +139,9 @@ export class Session { * @param data - The event payload; must be JSON-serializable. * @param opts - Optional surface metadata: `surfaceOp` controls how the * event enters the surface linked list; `sourceEventSeqs` records - * provenance (the seq numbers of events this one derives from). + * provenance (the seq numbers of events this one derives from). Only + * accepted for {@link SurfaceEventType} events — the compiler rejects + * surface opts for non-surface types like `turn/start` or `assistant/chunk`. * @throws if `data` is not losslessly JSON-serializable (BigInt, function, * symbol, undefined, non-finite number, circular ref, or an exotic object * like Map/Set/Date). The event log is the durable source of truth, so this @@ -147,7 +150,11 @@ export class Session { * throw surfaces at the buggy caller's append site, not asynchronously in a * backend flush. */ - append(type: T, data: SessionEventMap[T], opts?: SurfaceAppendOpts): SessionEvent { + append( + type: T, + data: SessionEventMap[T], + ...opts: T extends SurfaceEventType ? [opts?: SurfaceAppendOpts] : [] + ): SessionEvent { if (!isJsonValue(data)) { throw new Error(`session event "${type}" carries non-JSON-serializable data`) } @@ -164,18 +171,25 @@ export class Session { // Surface metadata is snapshot separately: sourceEventSeqs (number[] — // primitives, so array spread is a complete copy) and surfaceOp (a string // primitive, or cloned if it's a replace object). + const surfaceOpts: SurfaceAppendOpts | undefined = opts[0] + // Build the event shape with conditional surface fields via spreading. + // The result is cast through `unknown` because the conditional spreads + // produce an intersection type that the assignability checker can't + // narrow to a specific discriminated-union member when T is generic. + // This is a safe internal boundary: data was validated above, and + // surface metadata was snapshot from primitive/clone-safe values. const event = { type, seq: this.log.length, time: Date.now(), data: structuredClone(data), - ...opts?.sourceEventSeqs !== undefined ? { sourceEventSeqs: [...opts.sourceEventSeqs] } : {}, - ...opts?.surfaceOp !== undefined ? { - surfaceOp: typeof opts.surfaceOp === 'string' ? opts.surfaceOp : structuredClone(opts.surfaceOp), + ...surfaceOpts?.sourceEventSeqs !== undefined ? { sourceEventSeqs: [...surfaceOpts.sourceEventSeqs] } : {}, + ...surfaceOpts?.surfaceOp !== undefined ? { + surfaceOp: typeof surfaceOpts.surfaceOp === 'string' ? surfaceOpts.surfaceOp : structuredClone(surfaceOpts.surfaceOp), } : {}, - } as SessionEvent - this.log.push(event) - this.onAppend?.(event) + } as unknown as SessionEvent + this.log.push(event as unknown as SessionEvent) + this.onAppend?.(event as unknown as SessionEvent) return event } @@ -207,6 +221,10 @@ export class Session { // index by construction. The non-null assertion expresses that invariant. // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const msg = this._deriveOneMessage(this.log[node.seq]!) + // isSurfaceEvent guarantees only the five surface-eligible types + // enter the surface, and all five produce messages → msg is never + // null. Defensive guard retained for interface contract clarity. + /* v8 ignore next */ if (msg) messages.push(msg) } return messages diff --git a/packages/session/src/surface.ts b/packages/session/src/surface.ts index a09dbd001f..eaa8baeb4a 100644 --- a/packages/session/src/surface.ts +++ b/packages/session/src/surface.ts @@ -7,7 +7,33 @@ * @module @deepseek-ai/dsh-session/surface */ -import type { SessionEvent, SurfaceOp } from './types.ts' +import type { SessionEvent, SurfaceEvent, SurfaceEventType, SurfaceOp } from './types.ts' + +/** + * The set of event type strings that are eligible for the surface linked list. + * Mirrors the {@link SurfaceEventType} union; kept as a runtime set so the + * type guard can check membership without a chain of string comparisons. + */ +const SURFACE_EVENT_TYPES = new Set([ + 'user/message', + 'assistant/message', + 'tool/result', + 'context/message', + 'steering/message', +]) + +/** + * Narrow a {@link SessionEvent} to {@link SurfaceEvent}: checks that the + * event's `type` is surface-eligible AND that `surfaceOp` is present. + * The narrowed type has mandatory {@link SurfaceOp}. + */ +export function isSurfaceEvent(event: SessionEvent): event is SurfaceEvent { + if (!SURFACE_EVENT_TYPES.has(event.type)) return false + // surfaceOp is optional on SessionEvent (even for surface-eligible types) + // but mandatory on SurfaceEvent — this check is the narrowing gate. + if ((event as SessionEvent).surfaceOp === undefined) return false + return true +} /** One node in the surface linked list. */ export interface SurfaceNode { @@ -57,11 +83,12 @@ export class SurfaceManager { get hasSurface(): boolean { if (this._nodes.length > 0) return true // Never processed anything — scan the whole log. - if (this._lastProcessedSeq === -1) return this.log.some(e => e.surfaceOp !== undefined) + if (this._lastProcessedSeq === -1) return this.log.some(e => isSurfaceEvent(e)) // Processed up to _lastProcessedSeq without finding surface nodes; check // only new events. for (let i = this._lastProcessedSeq + 1; i < this.log.length; i++) { - if (this.log[i]?.surfaceOp !== undefined) return true + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + if (isSurfaceEvent(this.log[i]!)) return true } return false } @@ -72,8 +99,13 @@ export class SurfaceManager { */ private _processDelta(): void { for (let i = this._lastProcessedSeq + 1; i < this.log.length; i++) { - const event = this.log[i] - if (event === undefined || event.surfaceOp === undefined) continue + // Index is bounded by i < this.log.length — never undefined. + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const event = this.log[i]! + // isSurfaceEvent checks event.type first (is it a surface-eligible type?) + // then checks that surfaceOp is present. Only after both pass do we treat + // it as a SurfaceEvent with mandatory surfaceOp. + if (!isSurfaceEvent(event)) continue if (event.surfaceOp === 'append') { const tail = this._nodes.length > 0 ? this._nodes[this._nodes.length - 1] : undefined diff --git a/packages/session/src/types.ts b/packages/session/src/types.ts index d7e9b19259..be3ed61af2 100644 --- a/packages/session/src/types.ts +++ b/packages/session/src/types.ts @@ -175,8 +175,31 @@ export interface SessionEventMap { export type SessionEventType = keyof SessionEventMap /** - * How a session event entered the surface linked list. Absent for non-surface - * events (boundaries, chunks, usage, errors). + * The subset of {@link SessionEventType} values whose events produce LLM + * messages and are eligible to appear on the surface linked list. Only these + * event types may carry {@link SurfaceOp} and {@link SessionEvent.sourceEventSeqs}. + */ +export type SurfaceEventType = + | 'user/message' + | 'assistant/message' + | 'tool/result' + | 'context/message' + | 'steering/message' + +/** + * A {@link SessionEvent} that is **on** the surface linked list — its + * `surfaceOp` is guaranteed present (mandatory), narrowed from a + * surface-eligible {@link SessionEvent} by checking both `type` and + * `surfaceOp` at runtime. + * + * Use the `isSurfaceEvent` type guard (in `surface.ts`) to narrow a + * `SessionEvent` to this type. + */ +export type SurfaceEvent = SessionEvent & { surfaceOp: SurfaceOp } + +/** + * How a session event entered the surface linked list. Only valid on + * {@link SurfaceEventType} events. * * - `'append'`: added to the tail — normal path for user/assistant/tool/context * messages. @@ -196,6 +219,9 @@ export type SurfaceOp = * `sourceEventSeqs` records the seq numbers of events that are provenance * sources of this one (e.g. the `assistant/chunk` seqs behind an * `assistant/message`, or the shadowed nodes behind a compaction replacement). + * + * Only accepted for {@link SurfaceEventType} events — non-surface event types + * (`turn/start`, `assistant/chunk`, `error`, …) cannot carry surface metadata. */ export interface SurfaceAppendOpts { surfaceOp?: SurfaceOp @@ -207,6 +233,13 @@ export interface SurfaceAppendOpts { * * A proper discriminated union over `type` (not independent `type`/`data` * unions), so `switch (event.type)` narrows `event.data` without casts. + * + * The {@link sourceEventSeqs} and {@link surfaceOp} fields are conditional: + * they only exist on {@link SurfaceEventType} variants (`user/message`, + * `assistant/message`, `tool/result`, `context/message`, `steering/message`). + * Non-surface events (boundary markers, chunks, usage, errors) never carry + * surface metadata — the compiler enforces this at `Session.append()` + * call sites. */ export type SessionEvent = { [K in SessionEventType]: { @@ -216,6 +249,7 @@ export type SessionEvent = { /** Unix epoch milliseconds. */ time: number data: SessionEventMap[K] + } & (K extends SurfaceEventType ? { /** * Seq numbers of events that are provenance sources of this event * (e.g. the `assistant/chunk` seqs that built an `assistant/message`, @@ -224,5 +258,5 @@ export type SessionEvent = { sourceEventSeqs?: number[] /** How this event entered the surface; absent for non-surface events. */ surfaceOp?: SurfaceOp - } + } : object) }[T] diff --git a/packages/session/tests/repair.spec.ts b/packages/session/tests/repair.spec.ts index cf6fb2b51c..0b7dee9f2b 100644 --- a/packages/session/tests/repair.spec.ts +++ b/packages/session/tests/repair.spec.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest' import { CallId } from '@deepseek-ai/dsh-llm' import { interruptedTurnClosers } from '../src/index.ts' -import type { SessionEvent } from '../src/index.ts' +import type { SessionEvent, SurfaceEvent } from '../src/index.ts' /** * Unit coverage for the crash-recovery closer synthesis. The persistence @@ -135,8 +135,8 @@ describe('interruptedTurnClosers', () => { const closers = interruptedTurnClosers(events) expect(closers.map(e => e.type)).toEqual(['tool/result', 'step/end', 'turn/end']) const result = closers[0]! - expect(result.surfaceOp).toBe('append') - expect(result.sourceEventSeqs).toEqual([3]) + expect((result as SurfaceEvent).surfaceOp).toBe('append') + expect((result as SurfaceEvent).sourceEventSeqs).toEqual([3]) }) it('handles tool/call without a matching assistant/message entry gracefully', () => { diff --git a/packages/session/tests/surface.spec.ts b/packages/session/tests/surface.spec.ts index d8f503c5e5..51ddcbcd44 100644 --- a/packages/session/tests/surface.spec.ts +++ b/packages/session/tests/surface.spec.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest' -import type { SessionEvent } from '@deepseek-ai/dsh-session' +import type { SessionEvent, SurfaceEvent, SurfaceEventType } from '@deepseek-ai/dsh-session' import { Session, SessionId } from '@deepseek-ai/dsh-session' import { CallId } from '@deepseek-ai/dsh-llm' @@ -188,7 +188,7 @@ describe('SurfaceManager', () => { // Mutate caller's array after append. sources.push(30) sources[0] = 99 - const logged = s.events[0]! + const logged = s.events[0]! as SurfaceEvent expect(logged.sourceEventSeqs).toEqual([10, 20]) }) @@ -219,7 +219,7 @@ describe('SurfaceManager', () => { s.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 's' }] }, { surfaceOp: op, sourceEventSeqs: [0] }) // Mutate caller's object after append. op.start = 99 - const logged = s.events[1]! + const logged = s.events[1]! as SurfaceEvent expect(logged.surfaceOp).toEqual({ op: 'replace', start: 0, end: 0 }) }) }) @@ -288,8 +288,8 @@ describe('Session.append surface opts', () => { expect(event.sourceEventSeqs).toEqual([3, 5, 7]) expect(event.surfaceOp).toBe('append') // The logged event matches the returned event. - expect(s.events[0]!.sourceEventSeqs).toEqual([3, 5, 7]) - expect(s.events[0]!.surfaceOp).toBe('append') + expect((s.events[0]! as SurfaceEvent).sourceEventSeqs).toEqual([3, 5, 7]) + expect((s.events[0]! as SurfaceEvent).surfaceOp).toBe('append') }) it('deriveMessages skips surface nodes whose event type is not message-producing', () => { @@ -299,7 +299,7 @@ describe('Session.append surface opts', () => { const seed: SessionEvent[] = [ { type: 'turn/start', seq: 0, time: 1, data: { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } } }, { type: 'step/start', seq: 1, time: 2, data: { turn: 1, step: 1 } }, - { type: 'usage', seq: 2, time: 3, data: { turn: 1, step: 1, usage: { inputTokens: 0, outputTokens: 0 } }, surfaceOp: 'append' as const }, + { type: 'usage', seq: 2, time: 3, data: { turn: 1, step: 1, usage: { inputTokens: 0, outputTokens: 0 } }, surfaceOp: 'append' as const } as SessionEvent, { type: 'step/end', seq: 3, time: 4, data: { turn: 1, step: 1 } }, { type: 'turn/end', seq: 4, time: 5, data: { turn: 1, reason: { kind: 'completed' } } }, ] @@ -311,8 +311,8 @@ describe('Session.append surface opts', () => { it('append without surface opts produces an event without surface fields', () => { const s = new Session(SessionId('noopts')) s.append('user/message', { content: [{ type: 'text', text: 'x' }], source: { kind: 'user' } }) - expect(s.events[0]!.sourceEventSeqs).toBeUndefined() - expect(s.events[0]!.surfaceOp).toBeUndefined() + expect((s.events[0] as SessionEvent).sourceEventSeqs).toBeUndefined() + expect((s.events[0] as SessionEvent).surfaceOp).toBeUndefined() }) it('surfaceOp primitives are not cloned (they are immutable)', () => { From e45053f0f51d8da29a534c5f52ff9bb42d1e2565 Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Mon, 22 Jun 2026 14:52:30 +0800 Subject: [PATCH 04/17] =?UTF-8?q?feat(compact):=20compaction=20capability?= =?UTF-8?q?=20seam=20=E2=80=94=20abstract=20CompactService=20interface?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the @deepseek-ai/dsh-compact interface package: the abstract CompactService (ctx.compact) with compactIfNeeded / compactRegion, the compact/* session-event types via SessionEventMap declaration merging, and the capability-seam RFC. Wires the package into the three root tsconfigs and the cordis catalog. A backend implementation lands separately. --- docs/cordis-catalog/events-and-services.md | 20 +++- docs/module-graph.md | 3 + docs/rfc/README.md | 1 + .../2026-06-18-compaction-capability-seam.md | 57 ++++++++++ packages/compact/compact/README.md | 52 +++++++++ packages/compact/compact/package.json | 32 ++++++ packages/compact/compact/src/index.ts | 102 ++++++++++++++++++ packages/compact/compact/src/types.ts | 57 ++++++++++ .../compact/compact/tests/compact.spec.ts | 78 ++++++++++++++ packages/compact/compact/tsconfig.json | 14 +++ pnpm-lock.yaml | 12 +++ tsconfig.base.json | 1 + tsconfig.build.json | 1 + tsconfig.typecheck.json | 1 + 14 files changed, 430 insertions(+), 1 deletion(-) create mode 100644 docs/rfc/proposed/feature/2026-06-18-compaction-capability-seam.md create mode 100644 packages/compact/compact/README.md create mode 100644 packages/compact/compact/package.json create mode 100644 packages/compact/compact/src/index.ts create mode 100644 packages/compact/compact/src/types.ts create mode 100644 packages/compact/compact/tests/compact.spec.ts create mode 100644 packages/compact/compact/tsconfig.json diff --git a/docs/cordis-catalog/events-and-services.md b/docs/cordis-catalog/events-and-services.md index 1ee86d6a5e..400187dafa 100644 --- a/docs/cordis-catalog/events-and-services.md +++ b/docs/cordis-catalog/events-and-services.md @@ -279,7 +279,7 @@ Source: [`packages/core/tools/src/index.ts:43`](../../packages/core/tools/src/in ## Services -The 8 `ctx.` services the harness provides. An abstract seam (e.g. `ctx.bash`) is implemented by a separate package; the interface is what consumers code against. +The 9 `ctx.` services the harness provides. An abstract seam (e.g. `ctx.bash`) is implemented by a separate package; the interface is what consumers code against. ### `ctx.agentLoop` — `AgentLoop` @@ -339,6 +339,24 @@ Types: [BashExecRequest](../core-data-structures/bash.md) · [BashExecSpec](../c Source: [`packages/bash/bash/src/index.ts:59`](../../packages/bash/bash/src/index.ts) +### `ctx.compact` — `CompactService` (abstract seam) + +Abstract compaction service. Subclass implement the two abstract methods, and load the subclass as a plugin — it registers as `ctx.compact` (one implementation per context; loading a second throws, which is cordis' standard duplicate-service behavior). + +Both core methods are abstract: the contract states WHAT compaction does, while the entire strategy — token estimation, retention policy, event sequencing, summarization — is a HOW decision owned by the implementation. + +Implementations MUST honor: + +- **Surface contract**: a successful compaction shadows the compacted surface nodes with a SINGLE replacement node carrying the summary. Because `SurfaceEventType` is a closed union, that node is a `user/message` with `surfaceOp: { op:'replace', start, end }`; the `compact/*` events are log-only (lock + provenance). +- **Blocking**: no compaction begins while another is in progress for the same session. The recommended mechanism is the log-recorded lock — append `compact/start` before the slow work and `compact/end` after (even on failure) — so the lock is visible to replay and crash recovery. + +```ts cordis-catalog +abstract compactIfNeeded( session: Session, systemPrompt?: string, model?: string, ): Promise +abstract compactRegion( session: Session, start: number, end: number, model: string, ): Promise +``` + +Source: [`packages/compact/compact/src/index.ts:57`](../../packages/compact/compact/src/index.ts) + ### `ctx.llm` — `LlmService` The abstract `llm` service: an adapter registry plus a streaming model-call surface, interceptable via the `llm/stream` waterfall. diff --git a/docs/module-graph.md b/docs/module-graph.md index 9efe6e9669..d633fb7787 100644 --- a/docs/module-graph.md +++ b/docs/module-graph.md @@ -18,6 +18,8 @@ graph TD agent --> brand agent --> llm agent --> session + compact --> llm + compact --> session llm-replay --> llm llm-replay --> session session-persistence --> session @@ -78,6 +80,7 @@ graph TD | `session` | `brand`, `llm` | | `system-prompt` | `llm` | | `agent` | `brand`, `llm`, `session` | +| `compact` | `llm`, `session` | | `llm-replay` | `llm`, `session` | | `session-persistence` | `session` | | `invariants` | `agent`, `llm`, `session` | diff --git a/docs/rfc/README.md b/docs/rfc/README.md index dd34e66816..05029e1ac8 100644 --- a/docs/rfc/README.md +++ b/docs/rfc/README.md @@ -44,6 +44,7 @@ Do NOT write one for a mechanical or local choice (a variable name, a one-file r | [Agent Client Protocol (ACP) support for external editors](proposed/feature/2026-06-14-acp-agent-client-protocol.md) | 2026-06-14 | | [Multiplex concurrent ACP sessions over one connection](proposed/feature/2026-06-14-acp-multi-session.md) | 2026-06-14 | | [Optional Code Mode — model writes TypeScript against an SDK of all tools](proposed/feature/2026-06-15-optional-code-mode.md) | 2026-06-15 | +| [Compaction as a capability seam (abstract contract + basic backend)](proposed/feature/2026-06-18-compaction-capability-seam.md) | 2026-06-18 | ### Simplification diff --git a/docs/rfc/proposed/feature/2026-06-18-compaction-capability-seam.md b/docs/rfc/proposed/feature/2026-06-18-compaction-capability-seam.md new file mode 100644 index 0000000000..3725a3db84 --- /dev/null +++ b/docs/rfc/proposed/feature/2026-06-18-compaction-capability-seam.md @@ -0,0 +1,57 @@ +# RFC: Compaction as a capability seam (abstract contract + basic backend) + +Status: proposed (2026-06-18) + +## Context + +A long-running agent conversation grows without bound. As the event log accumulates turns, the derived message history eventually approaches the model's context window — the model then truncates mid-response (`max-tokens`) or degrades. **Compaction** is the mitigation: replace a run of older history with a concise summary, keeping recent context intact. + +The [session surface](../../implemented/architecture/2026-06-18-session-surface.md) was built as the foundation for exactly this — a linked list over the event log with a `surfaceOp: { op: 'replace', start, end }` operation purpose-built to shadow a range of nodes and insert a replacement, with `sourceEventSeqs` recording provenance so the decision replays deterministically. What remained was the plugin that *decides what to compact and produces the summary*. + +Two forces shape the design. First, compaction is **swappable**: token counting can be a char/4 heuristic or a real tokenizer, and summarization can be a model call, a template, or a remote service — these vary independently of *when* and *which range* to compact. Second, a later commit (`ce43c25`) closed `SurfaceEventType` to five event types (`user/message`, `assistant/message`, `tool/result`, `context/message`, `steering/message`); only those may carry `surfaceOp`. A bespoke `compaction/*` event therefore **cannot** itself appear on the surface — the compiler rejects `surfaceOp` on it and the invariants plugin rejects it at runtime. + +## Decision + +### Compaction is a capability seam, split interface / implementation + +Per the [capability-seams RFC](../../implemented/architecture/2026-06-13-capability-seams.md), compaction ships as separate packages so the contract, the algorithm, and (later) the consumer surface evolve independently: + +1. **Interface** — `@deepseek-ai/dsh-compact`: an abstract `CompactService` owning the `ctx.compact` key, the `CompactionResult` vocabulary, and the `compact/*` session events. It declares `compactIfNeeded()` and `compactRegion()` as **abstract** — the contract states *what* compaction does, not *how*. +2. **Implementation** — `@deepseek-ai/dsh-compact-basic`: a concrete `BasicCompactService` that owns the entire algorithm — token estimation (char/4 + per-block overhead), the tail→head retention walk, summarization via `ctx.llm.generate()`, the surface replacement, the lock, and the `agent/request` auto-compaction listener. A tokenizer-based or template-based backend is a sibling package (or a subclass overriding the two protected estimation/summarization hooks). +3. **Consumer** — deferred. A `/compact` tool and slash command will `inject: ['compact']` and call the contract; they are intentionally out of scope here so the seam settles first. + +### The contract depends on `dsh-session` and `dsh-llm` — a deliberate deviation + +The capability-seams RFC states the interface package "depends only on cordis" (true of `dsh-bash`, whose vocabulary is self-contained). Compaction **cannot** honor that: its verbs are defined *over* a `Session` (`compactRegion(session, start, end)`) and its output *is* the content vocabulary (`CompactionResult.summary: ContentBlock[]`). There is no way to express the contract without naming `Session`/`SessionEvent` (from `dsh-session`) and `ContentBlock` (from `dsh-llm`). + +This is not a coupling smell — it is the contract's domain. The "only cordis" guidance was always shorthand for "the interface depends only on what the contract genuinely names, and never on an implementation." `dsh-session` and `dsh-llm` are themselves interface/vocabulary packages, not implementations; `dsh-compact` still imports no backend. The seam's real invariant — *consumers and implementations evolve independently behind an abstract service* — holds intact. We record the deviation here so a future reader doesn't mistake it for an accident or "fix" it by smuggling `Session` behind an opaque handle. + +### Abstract `compactIfNeeded` / `compactRegion`, algorithm in the backend + +An earlier draft put the full algorithm (the retention walk, token-summing, text extraction) as concrete methods on the interface, with only `estimateContentTokens()` and `summarize()` abstract. That recouples the contract to one strategy: a backend that wants a different retention policy (e.g. turn-count instead of token-budget) or a different event-sequencing would have to fight inherited concrete code. Making both core methods abstract puts every *how* decision in the backend, where it belongs, and keeps the interface a pure statement of *what*. The backend remains internally factored — `estimateContentTokens()` and `summarize()` are `protected` hooks a sub-backend can override without reimplementing the walk — but that factoring is the backend's private concern, not the contract's. + +### Surface replacement: `compact/*` events are log-only; one `user/message` carries the summary + +Because `SurfaceEventType` is closed, the summary cannot ride on a `compact/*` event. The backend instead appends a **single `user/message`** with `surfaceOp: { op: 'replace', start, end }` whose `content` is the summary `ContentBlock[]` and whose `sourceEventSeqs` covers the shadowed nodes *and* the bookkeeping events. The `compact/*` events are pure log records (lock + provenance), never on the surface: + +``` +compact/start → log-only. Acquires the lock. +[summarize older range via the backend] +compact/summary → log-only. Provenance: summary, range, shadowed seqs, token count. +compact/end → log-only. Releases the lock. +user/message → surfaceOp { op:'replace', start, end }. THE surface mutation. + deriveMessages() renders it as a user-role message. +``` + +`deriveMessages()` then yields `[summary_as_user_message, ...retained_nodes]`. An alternative — extending `SurfaceEventType` to admit a `compact/*` type — was rejected: the closed union is a deliberate safety boundary (only message-producing events reach the model), and a summary genuinely *is* user-role context, so reusing `user/message` is honest rather than a workaround. + +### Blocking via a log-recorded lock, not a mutex + +Compaction must be serialized: no second compaction starts before the first finishes, and no ordinary events interleave the slow summarization. Rather than an in-memory mutex (invisible to replay, lost on crash), the lock **is** the log: `compactRegion` refuses to start if the last `compact/start` has no matching `compact/end` after it. `compact/start` is appended first (fast, synchronous), the slow model call runs, then `compact/end` is appended — in a `catch` that records the error, so a failed summarization can never wedge the lock. Because the backend runs compaction synchronously inside the `agent/request` waterfall, the loop is single-threaded for that window; the lock additionally gives observability and lets a persistence backend detect an orphaned `compact/start` on reload. + +## Consequences + +- **New packages**: `packages/compact/compact` (interface) and a sibling `compact-basic` (backend) under `packages/compact/`, wired into the three root tsconfigs. The consumer tier is deferred. +- **`SessionEventMap`** gains `compact/start` / `compact/summary` / `compact/end` by declaration merging (merge-extensible); `SurfaceEventType` is **not** touched. These are session events, not cordis `Events`, so the event-taxonomy gate needs no entry. +- **No changes** to `dsh-session`, `dsh-invariants`, or `dsh-agent-loop`: the surface replace op, the surface-metadata runtime guard, and the `agent/request` waterfall all already exist. Compaction is a pure plugin on documented seams. +- The capability-seams convention gains a second reference beyond bash, and a documented case where "interface depends only on cordis" relaxes to "depends only on interface/vocabulary packages the contract genuinely names." On acceptance, [AGENTS.md](../../../../AGENTS.md) § Conventions and [architecture.md](../../../architecture.md) § "Capability seams" should note this relaxation. diff --git a/packages/compact/compact/README.md b/packages/compact/compact/README.md new file mode 100644 index 0000000000..e98cdd52a6 --- /dev/null +++ b/packages/compact/compact/README.md @@ -0,0 +1,52 @@ +# @deepseek-ai/dsh-compact + +The **compaction seam**: an abstract `CompactService` (`ctx.compact`) defining WHAT compaction does — decide when history is too large and summarize an older range into a single surface node — without saying HOW. + +This package is the interface tier of the compaction capability, split so each concern evolves (and swaps) independently: + +| Package | Role | +|---|---| +| `@deepseek-ai/dsh-compact` (this) | the interface: abstract service + `compact/*` events + `CompactionResult` | +| `@deepseek-ai/dsh-compact-basic` | a backend: char/4 estimation + token-budget retention + `llm.stream()` summarization | +| `@deepseek-ai/dsh-tool-compact` (deferred) | the model-facing `/compact` tool over `ctx.compact` | + +Unlike the bash seam, this interface depends on `@deepseek-ai/dsh-session` and `@deepseek-ai/dsh-llm` — the contract's verbs are defined over a `Session` and its output is the `ContentBlock` vocabulary, so they cannot be expressed without naming those packages. That deviation from the "interface depends only on cordis" guidance is intentional and recorded in the [compaction capability-seam RFC](../../../docs/rfc/proposed/feature/2026-06-18-compaction-capability-seam.md). + +## Service API (`ctx.compact`) + +Both methods are **abstract** — the backend owns the entire strategy (token estimation, retention policy, event sequencing, summarization). + +| Member | Semantics | +|---|---| +| `compactIfNeeded(session, systemPrompt?, model?)` | Estimate the history size; if over the backend's threshold, compact an older range via `compactRegion`, keeping recent context intact. Returns the `CompactionResult`, or `null` if nothing needed compacting. | +| `compactRegion(session, start, end, model)` | Forcibly summarize surface nodes `[start, end]` (inclusive seqs) into a single replacement node. **Throws** if a compaction is already in progress, if `start`/`end` aren't surface nodes, or if `start > end`. | + +## Surface contract + +`SurfaceEventType` is a closed union — only `user/message`, `assistant/message`, `tool/result`, `context/message`, and `steering/message` may carry `surfaceOp`. A `compact/*` event therefore **cannot** appear on the surface. A successful compaction instead: + +1. appends `compact/start` (log-only) — acquires the lock, +2. summarizes the range, +3. appends `compact/summary` (log-only) — provenance: summary, range, shadowed seqs, token count, +4. appends `compact/end` (log-only) — releases the lock, +5. appends a single `user/message` with `surfaceOp: { op: 'replace', start, end }` carrying the summary — **the only surface mutation**. + +`deriveMessages()` then renders the summary as a user-role message followed by the retained nodes. The shadowed events remain in the raw log, so replay is deterministic. + +## Blocking + +Compaction is serialized via a log-recorded lock: `compactRegion` refuses to start if the last `compact/start` has no matching `compact/end` after it. The lock is the log (not an in-memory mutex), so it survives replay and a persistence backend can detect an orphaned `compact/start` on reload. `compact/end` is appended even when summarization throws, so a failure can never wedge the lock. + +## Events + +The `compact/*` events extend `SessionEventMap` (merge-extensible) via declaration merging — they are session events, not cordis `Events`: + +| Event | Payload | On surface? | +|---|---|---| +| `compact/start` | `{ turn }` | no (log-only) | +| `compact/summary` | `{ summary, compactedRange, compactedEventSeqs, tokenCount }` | no (log-only) | +| `compact/end` | `{ turn, error? }` | no (log-only) | + +## Implementing a backend + +Subclass `CompactService`, implement `compactIfNeeded` and `compactRegion`, and load the subclass as a plugin — it registers as `ctx.compact`. See `@deepseek-ai/dsh-compact-basic` for the reference implementation. diff --git a/packages/compact/compact/package.json b/packages/compact/compact/package.json new file mode 100644 index 0000000000..7658bd8f40 --- /dev/null +++ b/packages/compact/compact/package.json @@ -0,0 +1,32 @@ +{ + "name": "@deepseek-ai/dsh-compact", + "description": "Abstract compaction service seam (ctx.compact) for the DeepSeek Harness", + "version": "0.0.1", + "private": true, + "type": "module", + "main": "lib/index.js", + "types": "lib/index.d.ts", + "exports": { + ".": { + "types": "./lib/index.d.ts", + "default": "./lib/index.js" + }, + "./src/*": "./src/*", + "./package.json": "./package.json" + }, + "files": [ + "lib", + "src" + ], + "license": "BSD-3-Clause", + "peerDependencies": { + "@deepseek-ai/dsh-llm": "^0.0.1", + "@deepseek-ai/dsh-session": "^0.0.1", + "cordis": "^4.0.0-rc.6" + }, + "devDependencies": { + "@deepseek-ai/dsh-llm": "workspace:^", + "@deepseek-ai/dsh-session": "workspace:^", + "cordis": "^4.0.0-rc.6" + } +} diff --git a/packages/compact/compact/src/index.ts b/packages/compact/compact/src/index.ts new file mode 100644 index 0000000000..acd10d4c90 --- /dev/null +++ b/packages/compact/compact/src/index.ts @@ -0,0 +1,102 @@ +/** + * The compaction service seam (`ctx.compact`): an abstract service defining + * WHAT compaction does — decide when to compact, summarize a range of + * conversation history into a single surface node — without saying HOW. + * + * Implementations subclass {@link CompactService}, implement + * {@link CompactService.compactIfNeeded} and {@link CompactService.compactRegion}, + * and load as a plugin — registering as `ctx.compact` (one implementation per + * context). `@deepseek-ai/dsh-compact-basic` (char/4 estimation + token-budget + * retention + `ctx.llm.stream()` summarization) is the first. A tokenizer- or + * template-based backend swaps in without touching consumers. + * + * The split follows the capability-seams RFC — interface (this) / + * implementation (`dsh-compact-basic`) / consumer (a `/compact` tool, deferred) + * — modeled on the bash trio. Unlike `dsh-bash`, this interface necessarily + * depends on `dsh-session` and `dsh-llm`: the contract's verbs are defined over + * a `Session` and its output is the `ContentBlock` vocabulary. That deviation + * from the "interface depends only on cordis" guidance is intentional and + * recorded in the [compaction capability-seam RFC](../../../../docs/rfc/proposed/feature/2026-06-18-compaction-capability-seam.md). + * + * @module @deepseek-ai/dsh-compact + */ + +import { Context, Service } from 'cordis' +import type { Session } from '@deepseek-ai/dsh-session' +import type { CompactionResult } from './types.ts' + +export type { CompactionResult } from './types.ts' + +declare module 'cordis' { + interface Context { + compact: CompactService + } +} + +/** + * Abstract compaction service. Subclass implement the two abstract methods, + * and load the subclass as a plugin — it registers as `ctx.compact` (one + * implementation per context; loading a second throws, which is cordis' + * standard duplicate-service behavior). + * + * Both core methods are abstract: the contract states WHAT compaction does, + * while the entire strategy — token estimation, retention policy, event + * sequencing, summarization — is a HOW decision owned by the implementation. + * + * Implementations MUST honor: + * - **Surface contract**: a successful compaction shadows the compacted surface + * nodes with a SINGLE replacement node carrying the summary. Because + * `SurfaceEventType` is a closed union, that node is a `user/message` with + * `surfaceOp: { op:'replace', start, end }`; the `compact/*` events are + * log-only (lock + provenance). + * - **Blocking**: no compaction begins while another is in progress for the + * same session. The recommended mechanism is the log-recorded lock — append + * `compact/start` before the slow work and `compact/end` after (even on + * failure) — so the lock is visible to replay and crash recovery. + */ +export abstract class CompactService extends Service { + constructor(ctx: Context) { + super(ctx, 'compact') + } + + /** + * Check token pressure and compact if the conversation is too large. + * + * Estimates the current history size (optionally including a system prompt), + * and if it exceeds the backend's threshold, compacts an older range via + * {@link compactRegion}, keeping recent context intact. + * + * @param session - the session whose surface may be compacted. + * @param systemPrompt - optional system prompt, counted toward the estimate. + * @param model - optional summarization model (falls back to backend config). + * @returns the compaction result, or `null` if no compaction was needed. + */ + abstract compactIfNeeded( + session: Session, + systemPrompt?: string, + model?: string, + ): Promise + + /** + * Forcibly compact a range of surface nodes into a single summary node. + * + * `start` and `end` are inclusive seqs of surface nodes to shadow; the backend + * summarizes their content and appends a replacement surface node. Used by the + * (future) `/compact` tool and internally by {@link compactIfNeeded}. + * + * @param session - the session whose surface is mutated. + * @param start - inclusive seq of the first surface node to compact. + * @param end - inclusive seq of the last surface node to compact. + * @param model - summarization model. + * @throws if compaction is already in progress, or if `start`/`end` are not + * valid surface nodes, or if `start > end`. + */ + abstract compactRegion( + session: Session, + start: number, + end: number, + model: string, + ): Promise +} + +export default CompactService diff --git a/packages/compact/compact/src/types.ts b/packages/compact/compact/src/types.ts new file mode 100644 index 0000000000..08b37ef4c6 --- /dev/null +++ b/packages/compact/compact/src/types.ts @@ -0,0 +1,57 @@ +/** + * Compaction vocabulary: the result type and the `compact/*` session events. + * + * Extends {@link SessionEventMap} with `compact/*` event types via declaration + * merging. {@link SurfaceEventType} is deliberately NOT extended — `compact/*` + * events are log-only markers (lock + provenance); only the five + * surface-eligible types can carry `surfaceOp`. The actual surface mutation is + * performed by a separate `user/message` event carrying the summary (see the + * [compaction capability-seam RFC](../../../../docs/rfc/proposed/feature/2026-06-18-compaction-capability-seam.md)). + * + * Configuration lives in the backend, not here: the contract states WHAT + * compaction produces, while every tunable (context window, thresholds, + * retention budget) is a HOW decision owned by the implementation. + * + * @module @deepseek-ai/dsh-compact/types + */ + +import type { ContentBlock } from '@deepseek-ai/dsh-llm' + +declare module '@deepseek-ai/dsh-session' { + interface SessionEventMap { + /** Marks the start of a compaction — log-only, holds the lock until `compact/end`. */ + 'compact/start': { turn: number } + /** + * Provenance record of a completed summarization — log-only, no surfaceOp. + * The summary content is in `data.summary`; the actual surface replacement + * is performed by a subsequent `user/message` event that shadows the + * compacted range. + */ + 'compact/summary': { + summary: ContentBlock[] + compactedRange: { startSeq: number; endSeq: number } + compactedEventSeqs: number[] + tokenCount: number + } + /** Marks the end of a compaction — log-only, releases the lock. `error` set if summarization failed. */ + 'compact/end': { turn: number; error?: string } + } +} + +/** Result of a successful compaction operation. */ +export interface CompactionResult { + /** The seq of the appended `compact/start` event. */ + startSeq: number + /** The seq of the appended `compact/summary` event. */ + summarySeq: number + /** The seq of the appended `compact/end` event. */ + endSeq: number + /** The summary content blocks produced by the backend. */ + summary: ContentBlock[] + /** The seq range that was shadowed [start, end] inclusive. */ + shadowedRange: { start: number; end: number } + /** The seq numbers of all shadowed surface nodes. */ + shadowedSeqs: number[] + /** Estimated token count of the shadowed content. */ + compactedTokenCount: number +} diff --git a/packages/compact/compact/tests/compact.spec.ts b/packages/compact/compact/tests/compact.spec.ts new file mode 100644 index 0000000000..c4f0c0f838 --- /dev/null +++ b/packages/compact/compact/tests/compact.spec.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from 'vitest' +import { Context } from 'cordis' +import { CompactService } from '@deepseek-ai/dsh-compact' +import type { CompactionResult } from '@deepseek-ai/dsh-compact' +import { Session, SessionId } from '@deepseek-ai/dsh-session' + +/** + * A trivial concrete CompactService implementing the abstract contract. The + * interface package owns no algorithm — these tests exercise the seam itself: + * service registration, the abstract method shape, and the `compact/*` event + * declaration merge. + */ +class StubCompactService extends CompactService { + override async compactIfNeeded(_session: Session, _systemPrompt?: string, _model?: string): Promise { + return null + } + + override async compactRegion(session: Session, start: number, end: number, _model: string): Promise { + // Minimal stub honoring the lock + log-only event contract. + const startEvent = session.append('compact/start', { turn: 0 }) + const summaryEvent = session.append('compact/summary', { + summary: [{ type: 'text', text: 'stub' }], + compactedRange: { startSeq: start, endSeq: end }, + compactedEventSeqs: [], + tokenCount: 0, + }) + const endEvent = session.append('compact/end', { turn: 0 }) + return { + startSeq: startEvent.seq, + summarySeq: summaryEvent.seq, + endSeq: endEvent.seq, + summary: [{ type: 'text', text: 'stub' }], + shadowedRange: { start, end }, + shadowedSeqs: [], + compactedTokenCount: 0, + } + } +} + +describe('CompactService seam', () => { + it('registers as ctx.compact', () => { + const ctx = new Context() + void new StubCompactService(ctx) + expect(ctx.compact).toBeDefined() + expect(ctx.compact).toBeInstanceOf(StubCompactService) + }) + + it('disposing the fiber unregisters ctx.compact (HMR safety)', async () => { + const ctx = new Context() + const fiber = await ctx.plugin(StubCompactService) + expect(ctx.compact).toBeInstanceOf(StubCompactService) + await fiber.dispose() + expect(ctx.compact).toBeUndefined() + }) + + it('exposes the abstract contract methods', async () => { + const ctx = new Context() + const svc = new StubCompactService(ctx) + expect(await svc.compactIfNeeded(new Session(SessionId('s')))).toBeNull() + }) + + it('compact/* events merge into SessionEventMap and are log-only', async () => { + const ctx = new Context() + const svc = new StubCompactService(ctx) + const session = new Session(SessionId('s')) + + const result = await svc.compactRegion(session, 0, 0, 'm') + + const startEvent = session.events.find(e => e.type === 'compact/start') + expect(startEvent).toBeDefined() + // Log-only: the compiler rejects surfaceOp on compact/* (not a SurfaceEventType); + // verify the runtime value is absent. + const raw = startEvent as unknown as { surfaceOp?: unknown } + expect(raw.surfaceOp).toBeUndefined() + expect(result.summarySeq).toBeGreaterThan(result.startSeq) + expect(result.endSeq).toBeGreaterThan(result.summarySeq) + }) +}) diff --git a/packages/compact/compact/tsconfig.json b/packages/compact/compact/tsconfig.json new file mode 100644 index 0000000000..a16d13abac --- /dev/null +++ b/packages/compact/compact/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "lib" + }, + "include": ["src"], + "references": [ + { "path": "../../../vendor/cosmokit" }, + { "path": "../../../vendor/cordis" }, + { "path": "../../llm/llm" }, + { "path": "../../core/session" } + ] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3bca330308..4c4f33e613 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -118,6 +118,18 @@ importers: specifier: ^4.0.0-rc.6 version: 4.0.0-rc.6(@cordisjs/plugin-include@1.0.4)(@cordisjs/plugin-loader@1.0.0-rc.4) + packages/compact/compact: + devDependencies: + '@deepseek-ai/dsh-llm': + specifier: workspace:^ + version: link:../../llm/llm + '@deepseek-ai/dsh-session': + specifier: workspace:^ + version: link:../../core/session + cordis: + specifier: ^4.0.0-rc.6 + version: 4.0.0-rc.6(@cordisjs/plugin-include@1.0.4)(@cordisjs/plugin-loader@1.0.0-rc.4) + packages/core/agent: devDependencies: '@deepseek-ai/dsh-brand': diff --git a/tsconfig.base.json b/tsconfig.base.json index 8e2070fe28..9fa9c52369 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -43,6 +43,7 @@ "./packages/core/*/src", "./packages/llm/*/src", "./packages/bash/*/src", + "./packages/compact/*/src", "./packages/session-persistence/*/src", "./packages/ui/*/src", "./packages/util/*/src", diff --git a/tsconfig.build.json b/tsconfig.build.json index 27a17a3f17..d73da9ffc0 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -22,6 +22,7 @@ { "path": "./packages/core/agent-loop" }, { "path": "./packages/core/agent-core" }, { "path": "./packages/bash/bash" }, + { "path": "./packages/compact/compact" }, { "path": "./packages/llm/llm-deepseek" }, { "path": "./packages/llm/llm-pi-ai" }, { "path": "./packages/bash/bash-local" }, diff --git a/tsconfig.typecheck.json b/tsconfig.typecheck.json index 54769bdbc0..93008005d8 100644 --- a/tsconfig.typecheck.json +++ b/tsconfig.typecheck.json @@ -20,6 +20,7 @@ "./packages/core/*/src", "./packages/llm/*/src", "./packages/bash/*/src", + "./packages/compact/*/src", "./packages/session-persistence/*/src", "./packages/ui/*/src", "./packages/util/*/src", From 30805e1983d76d0846fcbdb49d0c2b06edb1a7b9 Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Tue, 23 Jun 2026 09:48:15 +0800 Subject: [PATCH 05/17] fix(sqlite): bump SCHEMA_VERSION to 3 for the new surface columns --- .../session-persistence-sqlite/src/schema.ts | 2 +- .../session-persistence-sqlite/tests/sqlite.spec.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/session-persistence/session-persistence-sqlite/src/schema.ts b/packages/session-persistence/session-persistence-sqlite/src/schema.ts index 6012916457..b747b7900c 100644 --- a/packages/session-persistence/session-persistence-sqlite/src/schema.ts +++ b/packages/session-persistence/session-persistence-sqlite/src/schema.ts @@ -15,7 +15,7 @@ import type { SessionEvent, SessionId, SessionHeader, SurfaceOp } from '@deepsee * layout; orthogonal to a session's own `version` (which versions the EVENT * vocabulary, stored per session in the `sessions` row). */ -export const SCHEMA_VERSION = 2 +export const SCHEMA_VERSION = 3 /** * A row of the `sessions` table — the out-of-log metadata ({@link SessionHeader}). diff --git a/packages/session-persistence/session-persistence-sqlite/tests/sqlite.spec.ts b/packages/session-persistence/session-persistence-sqlite/tests/sqlite.spec.ts index 86d72275f6..0b8b83cd4e 100644 --- a/packages/session-persistence/session-persistence-sqlite/tests/sqlite.spec.ts +++ b/packages/session-persistence/session-persistence-sqlite/tests/sqlite.spec.ts @@ -315,7 +315,7 @@ describe('SessionPersistenceSqlite: durability and crash semantics', () => { }) it('exposes the schema version constant', () => { - expect(SCHEMA_VERSION).toBe(2) + expect(SCHEMA_VERSION).toBe(3) }) }) From 09d497d5c5b2e7d321cded16110dd34bae2fff96 Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Tue, 23 Jun 2026 09:54:58 +0800 Subject: [PATCH 06/17] docs(rfc): clarify compaction rides the replace op on an existing event type --- docs/rfc/implemented/architecture/2026-06-18-session-surface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/rfc/implemented/architecture/2026-06-18-session-surface.md b/docs/rfc/implemented/architecture/2026-06-18-session-surface.md index 42c3872db6..644d526373 100644 --- a/docs/rfc/implemented/architecture/2026-06-18-session-surface.md +++ b/docs/rfc/implemented/architecture/2026-06-18-session-surface.md @@ -60,4 +60,4 @@ The dev-mode invariants plugin validates: `sourceEventSeqs` references (non-empt - **`packages/session-persistence/session-persistence-jsonl`**: No changes required. - **`packages/session-persistence/session-persistence`**: Abstract interface unchanged. -The surface is the foundation for future compaction: a compaction plugin appends a new event (e.g., `compaction/marker`, added to `SessionEventMap` via declaration merging) with `surfaceOp: { op: 'replace', start, end }` and `sourceEventSeqs` covering the shadowed nodes. Replay preserves the compaction decision deterministically. +The surface is the foundation for future history manipulation. A compaction or tool-result-prune plugin appends one of the existing message-producing event types (a `user/message` carrying the summary, say) with `surfaceOp: { op: 'replace', start, end }` and `sourceEventSeqs` covering the shadowed nodes — the new node takes the range's place on the surface while the plugin's own trace events (e.g. `compaction/start`, `compaction/end`) stay off it. Replay preserves the decision deterministically. From 6089e226bc2423229eea942ae7e253cc847799f8 Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Tue, 23 Jun 2026 13:05:59 +0800 Subject: [PATCH 07/17] refactor(session): make surface the sole derivation path, drop legacy fallback --- docs/cordis-catalog/events-and-services.md | 2 +- .../2026-06-18-session-surface.md | 2 +- packages/core/session/README.md | 6 +- packages/core/session/src/index.ts | 59 +++++++++---------- packages/core/session/src/surface.ts | 14 ----- packages/core/session/src/types.ts | 12 ++-- packages/core/session/tests/session.spec.ts | 32 +++++----- packages/core/session/tests/surface.spec.ts | 47 +-------------- .../tests/jsonl.spec.ts | 8 +-- .../tests/coordinator-contract.ts | 14 ++--- .../invariants/tests/invariants.spec.ts | 36 +++++------ 11 files changed, 87 insertions(+), 145 deletions(-) diff --git a/docs/cordis-catalog/events-and-services.md b/docs/cordis-catalog/events-and-services.md index 1ee86d6a5e..f8c6b12c0d 100644 --- a/docs/cordis-catalog/events-and-services.md +++ b/docs/cordis-catalog/events-and-services.md @@ -390,7 +390,7 @@ get(id: SessionId): Session | undefined list(): Session[] ``` -Source: [`packages/core/session/src/index.ts:303`](../../packages/core/session/src/index.ts) +Source: [`packages/core/session/src/index.ts:300`](../../packages/core/session/src/index.ts) ### `ctx.systemPrompt` — `SystemPrompt` diff --git a/docs/rfc/implemented/architecture/2026-06-18-session-surface.md b/docs/rfc/implemented/architecture/2026-06-18-session-surface.md index 644d526373..ae2475c786 100644 --- a/docs/rfc/implemented/architecture/2026-06-18-session-surface.md +++ b/docs/rfc/implemented/architecture/2026-06-18-session-surface.md @@ -53,7 +53,7 @@ The dev-mode invariants plugin validates: `sourceEventSeqs` references (non-empt ## Consequences -- **`packages/core/session`**: New `surface.ts` (`SurfaceManager`), new types (`SurfaceOp`, `SurfaceAppendOpts`), new fields on `SessionEvent`, modified `append()` (third optional `SurfaceAppendOpts` param), refactored `deriveMessages()` (surface path + legacy fallback), surface-aware `repair.ts`. +- **`packages/core/session`**: New `surface.ts` (`SurfaceManager`), new types (`SurfaceOp`, `SurfaceIntent`), new fields on `SessionEvent`, modified `append()` (third required `SurfaceIntent` param), refactored `deriveMessages()` (surface path + legacy fallback), surface-aware `repair.ts`. - **`packages/core/agent-loop`**: All surface-capable appends pass surface opts. Chunk seqs are collected for `assistant/message` provenance; `tool/call` seqs are captured for `tool/result` provenance. - **`packages/session-persistence/session-persistence-sqlite`**: Two new nullable TEXT columns (`source_event_seqs`, `surface_op`) on the `events` table; `SCHEMA_VERSION` bumped (bump-and-reject, no migration). - **`packages/support/invariants`**: Surface-related validation rules. diff --git a/packages/core/session/README.md b/packages/core/session/README.md index 206022e306..08323eff0c 100644 --- a/packages/core/session/README.md +++ b/packages/core/session/README.md @@ -34,8 +34,8 @@ Creates and holds event-sourced `Session` instances. Persistence is intentionall Plain class (not a Cordis Service). Create via `ctx.sessions.create()`. -- `session.append(type, data, opts?): SessionEvent` — synchronous, never blocks on I/O. **Throws** if `data` is not losslessly JSON-serializable (BigInt, function, symbol, undefined, non-finite number, circular ref, or an exotic object like Map/Set/Date) — the event log is the durable source of truth, so this invariant is enforced at the source (exported as `isJsonValue` for backends to reuse on their replay/fork entry points). An optional third parameter `opts: SurfaceAppendOpts` carries surface metadata: `surfaceOp` controls how the event enters the surface linked list, and `sourceEventSeqs` records provenance (the seq numbers of events this one derives from). -- `session.deriveMessages(): Message[]` — derive the LLM message history. If any event in the log carries `surfaceOp`, derivation walks the surface linked list (skipping non-surface events). Otherwise, falls back to a linear scan of the raw log (legacy sessions without surface markers). +- `session.append(type, data, opts?): SessionEvent` — synchronous, never blocks on I/O. **Throws** if `data` is not losslessly JSON-serializable (BigInt, function, symbol, undefined, non-finite number, circular ref, or an exotic object like Map/Set/Date) — the event log is the durable source of truth, so this invariant is enforced at the source (exported as `isJsonValue` for backends to reuse on their replay/fork entry points). A third parameter `opts: SurfaceIntent` carries surface metadata: `surfaceOp` controls how the event enters the surface linked list, and `sourceEventSeqs` records provenance (the seq numbers of events this one derives from). It is **required** for the five `SurfaceEventType` events (every message-producing event must declare how it joins the surface) and rejected by the compiler for non-surface types. +- `session.deriveMessages(): Message[]` — derive the LLM message history by walking the surface linked list (skipping non-surface events like chunks and boundaries; a `replace` shadows the nodes it covers). The surface is the single source of derived history — there is no raw-log fallback. - `session.surface: SurfaceManager` — the derived surface, lazily rebuilt from `surfaceOp` markers in the log. Processes only new events (delta) on each access — the log is append-only, so prior events never change. - `session.events`, `session.seq`, `session.id` - `session.header: SessionHeader` — immutable creation metadata (`version`, `id`, `createdAt`, optional `cwd`/`parentSession`). Kept out of the event log (a storage concern, not replayable state); a minimal header (stamped with the current `SESSION_FORMAT_VERSION`) is synthesized for bare `Session` construction. @@ -43,7 +43,7 @@ Plain class (not a Cordis Service). Create via `ctx.sessions.create()`. ### Surface types - `SurfaceOp` — how a surface node entered the linked list: `'append'` (normal tail append) or `{ op: 'replace', start, end }` (replace nodes from `start` through `end` inclusive — both must be valid surface node seqs; `start === end` replaces a single node). Used by compaction to shadow old nodes without deleting them. -- `SurfaceAppendOpts` — `{ surfaceOp?: SurfaceOp; sourceEventSeqs?: number[] }`, the optional third parameter to `session.append()`. +- `SurfaceIntent` — `{ surfaceOp: SurfaceOp; sourceEventSeqs?: number[] }`, the required third parameter to `session.append()` for surface-eligible types. - `SurfaceNode` — `{ seq: number; prev: number | null; next: number | null }`, one node in the surface linked list. ### Session event vocabulary (`types.ts`) diff --git a/packages/core/session/src/index.ts b/packages/core/session/src/index.ts index 23e65cebda..5df4db58e9 100644 --- a/packages/core/session/src/index.ts +++ b/packages/core/session/src/index.ts @@ -10,7 +10,7 @@ import { Context, Service } from 'cordis' import { isAbsolute } from 'node:path' import type { ContentBlock, Message, MessageSource } from '@deepseek-ai/dsh-llm' import { SESSION_FORMAT_VERSION, SessionId } from './types.ts' -import type { CreateSessionOptions, SessionEvent, SessionEventMap, SessionEventType, SessionHeader, SurfaceAppendOpts, SurfaceEventType } from './types.ts' +import type { CreateSessionOptions, SessionEvent, SessionEventMap, SessionEventType, SessionHeader, SurfaceIntent, SurfaceEventType } from './types.ts' import { isJsonValue } from './json.ts' import { SurfaceManager } from './surface.ts' @@ -149,11 +149,13 @@ export class Session { * * @param type - The event type (key of {@link SessionEventMap}). * @param data - The event payload; must be JSON-serializable. - * @param opts - Optional surface metadata: `surfaceOp` controls how the - * event enters the surface linked list; `sourceEventSeqs` records - * provenance (the seq numbers of events this one derives from). Only - * accepted for {@link SurfaceEventType} events — the compiler rejects - * surface opts for non-surface types like `turn/start` or `assistant/chunk`. + * @param opts - Surface metadata: `surfaceOp` controls how the event enters + * the surface linked list; `sourceEventSeqs` records provenance (the seq + * numbers of events this one derives from). REQUIRED for + * {@link SurfaceEventType} events (every message-producing event must + * declare how it joins the surface, the sole source of derived history) and + * rejected by the compiler for non-surface types like `turn/start` or + * `assistant/chunk`. * @throws if `data` is not losslessly JSON-serializable (BigInt, function, * symbol, undefined, non-finite number, circular ref, or an exotic object * like Map/Set/Date). The event log is the durable source of truth, so this @@ -165,7 +167,7 @@ export class Session { append( type: T, data: SessionEventMap[T], - ...opts: T extends SurfaceEventType ? [opts?: SurfaceAppendOpts] : [] + ...opts: T extends SurfaceEventType ? [opts: SurfaceIntent] : [] ): SessionEvent { if (!isJsonValue(data)) { throw new Error(`session event "${type}" carries non-JSON-serializable data`) @@ -183,7 +185,7 @@ export class Session { // Surface metadata is snapshot separately: sourceEventSeqs (number[] — // primitives, so array spread is a complete copy) and surfaceOp (a string // primitive, or cloned if it's a replace object). - const surfaceOpts: SurfaceAppendOpts | undefined = opts[0] + const surfaceOpts: SurfaceIntent | undefined = opts[0] // Build the event shape with conditional surface fields via spreading. // The result is cast through `unknown` because the conditional spreads // produce an intersection type that the assignability checker can't @@ -206,9 +208,12 @@ export class Session { } /** - * Derive the LLM message history from the session surface (when surface - * markers exist) or from a linear scan of the raw event log (legacy sessions - * without surface markers). + * Derive the LLM message history by walking the session surface — the linked + * list of message-producing events maintained by `surfaceOp` markers. The + * surface is the single source of derived history: every message-producing + * append records its `surfaceOp`, so a raw event with no marker (a chunk, a + * turn boundary) is correctly absent, and a compaction `replace` deletes the + * shadowed nodes from the derivation. * * - `user/message` → user message * - `assistant/message` → assistant message (chunks are skipped — they are @@ -229,33 +234,24 @@ export class Session { * negligible next to a model call. */ deriveMessages(): Message[] { - if (this.surface.hasSurface) { - const messages: Message[] = [] - for (const node of this.surface.nodes) { - // Surface nodes are built from this.log — node.seq is always a valid - // index by construction. The non-null assertion expresses that invariant. - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const msg = this._deriveOneMessage(this.log[node.seq]!) - // A surface node is one of the five message-producing types, but an - // empty-content assistant/message (a max-tokens step that hosts only - // usage) derives to null and must not enter the transcript. - if (msg) messages.push(msg) - } - return messages - } - // Legacy path: linear scan for sessions without surface markers. const messages: Message[] = [] - for (const event of this.log) { - const msg = this._deriveOneMessage(event) + for (const node of this.surface.nodes) { + // Surface nodes are built from this.log — node.seq is always a valid + // index by construction. The non-null assertion expresses that invariant. + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const msg = this._deriveOneMessage(this.log[node.seq]!) + // A surface node is one of the five message-producing types, but an + // empty-content assistant/message (a max-tokens step that hosts only + // usage) derives to null and must not enter the transcript. if (msg) messages.push(msg) } return messages } /** - * Derive a single LLM message from one event, or null if the event type - * does not produce a message. Extracted so both the surface path and the - * legacy linear-scan path share the same derivation rules. + * Derive a single LLM message from one surface event, or null if it produces + * no message (an empty-content assistant/message that exists only to host + * usage). */ private _deriveOneMessage(event: SessionEvent): Message | null { // Intentionally non-exhaustive: only message-producing events derive @@ -288,6 +284,7 @@ export class Session { const { content, source } = event.data return { role: 'user', content: renderTagged('steering', structuredClone(content), source) } } + /* v8 ignore next 2 -- unreachable: only surface nodes (the 5 message-producing types) reach here */ default: return null } diff --git a/packages/core/session/src/surface.ts b/packages/core/session/src/surface.ts index eaa8baeb4a..7ed1743af3 100644 --- a/packages/core/session/src/surface.ts +++ b/packages/core/session/src/surface.ts @@ -79,20 +79,6 @@ export class SurfaceManager { return this._nodes } - /** Whether any event in the log carries `surfaceOp` markers. */ - get hasSurface(): boolean { - if (this._nodes.length > 0) return true - // Never processed anything — scan the whole log. - if (this._lastProcessedSeq === -1) return this.log.some(e => isSurfaceEvent(e)) - // Processed up to _lastProcessedSeq without finding surface nodes; check - // only new events. - for (let i = this._lastProcessedSeq + 1; i < this.log.length; i++) { - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - if (isSurfaceEvent(this.log[i]!)) return true - } - return false - } - /** * Process events from `_lastProcessedSeq + 1` through the end of the log, * folding new surface markers into the existing linked list. diff --git a/packages/core/session/src/types.ts b/packages/core/session/src/types.ts index 1fcd150d37..44e8e35fdf 100644 --- a/packages/core/session/src/types.ts +++ b/packages/core/session/src/types.ts @@ -222,17 +222,19 @@ export type SurfaceOp = | { op: 'replace'; start: number; end: number } /** - * Optional surface metadata passed to {@link Session.append}. + * Surface metadata passed to {@link Session.append}. * `surfaceOp` controls how the event enters the surface linked list; * `sourceEventSeqs` records the seq numbers of events that are provenance * sources of this one (e.g. the `assistant/chunk` seqs behind an * `assistant/message`, or the shadowed nodes behind a compaction replacement). * - * Only accepted for {@link SurfaceEventType} events — non-surface event types - * (`turn/start`, `assistant/chunk`, `error`, …) cannot carry surface metadata. + * Required for {@link SurfaceEventType} events — every message-producing event + * MUST declare how it enters the surface, because the surface is the sole + * source of derived history. Non-surface event types (`turn/start`, + * `assistant/chunk`, `error`, …) cannot carry surface metadata. */ -export interface SurfaceAppendOpts { - surfaceOp?: SurfaceOp +export interface SurfaceIntent { + surfaceOp: SurfaceOp sourceEventSeqs?: number[] } diff --git a/packages/core/session/tests/session.spec.ts b/packages/core/session/tests/session.spec.ts index f095bdfb40..46f96742cc 100644 --- a/packages/core/session/tests/session.spec.ts +++ b/packages/core/session/tests/session.spec.ts @@ -7,7 +7,7 @@ describe('Session', () => { it('derives message history from the event log', () => { const session = new Session(SessionId('s1')) session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) - session.append('user/message', { content: [{ type: 'text', text: 'hello' }], source: { kind: 'user' } }) + session.append('user/message', { content: [{ type: 'text', text: 'hello' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) session.append('assistant/chunk', { turn: 1, step: 1, chunk: { type: 'text-delta', index: 0, text: 'hi' } }) session.append('assistant/message', { turn: 1, step: 1, @@ -15,8 +15,8 @@ describe('Session', () => { { type: 'text', text: 'let me check' }, { type: 'tool-call', id: CallId('c1'), name: 'echo', arguments: '{}' }, ], - }) - session.append('tool/result', { turn: 1, step: 1, callId: CallId('c1'), content: [{ type: 'text', text: 'ok' }], isError: false }) + }, { surfaceOp: 'append' }) + session.append('tool/result', { turn: 1, step: 1, callId: CallId('c1'), content: [{ type: 'text', text: 'ok' }], isError: false }, { surfaceOp: 'append' }) session.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) const messages = session.deriveMessages() @@ -44,12 +44,12 @@ describe('Session', () => { session.append('context/message', { content: [{ type: 'text', text: 'file changed: a.ts' }], source: { kind: 'plugin', plugin: 'watcher' }, - }) + }, { surfaceOp: 'append' }) session.append('steering/message', { turn: 1, content: [{ type: 'text', text: 'focus on tests' }], source: { kind: 'user' }, - }) + }, { surfaceOp: 'append' }) const [contextMessage, steeringMessage] = session.deriveMessages() expect(contextMessage!.role).toBe('user') @@ -60,8 +60,8 @@ describe('Session', () => { it('replays identically from a seeded event log', () => { const original = new Session(SessionId('s3')) - original.append('user/message', { content: [{ type: 'text', text: 'q' }], source: { kind: 'user' } }) - original.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 'a' }] }) + original.append('user/message', { content: [{ type: 'text', text: 'q' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) + original.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 'a' }] }, { surfaceOp: 'append' }) const replayed = new Session(SessionId('s3-replay'), [...original.events]) expect(replayed.deriveMessages()).toEqual(original.deriveMessages()) @@ -70,11 +70,11 @@ describe('Session', () => { it('isolates the log from mutation through a derived message (append-only contract)', () => { const session = new Session(SessionId('s4')) - session.append('user/message', { content: [{ type: 'text', text: 'original' }], source: { kind: 'user' } }) + session.append('user/message', { content: [{ type: 'text', text: 'original' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) session.append('tool/result', { turn: 1, step: 1, callId: CallId('c1'), content: [{ type: 'text', text: 'tool out' }], isError: false, - }) + }, { surfaceOp: 'append' }) const before = structuredClone(session.events) // A request middleware / adapter mutates the messages it was handed. @@ -95,7 +95,7 @@ describe('Session', () => { it('rejects non-JSON-serializable event data at the source (incl. sparse arrays)', () => { const session = new Session(SessionId('s5')) - const bad = (extra: unknown) => () => session.append('user/message', { content: [{ type: 'text', text: 'x' }], source: { kind: 'user' }, extra } as never) + const bad = (extra: unknown) => () => session.append('user/message', { content: [{ type: 'text', text: 'x' }], source: { kind: 'user' }, extra } as never, { surfaceOp: 'append' }) expect(bad(1n)).toThrow(/non-JSON-serializable/) expect(bad(() => 0)).toThrow(/non-JSON-serializable/) expect(bad(Symbol('s'))).toThrow(/non-JSON-serializable/) @@ -122,7 +122,7 @@ describe('Session', () => { it('accepts dense arrays and nested plain objects', () => { const session = new Session(SessionId('s6')) - expect(() => session.append('user/message', { content: [{ type: 'text', text: 'x' }], source: { kind: 'user' }, extra: [1, 2, [3, { a: null, b: true }]] } as never)).not.toThrow() + expect(() => session.append('user/message', { content: [{ type: 'text', text: 'x' }], source: { kind: 'user' }, extra: [1, 2, [3, { a: null, b: true }]] } as never, { surfaceOp: 'append' })).not.toThrow() expect(session.events).toHaveLength(1) }) @@ -174,7 +174,7 @@ describe('Session', () => { it('snapshots append data: mutating the passed object after append does not affect session.events', () => { const session = new Session(SessionId('append-snapshot')) const data = { content: [{ type: 'text' as const, text: 'original' }], source: { kind: 'user' as const } } - const event = session.append('user/message', data) + const event = session.append('user/message', data, { surfaceOp: 'append' }) // Mutate the caller's object after append returns. A shared reference would // make session.events diverge from the value that passed validation. data.content[0]!.text = 'HACKED' @@ -201,7 +201,7 @@ describe('SessionStore', () => { const session = ctx.sessions.create() expect(created).toEqual([session]) - session.append('user/message', { content: [{ type: 'text', text: 'x' }], source: { kind: 'user' } }) + session.append('user/message', { content: [{ type: 'text', text: 'x' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) expect(events).toHaveLength(1) expect(events[0]![0]).toBe(session) expect(events[0]![1].type).toBe('user/message') @@ -216,7 +216,7 @@ describe('SessionStore', () => { const a = ctx.sessions.create(SessionId('fixed')) expect(() => ctx.sessions.create(SessionId('fixed'))).toThrow('already exists') - a.append('user/message', { content: [{ type: 'text', text: 'q' }], source: { kind: 'user' } }) + a.append('user/message', { content: [{ type: 'text', text: 'q' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) const forked = ctx.sessions.create(SessionId('fork'), { seed: [...a.events] }) expect(forked.deriveMessages()).toEqual(a.deriveMessages()) }) @@ -309,7 +309,7 @@ describe('SessionStore', () => { await fiber.dispose() expect(ctx.sessions.get(SessionId('scoped'))).toBeUndefined() - session.append('user/message', { content: [{ type: 'text', text: 'late' }], source: { kind: 'user' } }) + session.append('user/message', { content: [{ type: 'text', text: 'late' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) expect(observed).toBe(0) }) @@ -332,7 +332,7 @@ describe('SessionStore', () => { ctx.on('session/event', (_session, event) => void events.push(event)) const session = ctx.sessions.create(SessionId('fixed')) expect(ctx.sessions.get(SessionId('fixed'))).toBe(session) - session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }) + session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) expect(events).toHaveLength(1) }) }) diff --git a/packages/core/session/tests/surface.spec.ts b/packages/core/session/tests/surface.spec.ts index b4e1022494..a7f4c13dad 100644 --- a/packages/core/session/tests/surface.spec.ts +++ b/packages/core/session/tests/surface.spec.ts @@ -28,38 +28,6 @@ describe('SurfaceManager', () => { expect(nodes[1]!.next).toBeNull() }) - it('hasSurface returns false when no events have surfaceOp', () => { - const s = new Session(SessionId('nosurface')) - s.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) - s.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }) - s.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) - expect(s.surface.hasSurface).toBe(false) - }) - - it('hasSurface returns true when any event has surfaceOp', () => { - const s = surfaceSession() - expect(s.surface.hasSurface).toBe(true) - }) - - it('hasSurface detects surface markers that arrive after initial processing', () => { - // Start with no surface markers. Access nodes first to set _lastProcessedSeq - // (via delta processing), keeping _nodes empty. Then append a mix of non-surface - // and surface events, and verify hasSurface detects via the delta-only check. - const s = new Session(SessionId('late')) - s.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) - s.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }) - // Access nodes to trigger processing: sets _lastProcessedSeq = 1, _nodes = []. - expect(s.surface.nodes.length).toBe(0) - // Append non-surface events first (exercises the loop-continue branch), then - // a surface event (exercises the return-true branch). - s.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) - s.append('turn/start', { turn: 2, trigger: { kind: 'continuation' } }) - s.append('assistant/message', { turn: 2, step: 1, content: [] }, { surfaceOp: 'append' }) - // hasSurface checks only new seqs [2, 3, 4]; skips 2 and 3 (non-surface), - // finds surfaceOp on seq 4 and returns true. - expect(s.surface.hasSurface).toBe(true) - }) - it('invalidate resets to full rebuild', () => { const s = surfaceSession() expect(s.surface.nodes.length).toBe(2) @@ -76,7 +44,6 @@ describe('SurfaceManager', () => { s.append('step/end', { turn: 1, step: 1 }) s.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) expect(s.surface.nodes.length).toBe(0) - expect(s.surface.hasSurface).toBe(false) // deriveMessages returns empty array expect(s.deriveMessages()).toEqual([]) }) @@ -235,16 +202,6 @@ describe('deriveMessages with surface', () => { expect(messages[1]!.content[0]).toMatchObject({ type: 'text', text: 'hi' }) }) - it('falls back to linear scan when no surface markers exist', () => { - const s = new Session(SessionId('legacy')) - s.append('user/message', { content: [{ type: 'text', text: 'q' }], source: { kind: 'user' } }) - s.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 'a' }] }) - const messages = s.deriveMessages() - expect(messages).toHaveLength(2) - expect(messages[0]!.role).toBe('user') - expect(messages[1]!.role).toBe('assistant') - }) - it('surface path skips non-surface events (chunks, boundaries)', () => { const s = new Session(SessionId('filter')) s.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) @@ -308,9 +265,9 @@ describe('Session.append surface opts', () => { expect(s.deriveMessages()).toHaveLength(0) }) - it('append without surface opts produces an event without surface fields', () => { + it('a non-surface event carries no surface fields', () => { const s = new Session(SessionId('noopts')) - s.append('user/message', { content: [{ type: 'text', text: 'x' }], source: { kind: 'user' } }) + s.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) expect((s.events[0] as SessionEvent).sourceEventSeqs).toBeUndefined() expect((s.events[0] as SessionEvent).surfaceOp).toBeUndefined() }) diff --git a/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts b/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts index 1df70f9c9a..7df0fd5180 100644 --- a/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts +++ b/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts @@ -276,8 +276,8 @@ describe('SessionPersistenceJsonl: write path (session/event → flush)', () => const a = ctx.sessions.create(SessionId('sa')) const b = ctx.sessions.create(SessionId('sb')) - a.append('user/message', { content: [{ type: 'text', text: 'A' }], source: { kind: 'user' } }) - b.append('user/message', { content: [{ type: 'text', text: 'B' }], source: { kind: 'user' } }) + a.append('user/message', { content: [{ type: 'text', text: 'A' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) + b.append('user/message', { content: [{ type: 'text', text: 'B' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) a.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) b.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) await ctx.parallel('session/flush', a) @@ -647,7 +647,7 @@ describe('SessionPersistenceJsonl: edge cases', () => { await ctx2.plugin(SessionPersistenceJsonl, { root }) const session = ctx2.sessions.create(SessionId('flush-fail')) // A full turn lands in the write-behind buffer. - session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }) + session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) session.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) // Make the durable materialize fail on the next flush. const backend = ctx2.sessionPersistence as unknown as { materialize: (...args: unknown[]) => Promise } @@ -695,7 +695,7 @@ describe('SessionPersistenceJsonl: edge cases', () => { // can never diverge from the live log. The throw surfaces at the caller's // append site, not asynchronously in a backend flush. expect(() => { - session.append('user/message', { content: [{ type: 'text', text: 'bad' }], source: { kind: 'user' }, bad: 1n } as never) + session.append('user/message', { content: [{ type: 'text', text: 'bad' }], source: { kind: 'user' }, bad: 1n } as never, { surfaceOp: 'append' }) }).toThrow(/non-JSON-serializable/) // The bad event was rejected, so the log stayed empty. expect(session.events.length).toBe(0) diff --git a/packages/session-persistence/session-persistence/tests/coordinator-contract.ts b/packages/session-persistence/session-persistence/tests/coordinator-contract.ts index 431d02b4cb..71cd296fb0 100644 --- a/packages/session-persistence/session-persistence/tests/coordinator-contract.ts +++ b/packages/session-persistence/session-persistence/tests/coordinator-contract.ts @@ -128,7 +128,7 @@ export function runCoordinatorContract(name: string, makeFixture: () => Promise< const { ctx, fiber } = await freshCtx(fix) try { const session = ctx.sessions.create(SessionId('mutate'), { meta: { cwd: WORK } }) - const ev = session.append('user/message', { content: [{ type: 'text', text: 'original' }], source: { kind: 'user' } }) + const ev = session.append('user/message', { content: [{ type: 'text', text: 'original' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // Mutate the live event object AFTER it was buffered by session/event. ;(ev.data as { content: { type: 'text'; text: string }[] }).content[0]!.text = 'HACKED' session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) @@ -232,7 +232,7 @@ export function runCoordinatorContract(name: string, makeFixture: () => Promise< await ctx.plugin(SessionStore) // A session exists BEFORE the persistence plugin is applied. const session = ctx.sessions.create(SessionId('pre-existing'), { meta: { cwd: WORK } }) - session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }) + session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) session.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) const fiber = await fix.mount(ctx) @@ -253,7 +253,7 @@ export function runCoordinatorContract(name: string, makeFixture: () => Promise< await ctx.plugin(SessionStore) const fiber = await fix.mount(ctx) const session = await liveSessionInFiber(ctx, 'drain', WORK) - session.append('user/message', { content: [{ type: 'text', text: 'buffered' }], source: { kind: 'user' } }) + session.append('user/message', { content: [{ type: 'text', text: 'buffered' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) session.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) // No explicit flush — dispose must drain. await fiber.dispose() @@ -279,7 +279,7 @@ export function runCoordinatorContract(name: string, makeFixture: () => Promise< // Backend instance 1 materializes the session. const backend1 = await fix.mount(ctx) session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) - session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }) + session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) session.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) await ctx.parallel('session/flush', session) @@ -290,7 +290,7 @@ export function runCoordinatorContract(name: string, makeFixture: () => Promise< await backend1.dispose() await fix.mount(ctx) session.append('turn/start', { turn: 2, trigger: { kind: 'message', source: { kind: 'user' } } }) - session.append('user/message', { content: [{ type: 'text', text: 'again' }], source: { kind: 'user' } }) + session.append('user/message', { content: [{ type: 'text', text: 'again' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) session.append('turn/end', { turn: 2, reason: { kind: 'completed' } }) await expect(ctx.parallel('session/flush', session)).resolves.not.toThrow() @@ -452,7 +452,7 @@ export function runCoordinatorContract(name: string, makeFixture: () => Promise< const { ctx, fiber } = await freshCtx(fix) try { const session = ctx.sessions.create(SessionId('idem'), { meta: { cwd: WORK } }) - session.append('user/message', { content: [{ type: 'text', text: 'x' }], source: { kind: 'user' } }) + session.append('user/message', { content: [{ type: 'text', text: 'x' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) session.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) await ctx.parallel('session/flush', session) // Re-emit session/created for the SAME live session (idempotent initFor). @@ -704,7 +704,7 @@ export function runCoordinatorContract(name: string, makeFixture: () => Promise< // async onCreated init has necessarily set state (exercises the // state-undefined cursor path). const session = ctx.sessions.create(SessionId('flush-nostate'), { meta: { cwd: WORK } }) - session.append('user/message', { content: [{ type: 'text', text: 'q' }], source: { kind: 'user' } }) + session.append('user/message', { content: [{ type: 'text', text: 'q' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) session.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) await ctx.parallel('session/flush', session) const loaded = await ctx.sessionPersistence.load(SessionId('flush-nostate')) diff --git a/packages/support/invariants/tests/invariants.spec.ts b/packages/support/invariants/tests/invariants.spec.ts index f66b6a8929..2eaae73169 100644 --- a/packages/support/invariants/tests/invariants.spec.ts +++ b/packages/support/invariants/tests/invariants.spec.ts @@ -25,12 +25,12 @@ describe('session-log invariants', () => { const session = ctx.sessions.create() expect(() => { session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) - session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }) + session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) session.append('step/start', { turn: 1, step: 1 }) session.append('assistant/chunk', { turn: 1, step: 1, chunk: { type: 'text-delta', index: 0, text: 'h' } }) - session.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'tool-call', id: CallId('c1'), name: 'echo', arguments: '{}' }] }) + session.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'tool-call', id: CallId('c1'), name: 'echo', arguments: '{}' }] }, { surfaceOp: 'append' }) session.append('tool/call', { turn: 1, step: 1, callId: CallId('c1'), name: 'echo', arguments: '{}' }) - session.append('tool/result', { turn: 1, step: 1, callId: CallId('c1'), content: [{ type: 'text', text: 'ok' }], isError: false }) + session.append('tool/result', { turn: 1, step: 1, callId: CallId('c1'), content: [{ type: 'text', text: 'ok' }], isError: false }, { surfaceOp: 'append' }) session.append('step/end', { turn: 1, step: 1 }) session.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) }).not.toThrow() @@ -89,9 +89,9 @@ describe('session-log invariants', () => { const { ctx } = await setup({ freeze: false }) const session = ctx.sessions.create() // No turn open: every message-bearing event must be turn-enclosed (the turn-enclosure RFC). - expect(() => session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } })) + expect(() => session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' })) .toThrow(/outside any open turn/) - expect(() => session.append('context/message', { content: [{ type: 'text', text: 'ctx' }], source: { kind: 'user' } })) + expect(() => session.append('context/message', { content: [{ type: 'text', text: 'ctx' }], source: { kind: 'user' } }, { surfaceOp: 'append' })) .toThrow(/outside any open turn/) }) @@ -100,7 +100,7 @@ describe('session-log invariants', () => { const session = ctx.sessions.create() // steering/message is turn-scoped: outside a turn it would land past the // commit boundary and be dropped on resume (the turn-enclosure RFC). - expect(() => session.append('steering/message', { turn: 1, content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } })) + expect(() => session.append('steering/message', { turn: 1, content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }, { surfaceOp: 'append' })) .toThrow(/outside any open turn/) // A PLUGIN-added (merge-extensible) event type is caught by the default too. // Cast through `any`: 'compaction/marker' is not in SessionEventType (it's @@ -115,7 +115,7 @@ describe('session-log invariants', () => { const { ctx } = await setup({ freeze: false }) const session = ctx.sessions.create() session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) - expect(() => session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } })) + expect(() => session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' })) .not.toThrow() }) @@ -124,7 +124,7 @@ describe('session-log invariants', () => { const session = ctx.sessions.create() session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) session.append('step/start', { turn: 1, step: 1 }) - expect(() => session.append('tool/result', { turn: 1, step: 1, callId: CallId('ghost'), content: [], isError: false })) + expect(() => session.append('tool/result', { turn: 1, step: 1, callId: CallId('ghost'), content: [], isError: false }, { surfaceOp: 'append' })) .toThrow(/no prior tool\/call/) }) @@ -136,7 +136,7 @@ describe('session-log invariants', () => { session.append('step/start', { turn: 1, step: 1 }) session.append('assistant/message', { turn: 1, step: 1, content: [ { type: 'tool-call', id: CallId('crashed'), name: 'bash', arguments: '{}' }, - ] }) + ] }, { surfaceOp: 'append' }) session.append('tool/result', { turn: 1, step: 1, @@ -144,7 +144,7 @@ describe('session-log invariants', () => { content: [{ type: 'text', text: 'interrupted' }], isError: true, error: { name: 'InterruptedError', code: 'interrupted' }, - }) + }, { surfaceOp: 'append' }) session.append('step/end', { turn: 1, step: 1 }) session.append('turn/end', { turn: 1, reason: { kind: 'interrupted' } }) }).not.toThrow() @@ -190,10 +190,10 @@ describe('session-log invariants', () => { expect(() => { session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) session.append('step/start', { turn: 1, step: 1 }) - session.append('assistant/message', { turn: 1, step: 1, content: [] }) + session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append' }) session.append('step/end', { turn: 1, step: 1 }) session.append('step/start', { turn: 1, step: 2 }) - session.append('assistant/message', { turn: 1, step: 2, content: [] }) + session.append('assistant/message', { turn: 1, step: 2, content: [] }, { surfaceOp: 'append' }) session.append('step/end', { turn: 1, step: 2 }) session.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) session.append('turn/start', { turn: 2, trigger: { kind: 'message', source: { kind: 'user' } } }) @@ -246,7 +246,7 @@ describe('session-log invariants', () => { // step ends with the call unresolved — pendingCalls is cleared. session.append('step/end', { turn: 1, step: 1 }) session.append('step/start', { turn: 1, step: 2 }) - expect(() => session.append('tool/result', { turn: 1, step: 2, callId: CallId('c1'), content: [], isError: false })) + expect(() => session.append('tool/result', { turn: 1, step: 2, callId: CallId('c1'), content: [], isError: false }, { surfaceOp: 'append' })) .toThrow(/no prior tool\/call in this step/) }) @@ -255,7 +255,7 @@ describe('session-log invariants', () => { const session = ctx.sessions.create() session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) session.append('step/start', { turn: 1, step: 1 }) - expect(() => session.append('assistant/message', { turn: 1, step: 2, content: [] })) + expect(() => session.append('assistant/message', { turn: 1, step: 2, content: [] }, { surfaceOp: 'append' })) .toThrow(/open is turn 1\/step 1/) }) }) @@ -287,7 +287,7 @@ describe('dev-freeze', () => { const { ctx } = await setup() // freeze defaults true const session = ctx.sessions.create() session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) - const event = session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }) + const event = session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) expect(Object.isFrozen(event)).toBe(true) expect(Object.isFrozen(event.data)).toBe(true) expect(Object.isFrozen(event.data.content)).toBe(true) @@ -298,7 +298,7 @@ describe('dev-freeze', () => { const { ctx } = await setup({ freeze: false }) const session = ctx.sessions.create() session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) - const event = session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }) + const event = session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) expect(Object.isFrozen(event)).toBe(false) }) @@ -324,7 +324,7 @@ describe('dev-freeze', () => { // the caller's input — read the event back and assert on its data. const innerContent: { type: 'text'; text: string }[] = [{ type: 'text', text: 'inner' }] const block = Object.freeze({ type: 'tool-result' as const, toolCallId: CallId('c1'), content: innerContent, isError: false }) - const event = session.append('user/message', { content: [block], source: { kind: 'user' } }) + const event = session.append('user/message', { content: [block], source: { kind: 'user' } }, { surfaceOp: 'append' }) const logged = event.data.content[0] as { content: { type: 'text'; text: string }[] } expect(Object.isFrozen(logged.content)).toBe(true) expect(Object.isFrozen(logged.content[0])).toBe(true) @@ -424,7 +424,7 @@ describe('HMR safety', () => { const spy = vi.fn() ctx.on('session/event', spy) const session = ctx.sessions.create() - session.append('user/message', { content: [{ type: 'text', text: 'x' }], source: { kind: 'user' } }) + session.append('user/message', { content: [{ type: 'text', text: 'x' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // our own spy fires, proving events still flow — but the plugin's frozen. expect(spy).toHaveBeenCalledOnce() expect(Object.isFrozen(session.events[0])).toBe(false) From 1ec8c40d0dcef27e689e6d81412ae464d29c91da Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Tue, 23 Jun 2026 13:26:45 +0800 Subject: [PATCH 08/17] refactor(surface): use nodeBySeq map for lookup in _replace, drop dead params --- docs/core-data-structures/session.md | 46 +++++++++++++++++++++++++++- packages/core/session/src/surface.ts | 28 ++++++++--------- scripts/type-equiv.manifest.json | 4 +++ 3 files changed, 63 insertions(+), 15 deletions(-) diff --git a/docs/core-data-structures/session.md b/docs/core-data-structures/session.md index 62f6d062a4..2618db43c9 100644 --- a/docs/core-data-structures/session.md +++ b/docs/core-data-structures/session.md @@ -66,7 +66,51 @@ type SessionEvent = { `SessionEventType = keyof SessionEventMap`. Because `SessionEventMap` is merge-extensible, switches over `SessionEvent` must NOT use `assertNever` — a plugin-added variant is a valid unknown value; handle the known cases and fall through `default`. -The five message-producing types (`SurfaceEventType` — `user/message`, `assistant/message`, `tool/result`, `context/message`, `steering/message`) additionally carry two optional surface fields: `surfaceOp` (how the event enters the derived surface linked list — `'append'` or a `{ op: 'replace', start, end }` shadow) and `sourceEventSeqs` (provenance). See the [session surface RFC](../rfc/implemented/architecture/2026-06-18-session-surface.md). +## Surface types + +The five message-producing types (`SurfaceEventType` — `user/message`, `assistant/message`, `tool/result`, `context/message`, `steering/message`) carry surface metadata declaring how they join the derived surface linked list. See the [session surface RFC](../rfc/implemented/architecture/2026-06-18-session-surface.md). + +### `SurfaceEventType` — the message-producing subset of event types + +```ts type-equiv +export type SurfaceEventType = + | 'user/message' + | 'assistant/message' + | 'tool/result' + | 'context/message' + | 'steering/message' +``` + +### `SurfaceOp` — how an event entered the surface + +```ts type-equiv +export type SurfaceOp = + | 'append' + | { op: 'replace'; start: number; end: number } +``` + +`'append'` is the normal tail-append path. `replace` shadows surface nodes from `start` through `end` inclusive (both must be valid surface node seqs; `start === end` replaces a single node) and inserts the new node in their place. + +### `SurfaceIntent` — the parameter to `session.append()` + +```ts type-equiv +export interface SurfaceIntent { + surfaceOp: SurfaceOp + sourceEventSeqs?: number[] +} +``` + +Required for `SurfaceEventType` events — every message-producing event must declare how it joins the surface, the sole source of derived history. Non-surface types reject it at compile time. + +### `SurfaceNode` — a node in the surface linked list + +```ts type-equiv +export interface SurfaceNode { + seq: number + prev: number | null + next: number | null +} +``` ## Derived history: `deriveMessages()` diff --git a/packages/core/session/src/surface.ts b/packages/core/session/src/surface.ts index 7ed1743af3..3615668674 100644 --- a/packages/core/session/src/surface.ts +++ b/packages/core/session/src/surface.ts @@ -55,7 +55,7 @@ export interface SurfaceNode { export class SurfaceManager { /** Surface nodes in linked-list order (head to tail). Empty until first access. */ private _nodes: SurfaceNode[] = [] - /** Map from event seq → node for O(1) lookup during replacements. */ + /** Map from event seq → node. */ private _nodeBySeq = new Map() /** The last processed seq. -1 forces a full rebuild on first access. */ private _lastProcessedSeq = -1 @@ -100,7 +100,7 @@ export class SurfaceManager { this._nodes.push(node) this._nodeBySeq.set(event.seq, node) } else { - this._replace(this._nodes, this._nodeBySeq, event.seq, event.surfaceOp) + this._replace(event.seq, event.surfaceOp) } } this._lastProcessedSeq = this.log.length - 1 @@ -108,31 +108,31 @@ export class SurfaceManager { /** Apply a replace operation to the in-progress surface. */ private _replace( - nodes: SurfaceNode[], - nodeBySeq: Map, newSeq: number, op: Extract, ): void { - const startIdx = nodes.findIndex(n => n.seq === op.start) - if (startIdx === -1) { + const startNode = this._nodeBySeq.get(op.start) + if (!startNode) { throw new Error(`surface replace: start seq ${op.start} not found in surface`) } - const endIdx = nodes.findIndex(n => n.seq === op.end) - if (endIdx === -1) { + const endNode = this._nodeBySeq.get(op.end) + if (!endNode) { throw new Error(`surface replace: end seq ${op.end} not found in surface`) } + const startIdx = this._nodes.indexOf(startNode) + const endIdx = this._nodes.indexOf(endNode) if (startIdx > endIdx) { throw new Error(`surface replace: start seq ${op.start} (index ${startIdx}) is after end seq ${op.end} (index ${endIdx})`) } // Remove shadowed nodes from `[startIdx, endIdx]` inclusive. const count = endIdx - startIdx + 1 - const removed = nodes.splice(startIdx, count) - for (const r of removed) nodeBySeq.delete(r.seq) + const removed = this._nodes.splice(startIdx, count) + for (const r of removed) this._nodeBySeq.delete(r.seq) // Insert the new node where the removed range was. - const prevNode = startIdx > 0 ? nodes[startIdx - 1] : undefined - const nextNode = startIdx < nodes.length ? nodes[startIdx] : undefined + const prevNode = startIdx > 0 ? this._nodes[startIdx - 1] : undefined + const nextNode = startIdx < this._nodes.length ? this._nodes[startIdx] : undefined const newNode: SurfaceNode = { seq: newSeq, @@ -141,7 +141,7 @@ export class SurfaceManager { } if (prevNode) prevNode.next = newSeq if (nextNode) nextNode.prev = newSeq - nodes.splice(startIdx, 0, newNode) - nodeBySeq.set(newSeq, newNode) + this._nodes.splice(startIdx, 0, newNode) + this._nodeBySeq.set(newSeq, newNode) } } diff --git a/scripts/type-equiv.manifest.json b/scripts/type-equiv.manifest.json index f46c4fca6b..30165e2388 100644 --- a/scripts/type-equiv.manifest.json +++ b/scripts/type-equiv.manifest.json @@ -19,6 +19,10 @@ { "doc": "docs/core-data-structures/session.md", "symbol": "SessionEvent", "source": "packages/core/session/src/types.ts" }, { "doc": "docs/core-data-structures/session.md", "symbol": "TurnTriggerMap", "source": "packages/core/session/src/types.ts" }, { "doc": "docs/core-data-structures/session.md", "symbol": "TurnEndReasonMap", "source": "packages/core/session/src/types.ts" }, + { "doc": "docs/core-data-structures/session.md", "symbol": "SurfaceEventType", "source": "packages/core/session/src/types.ts" }, + { "doc": "docs/core-data-structures/session.md", "symbol": "SurfaceOp", "source": "packages/core/session/src/types.ts" }, + { "doc": "docs/core-data-structures/session.md", "symbol": "SurfaceIntent", "source": "packages/core/session/src/types.ts" }, + { "doc": "docs/core-data-structures/session.md", "symbol": "SurfaceNode", "source": "packages/core/session/src/surface.ts" }, { "doc": "docs/core-data-structures/persistence.md", "symbol": "SessionHeader", "source": "packages/core/session/src/types.ts" }, { "doc": "docs/core-data-structures/persistence.md", "symbol": "CreateSessionOptions", "source": "packages/core/session/src/types.ts" }, From 358ae02c5610e923e7a1e6405da29f9fd1fe9f71 Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Tue, 23 Jun 2026 13:37:13 +0800 Subject: [PATCH 09/17] =?UTF-8?q?feat(invariants):=20enforce=20replace=20p?= =?UTF-8?q?rovenance=20=E2=80=94=20sourceEventSeqs=20must=20cover=20every?= =?UTF-8?q?=20shadowed=20surface=20node?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2026-06-18-session-surface.md | 2 +- packages/support/invariants/src/index.ts | 44 ++++++++- .../invariants/tests/invariants.spec.ts | 90 +++++++++++++++++++ 3 files changed, 132 insertions(+), 4 deletions(-) diff --git a/docs/rfc/implemented/architecture/2026-06-18-session-surface.md b/docs/rfc/implemented/architecture/2026-06-18-session-surface.md index ae2475c786..0ee163cbcf 100644 --- a/docs/rfc/implemented/architecture/2026-06-18-session-surface.md +++ b/docs/rfc/implemented/architecture/2026-06-18-session-surface.md @@ -49,7 +49,7 @@ The `repair.ts` module synthesizes `tool/result` closers for orphaned tool calls ### Invariants -The dev-mode invariants plugin validates: `sourceEventSeqs` references (non-empty, no duplicates, references earlier events, references known seqs) and `surfaceOp` (replace start ≤ end). +The dev-mode invariants plugin validates: `sourceEventSeqs` references (non-empty, no duplicates, references earlier events, references known seqs) and `surfaceOp` (replace `start ≤ end`, both endpoints are on the tracked surface, the range is non-reversed in surface position, and `sourceEventSeqs` includes every node the range shadows). ## Consequences diff --git a/packages/support/invariants/src/index.ts b/packages/support/invariants/src/index.ts index 492db4bdd0..b2372e28db 100644 --- a/packages/support/invariants/src/index.ts +++ b/packages/support/invariants/src/index.ts @@ -69,6 +69,13 @@ interface SessionTrace { pendingCalls: Set /** Every seq seen so far — validates `sourceEventSeqs` references. */ knownSeqs: Set + /** + * The seqs currently on the surface linked list, in linked-list order + * (head to tail). A replace reorders this relative to seq order (the new + * node takes the replaced range's position), so range validation is + * positional, not by seq comparison. + */ + surface: number[] } /** @@ -147,9 +154,39 @@ function checkEvent(trace: SessionTrace, event: SessionEvent): void { } } } - if (se.surfaceOp !== undefined && typeof se.surfaceOp !== 'string') { - if (se.surfaceOp.start > se.surfaceOp.end) { - throw new InvariantError(`surface replace: start ${se.surfaceOp.start} must be <= end ${se.surfaceOp.end}`) + // Fold this event into the tracked surface linked list, validating the + // replace contract as we go. `append` adds a tail node; `replace` shadows a + // positional range — every shadowed node must appear in sourceEventSeqs. + if (se.surfaceOp !== undefined) { + if (se.surfaceOp === 'append') { + trace.surface.push(event.seq) + } else { + const { start, end } = se.surfaceOp + if (start > end) { + throw new InvariantError(`surface replace: start ${start} must be <= end ${end}`) + } + const startIdx = trace.surface.indexOf(start) + if (startIdx === -1) { + throw new InvariantError(`surface replace: start seq ${start} is not on the surface`) + } + const endIdx = trace.surface.indexOf(end) + if (endIdx === -1) { + throw new InvariantError(`surface replace: end seq ${end} is not on the surface`) + } + if (startIdx > endIdx) { + throw new InvariantError(`surface replace: start seq ${start} (pos ${startIdx}) is after end seq ${end} (pos ${endIdx}) on the surface`) + } + // Every node the replace shadows (surface positions [startIdx, endIdx] + // inclusive) must appear in sourceEventSeqs — the provenance contract. + const shadowed = trace.surface.slice(startIdx, endIdx + 1) + const recorded = new Set(se.sourceEventSeqs ?? []) + const missing = shadowed.filter(seq => !recorded.has(seq)) + if (missing.length > 0) { + throw new InvariantError(`surface replace: sourceEventSeqs must include every shadowed surface node; missing ${missing.join(', ')}`) + } + // Apply the replace to the tracked surface: the new node takes the + // range's position so order stays in sync for later replaces. + trace.surface.splice(startIdx, shadowed.length, event.seq) } } @@ -288,6 +325,7 @@ export function apply(ctx: Context, config: Config = {}): void { nextStep: 1, pendingCalls: new Set(), knownSeqs: new Set(), + surface: [], }) /** Build (or rebuild) a session's trace by replaying its whole log; freeze it. */ diff --git a/packages/support/invariants/tests/invariants.spec.ts b/packages/support/invariants/tests/invariants.spec.ts index 2eaae73169..21f6356aee 100644 --- a/packages/support/invariants/tests/invariants.spec.ts +++ b/packages/support/invariants/tests/invariants.spec.ts @@ -542,6 +542,96 @@ describe('surface invariants', () => { }).toThrow(/must be <= end/) }) + it('rejects a replace whose sourceEventSeqs omits a shadowed surface node', async () => { + const { ctx } = await setup() + const session = ctx.sessions.create() + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + session.append('step/start', { turn: 1, step: 1 }) + session.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 2 + session.append('user/message', { content: [{ type: 'text', text: 'b' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 3 + // Replace shadows surface nodes [2, 3] but records provenance for only [2]. + expect(() => { + session.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 'sum' }] }, { surfaceOp: { op: 'replace', start: 2, end: 3 }, sourceEventSeqs: [2] }) + }).toThrow(/must include every shadowed surface node; missing 3/) + }) + + it('accepts a replace whose sourceEventSeqs covers every shadowed surface node', async () => { + const { ctx } = await setup() + const session = ctx.sessions.create() + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + session.append('step/start', { turn: 1, step: 1 }) + session.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 2 + session.append('user/message', { content: [{ type: 'text', text: 'b' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 3 + expect(() => { + session.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 'sum' }] }, { surfaceOp: { op: 'replace', start: 2, end: 3 }, sourceEventSeqs: [2, 3] }) + }).not.toThrow() + }) + + it('rejects a replace naming a start seq that is not on the surface', async () => { + const { ctx } = await setup() + const session = ctx.sessions.create() + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + session.append('step/start', { turn: 1, step: 1 }) + session.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 2 + // seq 1 (step/start) is a real earlier event but never entered the surface. + expect(() => { + session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 1, end: 2 }, sourceEventSeqs: [1, 2] }) + }).toThrow(/start seq 1 is not on the surface/) + }) + + it('rejects a replace naming an end seq that is not on the surface', async () => { + const { ctx } = await setup() + const session = ctx.sessions.create() + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + session.append('step/start', { turn: 1, step: 1 }) + session.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 2 + // start (2) is on the surface but end (99) never entered it. + expect(() => { + session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 2, end: 99 }, sourceEventSeqs: [2] }) + }).toThrow(/end seq 99 is not on the surface/) + }) + + it('rejects a replace whose range is reversed in surface position after a prior replace reordered it', async () => { + const { ctx } = await setup() + const session = ctx.sessions.create() + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + session.append('step/start', { turn: 1, step: 1 }) + session.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 2 + session.append('user/message', { content: [{ type: 'text', text: 'b' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 3 + // Replace node 2 (position 0) with seq 4 — surface is now [4, 3], so seq 4 + // precedes seq 3 in linked-list order even though 4 > 3 numerically. + session.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 's' }] }, { surfaceOp: { op: 'replace', start: 2, end: 2 }, sourceEventSeqs: [2] }) // seq 4 + // A replace with start=3, end=4 passes the seq check (3 <= 4) but is + // reversed positionally (3 is at pos 1, 4 is at pos 0). + expect(() => { + session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 3, end: 4 }, sourceEventSeqs: [3, 4] }) // seq 5 + }).toThrow(/is after end seq 4 .* on the surface/) + }) + + it('rejects a replace that omits sourceEventSeqs entirely', async () => { + const { ctx } = await setup() + const session = ctx.sessions.create() + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + session.append('step/start', { turn: 1, step: 1 }) + session.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 2 + // A replace with no sourceEventSeqs records no provenance for the node it shadows. + expect(() => { + session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 2, end: 2 } }) + }).toThrow(/must include every shadowed surface node; missing 2/) + }) + + it('catches an incomplete-provenance replace on the load/seed path', async () => { + const { ctx } = await setup({ freeze: false }) + const badSeed = [ + { type: 'turn/start' as const, seq: 0, time: 0, data: { turn: 1, trigger: { kind: 'message' as const, source: { kind: 'user' as const } } } }, + { type: 'step/start' as const, seq: 1, time: 0, data: { turn: 1, step: 1 } }, + { type: 'user/message' as const, seq: 2, time: 0, data: { content: [{ type: 'text' as const, text: 'a' }], source: { kind: 'user' as const } }, surfaceOp: 'append' as const }, + { type: 'user/message' as const, seq: 3, time: 0, data: { content: [{ type: 'text' as const, text: 'b' }], source: { kind: 'user' as const } }, surfaceOp: 'append' as const }, + { type: 'assistant/message' as const, seq: 4, time: 0, data: { turn: 1, step: 1, content: [{ type: 'text' as const, text: 'sum' }] }, surfaceOp: { op: 'replace' as const, start: 2, end: 3 }, sourceEventSeqs: [2] }, + ] + expect(() => ctx.sessions.create(undefined, { seed: badSeed })).toThrow(/must include every shadowed surface node; missing 3/) + }) + it('rejects sourceEventSeqs on a non-surface event', async () => { const { ctx } = await setup() const session = ctx.sessions.create() From f4180bd764307f418c66d810e18e7e8c767fc6e5 Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Tue, 23 Jun 2026 15:45:35 +0800 Subject: [PATCH 10/17] feat(compact): add optional cancellation signal to the compact seam methods --- docs/cordis-catalog/events-and-services.md | 4 +-- packages/compact/compact/README.md | 6 ++-- packages/compact/compact/src/index.ts | 10 ++++++ .../compact/compact/tests/compact.spec.ts | 33 +++++++++++++++++-- 4 files changed, 47 insertions(+), 6 deletions(-) diff --git a/docs/cordis-catalog/events-and-services.md b/docs/cordis-catalog/events-and-services.md index 623c30c3e5..26cd55968b 100644 --- a/docs/cordis-catalog/events-and-services.md +++ b/docs/cordis-catalog/events-and-services.md @@ -351,8 +351,8 @@ Implementations MUST honor: - **Blocking**: no compaction begins while another is in progress for the same session. The recommended mechanism is the log-recorded lock — append `compact/start` before the slow work and `compact/end` after (even on failure) — so the lock is visible to replay and crash recovery. ```ts cordis-catalog -abstract compactIfNeeded( session: Session, systemPrompt?: string, model?: string, ): Promise -abstract compactRegion( session: Session, start: number, end: number, model: string, ): Promise +abstract compactIfNeeded( session: Session, systemPrompt?: string, model?: string, signal?: AbortSignal, ): Promise +abstract compactRegion( session: Session, start: number, end: number, model: string, signal?: AbortSignal, ): Promise ``` Source: [`packages/compact/compact/src/index.ts:57`](../../packages/compact/compact/src/index.ts) diff --git a/packages/compact/compact/README.md b/packages/compact/compact/README.md index e98cdd52a6..96e3fc3f94 100644 --- a/packages/compact/compact/README.md +++ b/packages/compact/compact/README.md @@ -18,8 +18,10 @@ Both methods are **abstract** — the backend owns the entire strategy (token es | Member | Semantics | |---|---| -| `compactIfNeeded(session, systemPrompt?, model?)` | Estimate the history size; if over the backend's threshold, compact an older range via `compactRegion`, keeping recent context intact. Returns the `CompactionResult`, or `null` if nothing needed compacting. | -| `compactRegion(session, start, end, model)` | Forcibly summarize surface nodes `[start, end]` (inclusive seqs) into a single replacement node. **Throws** if a compaction is already in progress, if `start`/`end` aren't surface nodes, or if `start > end`. | +| `compactIfNeeded(session, systemPrompt?, model?, signal?)` | Estimate the history size; if over the backend's threshold, compact an older range via `compactRegion`, keeping recent context intact. Returns the `CompactionResult`, or `null` if nothing needed compacting. | +| `compactRegion(session, start, end, model, signal?)` | Forcibly summarize surface nodes `[start, end]` (inclusive seqs) into a single replacement node. **Throws** if a compaction is already in progress, if `start`/`end` aren't surface nodes, or if `start > end`. | + +Both methods take an optional `signal: AbortSignal`. A backend that summarizes via `ctx.llm.stream()` **must** forward it into the call's `GenerateOptions.signal`, so an abort or fiber dispose tears down the in-flight summarization instead of leaving an orphaned model call running past the cancellation. The turn that the `compact/*` events belong to is not a parameter — it is recoverable from the log (the currently-open turn), so the backend stamps it without the caller supplying it. ## Surface contract diff --git a/packages/compact/compact/src/index.ts b/packages/compact/compact/src/index.ts index acd10d4c90..9e58e5c905 100644 --- a/packages/compact/compact/src/index.ts +++ b/packages/compact/compact/src/index.ts @@ -69,12 +69,17 @@ export abstract class CompactService extends Service { * @param session - the session whose surface may be compacted. * @param systemPrompt - optional system prompt, counted toward the estimate. * @param model - optional summarization model (falls back to backend config). + * @param signal - optional cancellation signal. A backend that summarizes via + * `ctx.llm.stream()` MUST forward this into the call's `GenerateOptions.signal` + * so an abort/dispose tears down the in-flight summarization rather than + * leaving an orphaned model call running past the cancellation. * @returns the compaction result, or `null` if no compaction was needed. */ abstract compactIfNeeded( session: Session, systemPrompt?: string, model?: string, + signal?: AbortSignal, ): Promise /** @@ -88,6 +93,10 @@ export abstract class CompactService extends Service { * @param start - inclusive seq of the first surface node to compact. * @param end - inclusive seq of the last surface node to compact. * @param model - summarization model. + * @param signal - optional cancellation signal. A backend that summarizes via + * `ctx.llm.stream()` MUST forward this into the call's `GenerateOptions.signal` + * so an abort/dispose tears down the in-flight summarization rather than + * leaving an orphaned model call running past the cancellation. * @throws if compaction is already in progress, or if `start`/`end` are not * valid surface nodes, or if `start > end`. */ @@ -96,6 +105,7 @@ export abstract class CompactService extends Service { start: number, end: number, model: string, + signal?: AbortSignal, ): Promise } diff --git a/packages/compact/compact/tests/compact.spec.ts b/packages/compact/compact/tests/compact.spec.ts index c4f0c0f838..4a758f4364 100644 --- a/packages/compact/compact/tests/compact.spec.ts +++ b/packages/compact/compact/tests/compact.spec.ts @@ -11,11 +11,27 @@ import { Session, SessionId } from '@deepseek-ai/dsh-session' * declaration merge. */ class StubCompactService extends CompactService { - override async compactIfNeeded(_session: Session, _systemPrompt?: string, _model?: string): Promise { + /** Records the signal handed to the most recent call, to prove it threads through. */ + lastSignal: AbortSignal | undefined + + override async compactIfNeeded( + _session: Session, + _systemPrompt?: string, + _model?: string, + signal?: AbortSignal, + ): Promise { + this.lastSignal = signal return null } - override async compactRegion(session: Session, start: number, end: number, _model: string): Promise { + override async compactRegion( + session: Session, + start: number, + end: number, + _model: string, + signal?: AbortSignal, + ): Promise { + this.lastSignal = signal // Minimal stub honoring the lock + log-only event contract. const startEvent = session.append('compact/start', { turn: 0 }) const summaryEvent = session.append('compact/summary', { @@ -75,4 +91,17 @@ describe('CompactService seam', () => { expect(result.summarySeq).toBeGreaterThan(result.startSeq) expect(result.endSeq).toBeGreaterThan(result.summarySeq) }) + + it('threads the cancellation signal through to the backend', async () => { + const ctx = new Context() + const svc = new StubCompactService(ctx) + const session = new Session(SessionId('s')) + const controller = new AbortController() + + await svc.compactRegion(session, 0, 0, 'm', controller.signal) + expect(svc.lastSignal).toBe(controller.signal) + + await svc.compactIfNeeded(session, undefined, undefined, controller.signal) + expect(svc.lastSignal).toBe(controller.signal) + }) }) From bb013e934e0c9413768cd40b4cd8323578743684 Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Tue, 23 Jun 2026 16:13:55 +0800 Subject: [PATCH 11/17] docs(compact): bracket the surface mutation inside the compaction lock --- .../feature/2026-06-18-compaction-capability-seam.md | 8 +++++--- packages/compact/compact/README.md | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/rfc/proposed/feature/2026-06-18-compaction-capability-seam.md b/docs/rfc/proposed/feature/2026-06-18-compaction-capability-seam.md index 3725a3db84..1ddcd05b42 100644 --- a/docs/rfc/proposed/feature/2026-06-18-compaction-capability-seam.md +++ b/docs/rfc/proposed/feature/2026-06-18-compaction-capability-seam.md @@ -32,22 +32,24 @@ An earlier draft put the full algorithm (the retention walk, token-summing, text ### Surface replacement: `compact/*` events are log-only; one `user/message` carries the summary -Because `SurfaceEventType` is closed, the summary cannot ride on a `compact/*` event. The backend instead appends a **single `user/message`** with `surfaceOp: { op: 'replace', start, end }` whose `content` is the summary `ContentBlock[]` and whose `sourceEventSeqs` covers the shadowed nodes *and* the bookkeeping events. The `compact/*` events are pure log records (lock + provenance), never on the surface: +Because `SurfaceEventType` is closed, the summary cannot ride on a `compact/*` event. The backend instead appends a **single `user/message`** with `surfaceOp: { op: 'replace', start, end }` whose `content` is the summary `ContentBlock[]` and whose `sourceEventSeqs` covers the shadowed nodes *and* the bookkeeping events. The `compact/*` events are pure log records (lock + provenance), never on the surface. The surface mutation sits **inside** the lock — `compact/end` is the last event appended: ``` compact/start → log-only. Acquires the lock. [summarize older range via the backend] compact/summary → log-only. Provenance: summary, range, shadowed seqs, token count. -compact/end → log-only. Releases the lock. user/message → surfaceOp { op:'replace', start, end }. THE surface mutation. deriveMessages() renders it as a user-role message. +compact/end → log-only. Releases the lock. ``` +Ordering the surface mutation **before** `compact/end` is deliberate: `session.append()` commits one event at a time, so there is no multi-event transaction to make the sequence atomic. Releasing the lock last converts the crash window from *silent corruption* (a `compact/end` that claims compaction finished while the surface was never shadowed) into a *detectable orphaned lock* (a `compact/start` with no matching `compact/end`), which a persistence backend already detects on reload. A `session/event` listener on `compact/end` likewise never sees the lock free before the replacement has landed. + `deriveMessages()` then yields `[summary_as_user_message, ...retained_nodes]`. An alternative — extending `SurfaceEventType` to admit a `compact/*` type — was rejected: the closed union is a deliberate safety boundary (only message-producing events reach the model), and a summary genuinely *is* user-role context, so reusing `user/message` is honest rather than a workaround. ### Blocking via a log-recorded lock, not a mutex -Compaction must be serialized: no second compaction starts before the first finishes, and no ordinary events interleave the slow summarization. Rather than an in-memory mutex (invisible to replay, lost on crash), the lock **is** the log: `compactRegion` refuses to start if the last `compact/start` has no matching `compact/end` after it. `compact/start` is appended first (fast, synchronous), the slow model call runs, then `compact/end` is appended — in a `catch` that records the error, so a failed summarization can never wedge the lock. Because the backend runs compaction synchronously inside the `agent/request` waterfall, the loop is single-threaded for that window; the lock additionally gives observability and lets a persistence backend detect an orphaned `compact/start` on reload. +Compaction must be serialized: no second compaction starts before the first finishes, and no ordinary events interleave the slow summarization. Rather than an in-memory mutex (invisible to replay, lost on crash), the lock **is** the log: `compactRegion` refuses to start if the last `compact/start` has no matching `compact/end` after it. `compact/start` is appended first (fast, synchronous), the slow model call runs, then the `compact/summary` and `user/message` replacement land, and only then is `compact/end` appended — in a `catch` that records the error, so a failed summarization can never wedge the lock. Because the backend runs compaction synchronously inside the `agent/request` waterfall, the loop is single-threaded for that window; the lock additionally gives observability and lets a persistence backend detect an orphaned `compact/start` on reload. ## Consequences diff --git a/packages/compact/compact/README.md b/packages/compact/compact/README.md index 96e3fc3f94..a6e2176bd8 100644 --- a/packages/compact/compact/README.md +++ b/packages/compact/compact/README.md @@ -30,14 +30,16 @@ Both methods take an optional `signal: AbortSignal`. A backend that summarizes v 1. appends `compact/start` (log-only) — acquires the lock, 2. summarizes the range, 3. appends `compact/summary` (log-only) — provenance: summary, range, shadowed seqs, token count, -4. appends `compact/end` (log-only) — releases the lock, -5. appends a single `user/message` with `surfaceOp: { op: 'replace', start, end }` carrying the summary — **the only surface mutation**. +4. appends a single `user/message` with `surfaceOp: { op: 'replace', start, end }` carrying the summary — **the only surface mutation**, +5. appends `compact/end` (log-only) — releases the lock. + +The surface mutation (step 4) sits **inside** the lock bracket: `compact/end` is the last event, so the lock is never released before the mutation lands. A crash between `compact/start` and `compact/end` therefore leaves a detectable orphaned lock (a `compact/start` with no matching `compact/end`) rather than a `compact/end` that falsely claims compaction finished while the surface was never shadowed. `deriveMessages()` then renders the summary as a user-role message followed by the retained nodes. The shadowed events remain in the raw log, so replay is deterministic. ## Blocking -Compaction is serialized via a log-recorded lock: `compactRegion` refuses to start if the last `compact/start` has no matching `compact/end` after it. The lock is the log (not an in-memory mutex), so it survives replay and a persistence backend can detect an orphaned `compact/start` on reload. `compact/end` is appended even when summarization throws, so a failure can never wedge the lock. +Compaction is serialized via a log-recorded lock: `compactRegion` refuses to start if the last `compact/start` has no matching `compact/end` after it. The lock is the log (not an in-memory mutex), so it survives replay and a persistence backend can detect an orphaned `compact/start` on reload. The lock brackets the **whole** operation — summarization, the `compact/summary` provenance record, *and* the `user/message` surface replacement all happen before `compact/end` — so a `session/event` listener firing on `compact/end` never observes the lock free while the surface mutation is still pending. `compact/end` is appended even when summarization throws, so a failure can never wedge the lock. ## Events From 58d798492a9bde525fda2eeb9a741cf90ede8adb Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Tue, 23 Jun 2026 16:33:05 +0800 Subject: [PATCH 12/17] docs(compact): catalog the compaction seam in core-data-structures --- docs/core-data-structures/compaction.md | 46 +++++++++++++++++++++++++ docs/core-data-structures/core.md | 1 + docs/core-data-structures/session.md | 2 +- scripts/type-equiv.manifest.json | 4 ++- 4 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 docs/core-data-structures/compaction.md diff --git a/docs/core-data-structures/compaction.md b/docs/core-data-structures/compaction.md new file mode 100644 index 0000000000..75900bc5db --- /dev/null +++ b/docs/core-data-structures/compaction.md @@ -0,0 +1,46 @@ +# Compaction + +The compaction seam — a [capability seam](../rfc/implemented/architecture/2026-06-13-capability-seams.md) split like bash: interface ([dsh-compact](../../packages/compact/compact), `ctx.compact`), implementation (a backend such as `dsh-compact-basic`, deferred), and consumer (a `/compact` tool, deferred). Compaction is **one optional capability**, not part of the agent-loop spine — so its vocabulary lives here, not in [core.md](core.md). A tokenizer- or template-based backend is a sibling package implementing the same interface. Unlike bash, the interface necessarily depends on `dsh-session` and `dsh-llm`: its verbs are defined over a `Session` and its output is the `ContentBlock` vocabulary (see the [compaction capability-seam RFC](../rfc/proposed/feature/2026-06-18-compaction-capability-seam.md)). + +Source: [`packages/compact/compact/src/types.ts`](../../packages/compact/compact/src/types.ts) + +## The `compact/*` session events + +Compaction extends [`SessionEventMap`](session.md) with three event types via declaration merging. All three are **log-only** — they record the compaction lock and its provenance, and never join the surface. `SurfaceEventType` is deliberately NOT extended (only message-producing events reach the model), so the summary itself rides on a separate `user/message` with `surfaceOp: { op: 'replace', start, end }` — the only surface mutation. See the RFC for why reusing `user/message` is honest rather than a workaround. + +| Event | Payload | Role | +|---|---|---| +| `compact/start` | `{ turn }` | acquires the log-recorded lock | +| `compact/summary` | `{ summary, compactedRange, compactedEventSeqs, tokenCount }` | provenance: the summary blocks, the shadowed seq range, and the estimated token count | +| `compact/end` | `{ turn, error? }` | releases the lock (`error` set when summarization threw) | + +The lock brackets the **whole** operation: `compact/start` is appended first, then summarization, the `compact/summary` provenance record, and the `user/message` replacement all land, and only then `compact/end`. Releasing the lock last turns a crash mid-operation into a detectable orphaned lock (a `compact/start` with no matching `compact/end`) rather than a `compact/end` that falsely claims compaction finished. + +These variants are merged inside a `declare module '@deepseek-ai/dsh-session'` block, so — unlike the top-level types on the other sub-pages — they are not pasted as a drift-checked ` ```ts type-equiv ` block (the `verify-type-equiv` extractor matches only top-level declarations by name). The payload table above is the catalog entry; follow the source link for the authoritative shapes. + +## `CompactionResult` + +What a successful compaction returns to its caller: the seqs of the three appended `compact/*` events, the summary blocks, and the shadowed range/seqs plus the estimated token count. + +```ts type-equiv +interface CompactionResult { + /** The seq of the appended `compact/start` event. */ + startSeq: number + /** The seq of the appended `compact/summary` event. */ + summarySeq: number + /** The seq of the appended `compact/end` event. */ + endSeq: number + /** The summary content blocks produced by the backend. */ + summary: ContentBlock[] + /** The seq range that was shadowed [start, end] inclusive. */ + shadowedRange: { start: number; end: number } + /** The seq numbers of all shadowed surface nodes. */ + shadowedSeqs: number[] + /** Estimated token count of the shadowed content. */ + compactedTokenCount: number +} +``` + +## The service + +`CompactService` (`ctx.compact`, abstract — defined in [`packages/compact/compact/src/index.ts`](../../packages/compact/compact/src/index.ts)) declares two abstract methods: `compactIfNeeded(session, systemPrompt?, model?, signal?)` checks token pressure and compacts an older range if the history is too large (returning `null` when nothing needs it), and `compactRegion(session, start, end, model, signal?)` forcibly summarizes surface nodes `[start, end]` into a single replacement node. Both take an optional `signal: AbortSignal` that a backend summarizing via `ctx.llm.stream()` must forward into the call's `GenerateOptions.signal`, so an abort or dispose tears down the in-flight summarization. The entire strategy — token estimation, retention policy, event sequencing, summarization — is a HOW decision owned by the implementation. diff --git a/docs/core-data-structures/core.md b/docs/core-data-structures/core.md index b13cdc1de4..b59cfd334b 100644 --- a/docs/core-data-structures/core.md +++ b/docs/core-data-structures/core.md @@ -20,6 +20,7 @@ Everything else is documented on a **sub-page**, not here. The rule that draws t | [persistence.md](persistence.md) | the durability seam: `SessionPersistence`, JSONL + SQLite backends, `session/flush`, crash recovery, `SessionHeader` | | [tools.md](tools.md) | `ToolDefinition` full fields, the schema DSL, `ToolExecution`/`ToolResult`, tool-presentation UI types, the `tools/execute` waterfall | | [bash.md](bash.md) | the bash executor seam: `BashExecRequest`/`Spec`, `BashRunResult`, background `BashTask`s | +| [compaction.md](compaction.md) | the compaction seam: the `compact/*` session events, `CompactionResult`, the `CompactService` interface | > Type definitions on this page are pasted **verbatim** from source and drift-checked by `pnpm run verify-type-equiv` (see [development.md](../development.md#documenting-types-verbatim-ts-type-equiv)). Inline JSDoc is omitted for readability; follow the source link for the full contracts. diff --git a/docs/core-data-structures/session.md b/docs/core-data-structures/session.md index 2618db43c9..8f8ef4a800 100644 --- a/docs/core-data-structures/session.md +++ b/docs/core-data-structures/session.md @@ -6,7 +6,7 @@ Source: [`packages/core/session/src/types.ts`](../../packages/core/session/src/t ## `SessionEventMap` — the event vocabulary -The append-only event types. Merge-extensible: a plugin (e.g. compaction) declares extra event types via declaration merging. +The append-only event types. Merge-extensible: a plugin declares extra event types via declaration merging — e.g. the [compaction seam](compaction.md) adds `compact/start` / `compact/summary` / `compact/end`. ```ts type-equiv interface SessionEventMap { diff --git a/scripts/type-equiv.manifest.json b/scripts/type-equiv.manifest.json index 30165e2388..859e535095 100644 --- a/scripts/type-equiv.manifest.json +++ b/scripts/type-equiv.manifest.json @@ -39,6 +39,8 @@ { "doc": "docs/core-data-structures/bash.md", "symbol": "BashRunResult", "source": "packages/bash/bash/src/types.ts" }, { "doc": "docs/core-data-structures/bash.md", "symbol": "CollectedOutput", "source": "packages/bash/bash/src/types.ts" }, { "doc": "docs/core-data-structures/bash.md", "symbol": "BashTask", "source": "packages/bash/bash/src/types.ts" }, - { "doc": "docs/core-data-structures/bash.md", "symbol": "BashTaskRead", "source": "packages/bash/bash/src/types.ts" } + { "doc": "docs/core-data-structures/bash.md", "symbol": "BashTaskRead", "source": "packages/bash/bash/src/types.ts" }, + + { "doc": "docs/core-data-structures/compaction.md", "symbol": "CompactionResult", "source": "packages/compact/compact/src/types.ts" } ] } From 894653763eafec453e78bdf4d7f1810adc3e731b Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Tue, 23 Jun 2026 16:40:24 +0800 Subject: [PATCH 13/17] docs(compact): add the compact group/service to package and architecture docs --- docs/architecture.md | 3 ++- packages/README.md | 3 +++ packages/compact/README.md | 11 +++++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 packages/compact/README.md diff --git a/docs/architecture.md b/docs/architecture.md index 2b05e613b1..fb2d594283 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -53,6 +53,7 @@ Dependency rule: **extension** plugins depend on interface packages, never on `d | `ctx.agents` | `AgentRegistry` | dsh-agent | live `Agent` handles + the create/resume factory seam (returns an `AgentHandle` = `{ agent, dispose() }` for owned per-agent teardown) | | `ctx.agentLoop` | `AgentLoop` | dsh-agent-loop | creates `ReactLoopAgent`s and drives their loops | | `ctx.bash` | `BashExecutor` (abstract) | dsh-bash | bash execution seam: foreground runs + background tasks | +| `ctx.compact` | `CompactService` (abstract) | dsh-compact | compaction seam: decide when history is too large, summarize an older range into a single surface node | All registrations (`registerAdapter`, `section`, `tools`, `register`, …) go through `ctx.effect()` and return disposers, so plugin hot-reload (vendored HMR) and fiber disposal clean up automatically. @@ -191,7 +192,7 @@ Every MVP feature (including the TODO-marked ones), with the mechanism that impl | `/loop` | on `agent/turn-end`, `send()` the next iteration; or force-continue | | Dynamic workflow | orchestrator plugin on `agent/turn-end` / `agent/step-end` driving `send`/`steer` (+ sub-agents later) | | Queued + steering messages | core `Agent.send()` / `Agent.steer()` | -| Context compaction (auto + manual) | wrap `agent/request`: measure tokens, rewrite `req.messages`, append merged `compaction/*` session events; manual = a command plugin invoking the same routine | +| Context compaction (auto + manual) | the `ctx.compact` seam ([dsh-compact](../packages/compact/compact)): a backend summarizes an older surface range into a single `user/message` `replace` op, bracketed by log-only `compact/*` events; auto = check token pressure at turn boundaries, manual = a `/compact` tool. See the [compaction capability-seam RFC](rfc/proposed/feature/2026-06-18-compaction-capability-seam.md) | | System prompt configurability | `ctx.systemPrompt.section()` with ordering | | AGENTS.md (root) | a section provider reading the file | | AGENTS.md (subdir, on-touch) + file-change notices | `agent.inject()` from a watcher / tool-result listener | diff --git a/packages/README.md b/packages/README.md index f39d7b70a1..78eefbac9d 100644 --- a/packages/README.md +++ b/packages/README.md @@ -11,6 +11,7 @@ Packages are grouped by modular role at `packages///`. The group dir | [`core/`](core/README.md) | Product API spine: session, system-prompt, tools, agent, and the concrete loop | Product — stable surface | | [`llm/`](llm/README.md) | LLM capability family: the abstract service + provider adapters | Product — stable surface | | [`bash/`](bash/README.md) | Bash capability family: the executor seam, a local impl, and the model-facing tool | Product — stable surface | +| [`compact/`](compact/README.md) | Compaction capability family: the abstract seam (backend + tool deferred) | Product — stable surface | | [`session-persistence/`](session-persistence/README.md) | Persistence capability family: the seam + JSONL/SQLite backends | Product — stable surface | | [`ui/`](ui/README.md) | Editor/client integration surfaces (the ACP bridge) | Product — stable surface | | [`support/`](support/README.md) | Dev/test/example infrastructure (invariants, stdio UI, replay adapter) | Support — lower compatibility expectations | @@ -27,6 +28,7 @@ dsh-bash ← dsh-brand (abstract executor seam; b dsh-session ← dsh-llm, dsh-brand dsh-system-prompt ← dsh-llm dsh-agent ← dsh-llm, dsh-session, dsh-brand +dsh-compact ← dsh-session, dsh-llm (abstract compaction seam; backend + tool deferred) dsh-tools ← dsh-llm, dsh-system-prompt, dsh-agent dsh-bash-local ← dsh-bash (BashExecutor impl) dsh-tool-bash ← dsh-bash, dsh-tools (bash tool schemas) @@ -58,6 +60,7 @@ The rule: **extension** plugins depend on interfaces, never on the concrete loop | `bash/` | `bash` | Abstract bash executor seam (interface + vocabulary) | `ctx.bash` | | `bash-local/` | `bash` | Local-subprocess `BashExecutor` implementation | (registers `ctx.bash`) | | `tool-bash/` | `bash` | Model-facing `bash`/`bash_output`/`bash_kill` tool schemas | (registers on `ctx.tools`) | +| `compact/` | `compact` | Abstract compaction seam + `compact/*` events + `CompactionResult` | `ctx.compact` | | `llm-deepseek/` | `llm` | DeepSeek API adapter (hand-rolled fetch/SSE) | (registers on `ctx.llm`) | | `llm-pi-ai/` | `llm` | DeepSeek adapter via `@earendil-works/pi-ai` (design twin) | (registers on `ctx.llm`) | | `session-persistence/` | `session-persistence` | Persistence seam + write coordinator | `ctx.sessionPersistence` | diff --git a/packages/compact/README.md b/packages/compact/README.md new file mode 100644 index 0000000000..0d63b3b8cd --- /dev/null +++ b/packages/compact/README.md @@ -0,0 +1,11 @@ +# compact/ — compaction capability family + +A three-package capability seam (see [capability seams](../../docs/rfc/implemented/architecture/2026-06-13-capability-seams.md)): an abstract compaction interface, a backend that summarizes, and the model-facing tool that consumes it. Only the interface tier exists today; the backend and consumer are deferred. All **product** packages. + +| Package | Role | ctx key | +|---|---|---| +| `compact/` | Abstract compaction seam (interface + `compact/*` events + `CompactionResult`) | `ctx.compact` | +| `compact-basic/` (deferred) | A backend: char/4 estimation + token-budget retention + `llm.stream()` summarization | (registers `ctx.compact`) | +| `tool-compact/` (deferred) | Model-facing `/compact` tool over `ctx.compact` | (registers on `ctx.tools`) | + +The interface lives at `compact/compact/`. Unlike the bash seam, it depends on `dsh-session` and `dsh-llm` — its verbs are defined over a `Session` and its output is the `ContentBlock` vocabulary, so the contract cannot be expressed without naming them. That deviation from the "interface depends only on cordis" guidance is intentional and recorded in the [compaction capability-seam RFC](../../docs/rfc/proposed/feature/2026-06-18-compaction-capability-seam.md). A tokenizer- or template-based backend would replace `compact-basic` without touching the interface or the tool. From 442f85469e56798382d65786079fbee9ef773a23 Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Tue, 23 Jun 2026 16:51:20 +0800 Subject: [PATCH 14/17] refactor(compact): align compact/summary and CompactionResult on shadowed* naming --- docs/core-data-structures/compaction.md | 4 ++-- .../feature/2026-06-18-compaction-capability-seam.md | 2 +- packages/compact/compact/README.md | 2 +- packages/compact/compact/src/types.ts | 8 ++++---- packages/compact/compact/tests/compact.spec.ts | 8 ++++---- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/core-data-structures/compaction.md b/docs/core-data-structures/compaction.md index 75900bc5db..9bdb987f81 100644 --- a/docs/core-data-structures/compaction.md +++ b/docs/core-data-structures/compaction.md @@ -11,7 +11,7 @@ Compaction extends [`SessionEventMap`](session.md) with three event types via de | Event | Payload | Role | |---|---|---| | `compact/start` | `{ turn }` | acquires the log-recorded lock | -| `compact/summary` | `{ summary, compactedRange, compactedEventSeqs, tokenCount }` | provenance: the summary blocks, the shadowed seq range, and the estimated token count | +| `compact/summary` | `{ summary, shadowedRange, shadowedSeqs, shadowedTokenCount }` | provenance: the summary blocks, the shadowed seq range, and the estimated token count | | `compact/end` | `{ turn, error? }` | releases the lock (`error` set when summarization threw) | The lock brackets the **whole** operation: `compact/start` is appended first, then summarization, the `compact/summary` provenance record, and the `user/message` replacement all land, and only then `compact/end`. Releasing the lock last turns a crash mid-operation into a detectable orphaned lock (a `compact/start` with no matching `compact/end`) rather than a `compact/end` that falsely claims compaction finished. @@ -37,7 +37,7 @@ interface CompactionResult { /** The seq numbers of all shadowed surface nodes. */ shadowedSeqs: number[] /** Estimated token count of the shadowed content. */ - compactedTokenCount: number + shadowedTokenCount: number } ``` diff --git a/docs/rfc/proposed/feature/2026-06-18-compaction-capability-seam.md b/docs/rfc/proposed/feature/2026-06-18-compaction-capability-seam.md index 1ddcd05b42..2d559fa65c 100644 --- a/docs/rfc/proposed/feature/2026-06-18-compaction-capability-seam.md +++ b/docs/rfc/proposed/feature/2026-06-18-compaction-capability-seam.md @@ -22,7 +22,7 @@ Per the [capability-seams RFC](../../implemented/architecture/2026-06-13-capabil ### The contract depends on `dsh-session` and `dsh-llm` — a deliberate deviation -The capability-seams RFC states the interface package "depends only on cordis" (true of `dsh-bash`, whose vocabulary is self-contained). Compaction **cannot** honor that: its verbs are defined *over* a `Session` (`compactRegion(session, start, end)`) and its output *is* the content vocabulary (`CompactionResult.summary: ContentBlock[]`). There is no way to express the contract without naming `Session`/`SessionEvent` (from `dsh-session`) and `ContentBlock` (from `dsh-llm`). +The capability-seams RFC states the interface package "depends only on cordis" (true of `dsh-bash`, whose vocabulary is self-contained). Compaction **cannot** honor that: its verbs are defined *over* a `Session` (`compactRegion(session, start, end)`) and its output *is* the content vocabulary (`CompactionResult.summary: ContentBlock[]`). There is no way to express the contract without naming `Session`/`SessionEvent` (from `dsh-session`) and `ContentBlock` (from `dsh-llm`). This is not a coupling smell — it is the contract's domain. The "only cordis" guidance was always shorthand for "the interface depends only on what the contract genuinely names, and never on an implementation." `dsh-session` and `dsh-llm` are themselves interface/vocabulary packages, not implementations; `dsh-compact` still imports no backend. The seam's real invariant — *consumers and implementations evolve independently behind an abstract service* — holds intact. We record the deviation here so a future reader doesn't mistake it for an accident or "fix" it by smuggling `Session` behind an opaque handle. diff --git a/packages/compact/compact/README.md b/packages/compact/compact/README.md index a6e2176bd8..9ef5b73005 100644 --- a/packages/compact/compact/README.md +++ b/packages/compact/compact/README.md @@ -48,7 +48,7 @@ The `compact/*` events extend `SessionEventMap` (merge-extensible) via declarati | Event | Payload | On surface? | |---|---|---| | `compact/start` | `{ turn }` | no (log-only) | -| `compact/summary` | `{ summary, compactedRange, compactedEventSeqs, tokenCount }` | no (log-only) | +| `compact/summary` | `{ summary, shadowedRange, shadowedSeqs, shadowedTokenCount }` | no (log-only) | | `compact/end` | `{ turn, error? }` | no (log-only) | ## Implementing a backend diff --git a/packages/compact/compact/src/types.ts b/packages/compact/compact/src/types.ts index 08b37ef4c6..36dd5fb629 100644 --- a/packages/compact/compact/src/types.ts +++ b/packages/compact/compact/src/types.ts @@ -29,9 +29,9 @@ declare module '@deepseek-ai/dsh-session' { */ 'compact/summary': { summary: ContentBlock[] - compactedRange: { startSeq: number; endSeq: number } - compactedEventSeqs: number[] - tokenCount: number + shadowedRange: { start: number; end: number } + shadowedSeqs: number[] + shadowedTokenCount: number } /** Marks the end of a compaction — log-only, releases the lock. `error` set if summarization failed. */ 'compact/end': { turn: number; error?: string } @@ -53,5 +53,5 @@ export interface CompactionResult { /** The seq numbers of all shadowed surface nodes. */ shadowedSeqs: number[] /** Estimated token count of the shadowed content. */ - compactedTokenCount: number + shadowedTokenCount: number } diff --git a/packages/compact/compact/tests/compact.spec.ts b/packages/compact/compact/tests/compact.spec.ts index 4a758f4364..b3ad9d1501 100644 --- a/packages/compact/compact/tests/compact.spec.ts +++ b/packages/compact/compact/tests/compact.spec.ts @@ -36,9 +36,9 @@ class StubCompactService extends CompactService { const startEvent = session.append('compact/start', { turn: 0 }) const summaryEvent = session.append('compact/summary', { summary: [{ type: 'text', text: 'stub' }], - compactedRange: { startSeq: start, endSeq: end }, - compactedEventSeqs: [], - tokenCount: 0, + shadowedRange: { start, end }, + shadowedSeqs: [], + shadowedTokenCount: 0, }) const endEvent = session.append('compact/end', { turn: 0 }) return { @@ -48,7 +48,7 @@ class StubCompactService extends CompactService { summary: [{ type: 'text', text: 'stub' }], shadowedRange: { start, end }, shadowedSeqs: [], - compactedTokenCount: 0, + shadowedTokenCount: 0, } } } From 828c3f85c9d61a5bcea7ccb894feb9896343de33 Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Wed, 24 Jun 2026 17:45:48 +0800 Subject: [PATCH 15/17] fix review findings: skip collided SCHEMA_VERSION 3; reject marker-less surface events MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P1: both merge parents shipped SCHEMA_VERSION=3 for different layouts (surface columns vs seed_length), so an on-disk 3 was ambiguous and wrongly accepted. Bump to 4 (merged layout) so the version check rejects both sibling v3s. P2: a surface-eligible event with no surfaceOp lands in the log but vanishes from deriveMessages() (surface is the sole derivation path). The typed append overload enforces the marker only when the type arg is a literal; it collapses to optional when widened to the union (a caller iterating raw events). Guard at runtime in both append() and the seed constructor — no backward-compat for surface-less logs. Shared seed fixtures carry surfaceOp explicitly and the appendLog helper forwards it verbatim (no synthesized default). Exports isSurfaceEligibleType. Regression tests for all three, each verified to fail on the unfixed code. Gates: typecheck, test (1115), snapshot (14), doc-sync, lint, build, hygiene green. --- docs/cordis-catalog/events-and-services.md | 2 +- .../2026-06-18-session-surface.md | 4 ++- ...6-06-22-fork-child-replay-seed-boundary.md | 2 +- packages/core/session/README.md | 5 +-- packages/core/session/src/index.ts | 27 +++++++++++++-- packages/core/session/src/surface.ts | 12 +++++++ .../core/session/tests/properties.spec.ts | 27 ++++++++++----- packages/core/session/tests/session.spec.ts | 33 +++++++++++++++++-- .../tests/jsonl.spec.ts | 6 ++-- .../session-persistence-sqlite/src/schema.ts | 14 +++++--- .../tests/sqlite.spec.ts | 32 +++++++++++++++--- .../session-persistence/tests/contract.ts | 32 ++++++++++++++++-- .../tests/coordinator-contract.ts | 4 +-- .../invariants/tests/invariants.spec.ts | 2 +- 14 files changed, 166 insertions(+), 36 deletions(-) diff --git a/docs/cordis-catalog/events-and-services.md b/docs/cordis-catalog/events-and-services.md index 737a4cc8ef..bac283b1f8 100644 --- a/docs/cordis-catalog/events-and-services.md +++ b/docs/cordis-catalog/events-and-services.md @@ -412,7 +412,7 @@ get(id: SessionId): Session | undefined list(): Session[] ``` -Source: [`packages/core/session/src/index.ts:300`](../../packages/core/session/src/index.ts) +Source: [`packages/core/session/src/index.ts:321`](../../packages/core/session/src/index.ts) ### `ctx.subagents` — `SubagentService` diff --git a/docs/rfc/implemented/architecture/2026-06-18-session-surface.md b/docs/rfc/implemented/architecture/2026-06-18-session-surface.md index 0ee163cbcf..3f4419b5ca 100644 --- a/docs/rfc/implemented/architecture/2026-06-18-session-surface.md +++ b/docs/rfc/implemented/architecture/2026-06-18-session-surface.md @@ -51,9 +51,11 @@ The `repair.ts` module synthesizes `tool/result` closers for orphaned tool calls The dev-mode invariants plugin validates: `sourceEventSeqs` references (non-empty, no duplicates, references earlier events, references known seqs) and `surfaceOp` (replace `start ≤ end`, both endpoints are on the tracked surface, the range is non-reversed in surface position, and `sourceEventSeqs` includes every node the range shadows). +Because the surface is the SOLE derivation path, a surface-eligible event that carries no `surfaceOp` marker is invisible to `deriveMessages()` — it would land in the log yet silently drop from history on resume/fork. `append`'s typed overload makes the marker mandatory for `SurfaceEventType` events at compile time, but only when the type argument is a SPECIFIC literal; when it widens to the `SessionEventType` union (a caller iterating raw events, e.g. `for (const e of log) append(e.type, e.data)`) the conditional rest collapses to optional and the compiler stops enforcing it. The marker requirement is therefore ALSO checked at runtime in two places: `append` itself throws on a marker-less surface-eligible event (covering the union-widening loophole), and the `Session` seed constructor re-checks the same invariant (alongside its seq-contiguity and JSON-serializability checks) so a seed/load/fork — which arrives as raw `SessionEvent[]`, bypassing `append` — is REJECTED rather than constructing a session that resumes with missing history. (No backward-compat path for surface-less logs: per the pre-release stance there is no persisted user data to preserve, so such a log is rejected, not upgraded.) + ## Consequences -- **`packages/core/session`**: New `surface.ts` (`SurfaceManager`), new types (`SurfaceOp`, `SurfaceIntent`), new fields on `SessionEvent`, modified `append()` (third required `SurfaceIntent` param), refactored `deriveMessages()` (surface path + legacy fallback), surface-aware `repair.ts`. +- **`packages/core/session`**: New `surface.ts` (`SurfaceManager`), new types (`SurfaceOp`, `SurfaceIntent`), new fields on `SessionEvent`, modified `append()` (third required `SurfaceIntent` param), refactored `deriveMessages()` (walks the surface as the sole derivation path), surface-aware `repair.ts`. The seed constructor rejects a surface-eligible seed event missing its `surfaceOp` marker (see § Invariants). - **`packages/core/agent-loop`**: All surface-capable appends pass surface opts. Chunk seqs are collected for `assistant/message` provenance; `tool/call` seqs are captured for `tool/result` provenance. - **`packages/session-persistence/session-persistence-sqlite`**: Two new nullable TEXT columns (`source_event_seqs`, `surface_op`) on the `events` table; `SCHEMA_VERSION` bumped (bump-and-reject, no migration). - **`packages/support/invariants`**: Surface-related validation rules. diff --git a/docs/rfc/implemented/testing/2026-06-22-fork-child-replay-seed-boundary.md b/docs/rfc/implemented/testing/2026-06-22-fork-child-replay-seed-boundary.md index a45e62639b..a60d487e91 100644 --- a/docs/rfc/implemented/testing/2026-06-22-fork-child-replay-seed-boundary.md +++ b/docs/rfc/implemented/testing/2026-06-22-fork-child-replay-seed-boundary.md @@ -27,7 +27,7 @@ Record where a session's **inherited** prefix ends, persist it, and have the rep - **JSONL**: a `seedLength` field on the header line (`toHeaderLine`/`fromHeaderLine`). - **SQLite**: a `seed_length` column on the `sessions` table. -The SQLite change is a breaking table-layout change, so `SCHEMA_VERSION` bumps **2 → 3**. Per the repo's pre-release stance (§ "Pre-release stance" in AGENTS.md) the backend **rejects** a non-current `user_version` on open rather than migrating it — there is no persisted user data to preserve, so no migration code is written (the existing reject-not-migrate path at `openDatabase` already enforces this; v1 and now v2 are both rejected). +The SQLite change is a breaking table-layout change, so `SCHEMA_VERSION` bumps. This branch added `seed_length` under version **3**; it later merged with the session-surface branch, which had independently shipped its OWN version-3 layout (the `source_event_seqs`/`surface_op` columns). Because an on-disk `3` is ambiguous between the two sibling layouts, the merged build is version **4** (every column), and an on-disk `3` is rejected like any other non-current version. Per the repo's pre-release stance (§ "Pre-release stance" in AGENTS.md) the backend **rejects** a non-current `user_version` on open rather than migrating it — there is no persisted user data to preserve, so no migration code is written (the existing reject-not-migrate path at `openDatabase` already enforces this; v1, v2, and the collided v3 are all rejected). ### 3. Replay derives a child script after the boundary diff --git a/packages/core/session/README.md b/packages/core/session/README.md index 08323eff0c..2a90d0f792 100644 --- a/packages/core/session/README.md +++ b/packages/core/session/README.md @@ -34,7 +34,7 @@ Creates and holds event-sourced `Session` instances. Persistence is intentionall Plain class (not a Cordis Service). Create via `ctx.sessions.create()`. -- `session.append(type, data, opts?): SessionEvent` — synchronous, never blocks on I/O. **Throws** if `data` is not losslessly JSON-serializable (BigInt, function, symbol, undefined, non-finite number, circular ref, or an exotic object like Map/Set/Date) — the event log is the durable source of truth, so this invariant is enforced at the source (exported as `isJsonValue` for backends to reuse on their replay/fork entry points). A third parameter `opts: SurfaceIntent` carries surface metadata: `surfaceOp` controls how the event enters the surface linked list, and `sourceEventSeqs` records provenance (the seq numbers of events this one derives from). It is **required** for the five `SurfaceEventType` events (every message-producing event must declare how it joins the surface) and rejected by the compiler for non-surface types. +- `session.append(type, data, opts?): SessionEvent` — synchronous, never blocks on I/O. **Throws** if `data` is not losslessly JSON-serializable (BigInt, function, symbol, undefined, non-finite number, circular ref, or an exotic object like Map/Set/Date) — the event log is the durable source of truth, so this invariant is enforced at the source (exported as `isJsonValue` for backends to reuse on their replay/fork entry points). A third parameter `opts: SurfaceIntent` carries surface metadata: `surfaceOp` controls how the event enters the surface linked list, and `sourceEventSeqs` records provenance (the seq numbers of events this one derives from). It is **required** for the five `SurfaceEventType` events (every message-producing event must declare how it joins the surface) and rejected by the compiler for non-surface types. The marker requirement is enforced two ways: the typed overload makes `opts` mandatory when `type` is a specific `SurfaceEventType` literal, AND `append` **throws** at runtime if a surface-eligible event arrives with no `surfaceOp` — covering the case where `type` widens to the `SessionEventType` union (a caller iterating raw events, where the conditional overload collapses to optional) so a marker-less message event can never silently land in the log and vanish from `deriveMessages()`. - `session.deriveMessages(): Message[]` — derive the LLM message history by walking the surface linked list (skipping non-surface events like chunks and boundaries; a `replace` shadows the nodes it covers). The surface is the single source of derived history — there is no raw-log fallback. - `session.surface: SurfaceManager` — the derived surface, lazily rebuilt from `surfaceOp` markers in the log. Processes only new events (delta) on each access — the log is append-only, so prior events never change. - `session.events`, `session.seq`, `session.id` @@ -45,6 +45,7 @@ Plain class (not a Cordis Service). Create via `ctx.sessions.create()`. - `SurfaceOp` — how a surface node entered the linked list: `'append'` (normal tail append) or `{ op: 'replace', start, end }` (replace nodes from `start` through `end` inclusive — both must be valid surface node seqs; `start === end` replaces a single node). Used by compaction to shadow old nodes without deleting them. - `SurfaceIntent` — `{ surfaceOp: SurfaceOp; sourceEventSeqs?: number[] }`, the required third parameter to `session.append()` for surface-eligible types. - `SurfaceNode` — `{ seq: number; prev: number | null; next: number | null }`, one node in the surface linked list. +- `isSurfaceEvent(event)` / `isSurfaceEligibleType(type)` — the first narrows a `SessionEvent` to a fully-formed surface node (type is surface-eligible AND `surfaceOp` present); the second is the type-only check (is this one of the five `SurfaceEventType` values?), used to detect a surface-eligible event MISSING its marker — e.g. when validating a seed/load log. ### Session event vocabulary (`types.ts`) @@ -66,7 +67,7 @@ Every `SessionEvent` carries two optional top-level fields (structural metadata) ### Extension points - Persistence plugins: subscribe to `session/event` (write-behind) and drain on `session/flush` (awaited) and fiber dispose. A durable backend reads the log and reloads it into a live session; the metadata seam (`SessionHeader`, `session.header`) is what such a backend stores beside the log. -- Replay/fork: `ctx.sessions.create(id, { seed })` seeds a new session with an existing event log. The surface rebuilds deterministically from `surfaceOp` markers in the seeded events. +- Replay/fork: `ctx.sessions.create(id, { seed })` seeds a new session with an existing event log. The surface rebuilds deterministically from `surfaceOp` markers in the seeded events. The seed is validated to the SAME invariants `append` enforces — including that every surface-eligible event (`SurfaceEventType`) carries a `surfaceOp` marker — so a marker-less message event is rejected at construction rather than silently vanishing from `deriveMessages()` (the surface is the sole derivation path) on resume. - Compaction: a future plugin appends a new event with `surfaceOp: { op: 'replace', start, end }` to shadow old surface nodes. ### What is NOT here (TODO) diff --git a/packages/core/session/src/index.ts b/packages/core/session/src/index.ts index 8908059dae..2002c93051 100644 --- a/packages/core/session/src/index.ts +++ b/packages/core/session/src/index.ts @@ -12,13 +12,13 @@ import type { ContentBlock, Message, MessageSource } from '@deepseek-ai/dsh-llm' import { SESSION_FORMAT_VERSION, SessionId } from './types.ts' import type { CreateSessionOptions, SessionEvent, SessionEventMap, SessionEventType, SessionHeader, SurfaceIntent, SurfaceEventType } from './types.ts' import { isJsonValue } from './json.ts' -import { SurfaceManager } from './surface.ts' +import { SurfaceManager, isSurfaceEligibleType } from './surface.ts' export * from './types.ts' export { isJsonValue } from './json.ts' export { interruptedTurnClosers } from './repair.ts' export type { SurfaceNode } from './surface.ts' -export { isSurfaceEvent } from './surface.ts' +export { isSurfaceEvent, isSurfaceEligibleType } from './surface.ts' declare module 'cordis' { interface Context { @@ -120,6 +120,16 @@ export class Session { if (!isJsonValue(event.data)) { throw new Error(`seed event "${event.type}" (seq ${event.seq}) carries non-JSON-serializable data`) } + // Surface-eligible events MUST carry a surfaceOp marker — the surface is + // the sole source of derived history, so a marker-less message event + // would load fine yet vanish from deriveMessages(). `append` enforces + // this at compile time via its typed overload; a seed arrives as raw + // SessionEvent[] (replay/fork/load), bypassing that, so re-check at + // runtime here rather than silently resuming with empty history. + if (isSurfaceEligibleType(event.type) + && (event as SessionEvent).surfaceOp === undefined) { + throw new Error(`seed event "${event.type}" (seq ${event.seq}) is surface-eligible but carries no surfaceOp marker`) + } }) // Deep-clone each seed event, NOT just the array: the seed events and // their `data` are still owned by the caller (or the source session of a @@ -172,6 +182,18 @@ export class Session { if (!isJsonValue(data)) { throw new Error(`session event "${type}" carries non-JSON-serializable data`) } + const surfaceOpts: SurfaceIntent | undefined = opts[0] + // Surface-eligible events MUST carry a surfaceOp marker — the surface is the + // sole source of derived history, so a marker-less message event would be + // logged yet vanish from deriveMessages(). The typed `opts` overload makes + // the marker mandatory only when `T` is a SPECIFIC SurfaceEventType literal; + // when `T` widens to the SessionEventType union (a caller iterating raw + // events: `for (const e of log) append(e.type, e.data)`), the conditional + // rest collapses to optional and the compiler stops enforcing it. Re-check + // at runtime so that loophole can't silently drop history. + if (isSurfaceEligibleType(type) && surfaceOpts?.surfaceOp === undefined) { + throw new Error(`session event "${type}" is surface-eligible and requires a surfaceOp marker`) + } // Snapshot `data` into the log, NOT the caller's reference: the validation // above proves it is JSON-serializable AT THIS MOMENT, but the caller still // owns the object and could mutate it afterwards (before a persistence @@ -185,7 +207,6 @@ export class Session { // Surface metadata is snapshot separately: sourceEventSeqs (number[] — // primitives, so array spread is a complete copy) and surfaceOp (a string // primitive, or cloned if it's a replace object). - const surfaceOpts: SurfaceIntent | undefined = opts[0] // Build the event shape with conditional surface fields via spreading. // The result is cast through `unknown` because the conditional spreads // produce an intersection type that the assignability checker can't diff --git a/packages/core/session/src/surface.ts b/packages/core/session/src/surface.ts index 3615668674..57f9a65864 100644 --- a/packages/core/session/src/surface.ts +++ b/packages/core/session/src/surface.ts @@ -22,6 +22,18 @@ const SURFACE_EVENT_TYPES = new Set([ 'steering/message', ]) +/** + * Whether an event's `type` is surface-eligible (one of the five + * message-producing {@link SurfaceEventType} values). This is the TYPE check + * only — it does NOT require `surfaceOp` to be present. Use it to detect a + * surface-eligible event that is MISSING its mandatory marker (e.g. validating + * a seed/load log); use {@link isSurfaceEvent} to narrow to a fully-formed + * {@link SurfaceEvent} with `surfaceOp` present. + */ +export function isSurfaceEligibleType(type: string): boolean { + return SURFACE_EVENT_TYPES.has(type) +} + /** * Narrow a {@link SessionEvent} to {@link SurfaceEvent}: checks that the * event's `type` is surface-eligible AND that `surfaceOp` is present. diff --git a/packages/core/session/tests/properties.spec.ts b/packages/core/session/tests/properties.spec.ts index 42149515f2..4be0cbcf6d 100644 --- a/packages/core/session/tests/properties.spec.ts +++ b/packages/core/session/tests/properties.spec.ts @@ -11,22 +11,29 @@ import { describe, expect, it } from 'vitest' import fc from 'fast-check' import { CallId } from '@deepseek-ai/dsh-llm' import { Session, SessionId } from '@deepseek-ai/dsh-session' -import type { SessionEventMap, SessionEventType } from '@deepseek-ai/dsh-session' +import type { SessionEventMap, SessionEventType, SurfaceIntent } from '@deepseek-ai/dsh-session' -type Appendable = { [T in SessionEventType]: { type: T; data: SessionEventMap[T] } }[SessionEventType] +// An appendable event: its type/data plus, for surface-eligible types, the +// explicit surface intent the generator declares (mirroring how a real caller +// passes it). The intent is part of the generated fixture, NOT synthesized by +// `build`, so each arbitrary states the marker it produces. +type Appendable = { + [T in SessionEventType]: { type: T; data: SessionEventMap[T]; intent?: SurfaceIntent } +}[SessionEventType] const textContentArb = fc.array( fc.record({ type: fc.constant<'text'>('text'), text: fc.string() }), { maxLength: 3 }, ) -// A message-producing event (these DO affect derived history). +// A message-producing event (these DO affect derived history). Each carries an +// explicit `surfaceOp: 'append'` intent — the marker the real loop passes. const messageEventArb: fc.Arbitrary = fc.oneof( - textContentArb.map((content): Appendable => ({ type: 'user/message', data: { content, source: { kind: 'user' } } })), - textContentArb.map((content): Appendable => ({ type: 'assistant/message', data: { turn: 1, step: 1, content } })), - textContentArb.map((content): Appendable => ({ type: 'assistant/message', data: { turn: 1, step: 1, content, usage: { inputTokens: 1, outputTokens: 1 } } })), + textContentArb.map((content): Appendable => ({ type: 'user/message', data: { content, source: { kind: 'user' } }, intent: { surfaceOp: 'append' } })), + textContentArb.map((content): Appendable => ({ type: 'assistant/message', data: { turn: 1, step: 1, content }, intent: { surfaceOp: 'append' } })), + textContentArb.map((content): Appendable => ({ type: 'assistant/message', data: { turn: 1, step: 1, content, usage: { inputTokens: 1, outputTokens: 1 } }, intent: { surfaceOp: 'append' } })), fc.record({ id: fc.string({ minLength: 1 }), content: textContentArb, isError: fc.boolean() }) - .map((r): Appendable => ({ type: 'tool/result', data: { turn: 1, step: 1, callId: CallId(r.id), content: r.content, isError: r.isError } })), + .map((r): Appendable => ({ type: 'tool/result', data: { turn: 1, step: 1, callId: CallId(r.id), content: r.content, isError: r.isError }, intent: { surfaceOp: 'append' } })), ) // A non-message event (trace/replay data — must NOT affect derived history). @@ -44,7 +51,11 @@ const logArb = fc.array(anyEventArb, { maxLength: 25 }) let counter = 0 function build(events: Appendable[]): Session { const session = new Session(SessionId(`prop-${counter++}`)) - for (const e of events) session.append(e.type, e.data) + for (const e of events) { + // Forward the generated intent verbatim; non-surface events carry none. + if (e.intent !== undefined) session.append(e.type, e.data, e.intent) + else session.append(e.type, e.data) + } return session } diff --git a/packages/core/session/tests/session.spec.ts b/packages/core/session/tests/session.spec.ts index 46f96742cc..6138a479f1 100644 --- a/packages/core/session/tests/session.spec.ts +++ b/packages/core/session/tests/session.spec.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest' import { Context } from 'cordis' import { CallId } from '@deepseek-ai/dsh-llm' import SessionStore, { SESSION_FORMAT_VERSION, Session, SessionEvent, SessionId } from '@deepseek-ai/dsh-session' +import type { SessionEventType } from '@deepseek-ai/dsh-session' describe('Session', () => { it('derives message history from the event log', () => { @@ -120,6 +121,21 @@ describe('Session', () => { expect(session.events).toHaveLength(0) }) + it('rejects a surface-eligible append with no surfaceOp marker (runtime guard for the union-widening loophole)', () => { + const session = new Session(SessionId('s5b')) + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + // The typed overload makes surfaceOp mandatory only when the type argument is + // a SPECIFIC SurfaceEventType literal. A caller iterating raw events widens it + // to the SessionEventType union, where the conditional rest collapses to + // optional — the exact shape `for (const e of log) append(e.type, e.data)` + // produces. Reproduce that here and assert the runtime guard rejects it. + const widenedType = 'user/message' as SessionEventType + expect(() => session.append(widenedType, { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } })) + .toThrow(/surface-eligible and requires a surfaceOp marker/) + // The rejected append never entered the log (only turn/start is present). + expect(session.events).toHaveLength(1) + }) + it('accepts dense arrays and nested plain objects', () => { const session = new Session(SessionId('s6')) expect(() => session.append('user/message', { content: [{ type: 'text', text: 'x' }], source: { kind: 'user' }, extra: [1, 2, [3, { a: null, b: true }]] } as never, { surfaceOp: 'append' })).not.toThrow() @@ -143,10 +159,23 @@ describe('Session', () => { expect(() => new Session(SessionId('seed-gap'), gapSeed)).toThrow(/contiguous|seq/) }) + it('validates seed events: rejects a surface-eligible event missing its surfaceOp marker', () => { + // A surface-eligible event (user/message) with no surfaceOp would load fine + // but vanish from deriveMessages() (the surface is the sole derivation path), + // so a resume/fork would silently lose history. append() forbids this at + // compile time; a raw seed must be rejected at runtime to match. + const markerlessSeed = [ + { type: 'turn/start' as const, seq: 0, time: 1, data: { turn: 1, trigger: { kind: 'message' as const, source: { kind: 'user' as const } } } }, + { type: 'user/message' as const, seq: 1, time: 2, data: { content: [{ type: 'text' as const, text: 'hi' }], source: { kind: 'user' as const } } }, + { type: 'turn/end' as const, seq: 2, time: 3, data: { turn: 1, reason: { kind: 'completed' as const } } }, + ] as SessionEvent[] + expect(() => new Session(SessionId('seed-no-marker'), markerlessSeed)).toThrow(/surface-eligible but carries no surfaceOp/) + }) + it('accepts a well-formed contiguous serializable seed', () => { const goodSeed = [ { type: 'turn/start' as const, seq: 0, time: 1, data: { turn: 1, trigger: { kind: 'message' as const, source: { kind: 'user' as const } } } }, - { type: 'user/message' as const, seq: 1, time: 2, data: { content: [{ type: 'text' as const, text: 'hi' }], source: { kind: 'user' as const } } }, + { type: 'user/message' as const, seq: 1, time: 2, data: { content: [{ type: 'text' as const, text: 'hi' }], source: { kind: 'user' as const } }, surfaceOp: 'append' as const }, { type: 'turn/end' as const, seq: 2, time: 3, data: { turn: 1, reason: { kind: 'completed' as const } } }, ] as SessionEvent[] const session = new Session(SessionId('seed-ok'), goodSeed) @@ -156,7 +185,7 @@ describe('Session', () => { it('snapshots the seed: mutating the original after construction does not affect session.events', () => { const seed = [ { type: 'turn/start' as const, seq: 0, time: 1, data: { turn: 1, trigger: { kind: 'message' as const, source: { kind: 'user' as const } } } }, - { type: 'user/message' as const, seq: 1, time: 2, data: { content: [{ type: 'text' as const, text: 'original' }], source: { kind: 'user' as const } } }, + { type: 'user/message' as const, seq: 1, time: 2, data: { content: [{ type: 'text' as const, text: 'original' }], source: { kind: 'user' as const } }, surfaceOp: 'append' as const }, { type: 'turn/end' as const, seq: 2, time: 3, data: { turn: 1, reason: { kind: 'completed' as const } } }, ] as SessionEvent[] const session = new Session(SessionId('seed-snapshot'), seed) diff --git a/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts b/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts index 7df0fd5180..c6b7903357 100644 --- a/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts +++ b/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts @@ -7,7 +7,7 @@ import SessionStore, { SessionId } from '@deepseek-ai/dsh-session' import type { Session, SessionEvent } from '@deepseek-ai/dsh-session' import SessionPersistenceJsonl from '@deepseek-ai/dsh-session-persistence-jsonl' import { encodeSegment, logPath, scanLog, sessionDir } from '../src/format.ts' -import { runPersistenceContract, meta, oneTurnLog } from '../../session-persistence/tests/contract.ts' +import { runPersistenceContract, meta, oneTurnLog, appendLog } from '../../session-persistence/tests/contract.ts' import { runCoordinatorContract, type CoordinatorFixture } from '../../session-persistence/tests/coordinator-contract.ts' let root: string @@ -121,7 +121,7 @@ describe('SessionPersistenceJsonl: durability and crash semantics', () => { { type: 'step/start', seq: 1, time: 2, data: { turn: 1, step: 1 } }, { type: 'assistant/chunk', seq: 2, time: 3, data: { turn: 1, step: 1, chunk: { type: 'text-delta', index: 0, text: 'he' } } }, { type: 'assistant/chunk', seq: 3, time: 4, data: { turn: 1, step: 1, chunk: { type: 'text-delta', index: 0, text: 'llo' } } }, - { type: 'assistant/message', seq: 4, time: 5, data: { turn: 1, step: 1, content: [{ type: 'text', text: 'hello' }] } }, + { type: 'assistant/message', seq: 4, time: 5, data: { turn: 1, step: 1, content: [{ type: 'text', text: 'hello' }] }, surfaceOp: 'append', sourceEventSeqs: [2, 3] }, { type: 'step/end', seq: 5, time: 6, data: { turn: 1, step: 1 } }, { type: 'turn/end', seq: 6, time: 7, data: { turn: 1, reason: { kind: 'completed' } } }, ] @@ -452,7 +452,7 @@ describe('SessionPersistenceJsonl: edge cases', () => { // Session A materializes a log under id "reuse". const sessFiberA = await ctx.plugin(Object.assign((inner: Context) => { const a = inner.sessions.create(SessionId('reuse'), { meta: { cwd: '/a' } }) - for (const e of oneTurnLog()) a.append(e.type, e.data) + appendLog(a, oneTurnLog()) }, { inject: ['sessions'] })) // Drain A, then dispose ITS fiber (the live session A is gone) while the // backend stays loaded. diff --git a/packages/session-persistence/session-persistence-sqlite/src/schema.ts b/packages/session-persistence/session-persistence-sqlite/src/schema.ts index 37acc6f0f0..d8db0e087b 100644 --- a/packages/session-persistence/session-persistence-sqlite/src/schema.ts +++ b/packages/session-persistence/session-persistence-sqlite/src/schema.ts @@ -15,7 +15,7 @@ import type { SessionEvent, SessionId, SessionHeader, SurfaceOp } from '@deepsee * layout; orthogonal to a session's own `version` (which versions the EVENT * vocabulary, stored per session in the `sessions` row). */ -export const SCHEMA_VERSION = 3 +export const SCHEMA_VERSION = 4 /** * A row of the `sessions` table — the out-of-log metadata ({@link SessionHeader}). @@ -56,9 +56,15 @@ export interface EventRow { * current {@link SCHEMA_VERSION}; an existing database whose version is NOT the * current one (written by a different, incompatible build — older or newer) is * REJECTED rather than opened against a layout this build does not understand. - * There are no migrations: an earlier layout (v1's different `sessions` shape, - * v2 without the `seed_length`/`source_event_seqs`/`surface_op` columns) is not - * upgraded in place — it is rejected. + * There are no migrations: an earlier layout is not upgraded in place — it is + * rejected. v1 had a different `sessions` shape; v2 lacked all of + * `seed_length`/`source_event_seqs`/`surface_op`. v3 is SKIPPED: two unmerged + * branches each shipped a DISTINCT v3 (one adding only `seed_length`, the other + * adding only the surface columns), so an on-disk v3 is ambiguous — it could be + * either sibling layout, neither of which has all of this build's columns. v4 + * is the merged layout carrying every column; bumping past the collided v3 + * makes the version check reject both sibling v3 databases instead of opening + * one against columns it does not have. */ export function openDatabase(path: string): DatabaseSync { const db = new DatabaseSync(path) diff --git a/packages/session-persistence/session-persistence-sqlite/tests/sqlite.spec.ts b/packages/session-persistence/session-persistence-sqlite/tests/sqlite.spec.ts index 0b8b83cd4e..7138718aa5 100644 --- a/packages/session-persistence/session-persistence-sqlite/tests/sqlite.spec.ts +++ b/packages/session-persistence/session-persistence-sqlite/tests/sqlite.spec.ts @@ -7,7 +7,7 @@ import SessionStore, { SessionId } from '@deepseek-ai/dsh-session' import type { Session, SessionEvent, SurfaceEvent, SurfaceEventType } from '@deepseek-ai/dsh-session' import SessionPersistenceSqlite, { SCHEMA_VERSION } from '@deepseek-ai/dsh-session-persistence-sqlite' import { openDatabase, rowToEvent, scanRows, type EventRow } from '../src/schema.ts' -import { runPersistenceContract, meta, oneTurnLog } from '../../session-persistence/tests/contract.ts' +import { runPersistenceContract, meta, oneTurnLog, appendLog } from '../../session-persistence/tests/contract.ts' import { runCoordinatorContract, type CoordinatorFixture } from '../../session-persistence/tests/coordinator-contract.ts' const dirs: string[] = [] @@ -66,9 +66,17 @@ runCoordinatorContract('sqlite', async (): Promise => { describe('scanRows', () => { // scanRows works off EventRows (data is a JSON string column); build them from - // SessionEvents so the unit tests read in terms of the event vocabulary. + // SessionEvents so the unit tests read in terms of the event vocabulary. Surface + // fields are serialized to their nullable columns so a round trip is faithful. const rows = (events: SessionEvent[]): EventRow[] => - events.map(e => ({ seq: e.seq, type: e.type, time: e.time, data: JSON.stringify(e.data), source_event_seqs: null, surface_op: null })) + events.map((e) => { + const se = e as SessionEvent + return { + seq: e.seq, type: e.type, time: e.time, data: JSON.stringify(e.data), + source_event_seqs: se.sourceEventSeqs !== undefined ? JSON.stringify(se.sourceEventSeqs) : null, + surface_op: se.surfaceOp !== undefined ? JSON.stringify(se.surfaceOp) : null, + } + }) it('preserves the full log when it ends exactly on a turn/end (no torn tail)', () => { const { preserved, tornFrom } = scanRows(rows(oneTurnLog())) @@ -246,6 +254,20 @@ describe('SessionPersistenceSqlite: durability and crash semantics', () => { expect(() => openDatabase(olderPath)).toThrow(/incompatible with this build/) }) + it('rejects a sibling v3 database (the merge-collided version) rather than opening it against missing columns', async () => { + // Two unmerged branches each shipped a DISTINCT layout under user_version 3 + // (one added only `seed_length`, the other only the surface columns). The + // merged build is v4; an on-disk v3 is ambiguous and is missing at least one + // of this build's columns, so it MUST be rejected, not opened. Stamp a v3 + // database and confirm the version check refuses it. + const path = await freshDbPath() + openDatabase(path).close() // creates + stamps user_version = SCHEMA_VERSION (4) + const db = openDatabase(path) + db.exec('PRAGMA user_version = 3') + db.close() + expect(() => openDatabase(path)).toThrow(/schema version 3, incompatible with this build/) + }) + it('a corrupt-JSON row in the uncommitted tail is discarded on load, not unloadable', async () => { const path = await freshDbPath() const m = meta('corrupt-tail') @@ -315,7 +337,7 @@ describe('SessionPersistenceSqlite: durability and crash semantics', () => { }) it('exposes the schema version constant', () => { - expect(SCHEMA_VERSION).toBe(3) + expect(SCHEMA_VERSION).toBe(4) }) }) @@ -353,7 +375,7 @@ describe('SessionPersistenceSqlite: edge cases', () => { // Instance 1 materializes a session and disposes. const b1 = await backend(path) const s1 = b1.ctx.sessions.create(SessionId('hmr-collide')) - for (const e of oneTurnLog()) s1.append(e.type, e.data) + appendLog(s1, oneTurnLog()) await b1.ctx.parallel('session/flush', s1) await b1.dispose() diff --git a/packages/session-persistence/session-persistence/tests/contract.ts b/packages/session-persistence/session-persistence/tests/contract.ts index a0f0e7bfa0..9f2facb827 100644 --- a/packages/session-persistence/session-persistence/tests/contract.ts +++ b/packages/session-persistence/session-persistence/tests/contract.ts @@ -10,7 +10,7 @@ import { describe, expect, it } from 'vitest' import { SESSION_FORMAT_VERSION, SessionId } from '@deepseek-ai/dsh-session' -import type { SessionEvent, SessionHeader } from '@deepseek-ai/dsh-session' +import type { Session, SessionEvent, SessionHeader, SurfaceEventType, SurfaceIntent } from '@deepseek-ai/dsh-session' import { CallId } from '@deepseek-ai/dsh-llm' import type { SessionPersistence } from '../src/index.ts' @@ -34,14 +34,40 @@ export function meta(id: string, cwd?: string): SessionHeader { export function oneTurnLog(): SessionEvent[] { return [ { type: 'turn/start', seq: 0, time: 1, data: { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } } }, - { type: 'user/message', seq: 1, time: 2, data: { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } } }, + { type: 'user/message', seq: 1, time: 2, data: { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, surfaceOp: 'append' }, { type: 'step/start', seq: 2, time: 3, data: { turn: 1, step: 1 } }, - { type: 'assistant/message', seq: 3, time: 4, data: { turn: 1, step: 1, content: [{ type: 'text', text: 'hello' }] } }, + { type: 'assistant/message', seq: 3, time: 4, data: { turn: 1, step: 1, content: [{ type: 'text', text: 'hello' }] }, surfaceOp: 'append' }, { type: 'step/end', seq: 4, time: 5, data: { turn: 1, step: 1 } }, { type: 'turn/end', seq: 5, time: 6, data: { turn: 1, reason: { kind: 'completed' } } }, ] } +/** + * Append a whole event log to a LIVE session, event by event, forwarding the + * surface metadata each event already carries. A bare `append(e.type, e.data)` + * over a `SessionEvent[]` widens the type argument to the union, where the + * typed overload's mandatory-marker rule collapses to optional — and `append`'s + * runtime guard then rejects a surface-eligible event with no marker. This + * helper forwards the `surfaceOp`/`sourceEventSeqs` VERBATIM from the source + * event (it does not synthesize a default), so a well-formed recorded log + * round-trips through a live session intact and a fixture that forgot a marker + * still trips the guard. + */ +export function appendLog(session: Session, events: readonly SessionEvent[]): void { + for (const e of events) { + const se = e as SessionEvent + if (se.surfaceOp !== undefined) { + const intent: SurfaceIntent = { + surfaceOp: se.surfaceOp, + ...se.sourceEventSeqs !== undefined ? { sourceEventSeqs: se.sourceEventSeqs } : {}, + } + session.append(e.type, e.data, intent) + } else { + session.append(e.type, e.data) + } + } +} + /** * Run the backend-agnostic contract suite. `make()` MUST return a fresh, empty * backend each call. diff --git a/packages/session-persistence/session-persistence/tests/coordinator-contract.ts b/packages/session-persistence/session-persistence/tests/coordinator-contract.ts index a21e5a7183..42583c4fe3 100644 --- a/packages/session-persistence/session-persistence/tests/coordinator-contract.ts +++ b/packages/session-persistence/session-persistence/tests/coordinator-contract.ts @@ -31,7 +31,7 @@ import { Context, type Fiber } from 'cordis' import SessionStore, { SESSION_FORMAT_VERSION, SessionId } from '@deepseek-ai/dsh-session' import type { Session, SessionEvent } from '@deepseek-ai/dsh-session' import type { SessionPersistence } from '../src/index.ts' -import { meta, oneTurnLog } from './contract.ts' +import { meta, oneTurnLog, appendLog } from './contract.ts' /** * The backend-specific capabilities the orchestration suite needs beyond the @@ -76,7 +76,7 @@ function inits(persistence: SessionPersistence): Map> { /** Append a whole event log to a live session, event by event (drives session/event). */ function send(session: Session, events: readonly SessionEvent[]): void { - for (const e of events) session.append(e.type, e.data) + appendLog(session, events) } /** A live session created inside its OWN fiber, so it survives a backend reload. */ diff --git a/packages/support/invariants/tests/invariants.spec.ts b/packages/support/invariants/tests/invariants.spec.ts index 21f6356aee..6ae2b0edf1 100644 --- a/packages/support/invariants/tests/invariants.spec.ts +++ b/packages/support/invariants/tests/invariants.spec.ts @@ -306,7 +306,7 @@ describe('dev-freeze', () => { const { ctx } = await setup() const seed = [ { type: 'turn/start' as const, seq: 0, time: 0, data: { turn: 1, trigger: { kind: 'message' as const, source: { kind: 'user' as const } } } }, - { type: 'user/message' as const, seq: 1, time: 0, data: { content: [{ type: 'text' as const, text: 'seeded' }], source: { kind: 'user' as const } } }, + { type: 'user/message' as const, seq: 1, time: 0, data: { content: [{ type: 'text' as const, text: 'seeded' }], source: { kind: 'user' as const } }, surfaceOp: 'append' as const }, ] const session = ctx.sessions.create(undefined, { seed }) expect(Object.isFrozen(session.events[0])).toBe(true) From 0619ce62b6311f207946ec32dda643ea8ff9d176 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Fri, 26 Jun 2026 23:38:53 +0800 Subject: [PATCH 16/17] ci: raise Node heap ceiling for type-aware ESLint lint step Type-aware ESLint loads every package tsconfig through the project service and peaks at ~3.4GB RSS. The default V8 old-space ceiling (~2GB) OOMs it (FATAL ERROR: Ineffective mark-compacts near heap limit, exit 134) on both node 24 and 26. Set NODE_OPTIONS with an 8GB ceiling for the Lint step, comfortably above the peak. --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f164e3c4c4..9064a38cea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,8 +39,13 @@ jobs: - name: Typecheck (src + tests + examples) run: pnpm run typecheck + # Type-aware ESLint loads every package tsconfig through the project + # service and peaks at ~3.4GB; the default V8 old-space ceiling (~2GB) + # OOMs it (exit 134). Raise the ceiling well above the peak. - name: Lint run: pnpm run lint + env: + NODE_OPTIONS: --max-old-space-size=8192 # Doc-sync gates (doc-sync-enforcement RFC). doc-typecheck compiles the # fenced ts blocks against the root project-reference graph. The cordis From 489a26e86515670b625779e662232de19df0801a Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Fri, 26 Jun 2026 23:47:54 +0800 Subject: [PATCH 17/17] test(session): cover the isSurfaceEvent / isSurfaceEligibleType guards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-file 100% coverage gate flagged surface.ts line 46 — the branch where a surface-eligible event type carries no surfaceOp marker (isSurfaceEvent returns false). Exercise both guards directly: the type-only eligibility check, the positive narrowing path, a non-eligible type, and the markerless-but-eligible branch. --- packages/core/session/tests/surface.spec.ts | 40 ++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/packages/core/session/tests/surface.spec.ts b/packages/core/session/tests/surface.spec.ts index a7f4c13dad..4e6bc1f433 100644 --- a/packages/core/session/tests/surface.spec.ts +++ b/packages/core/session/tests/surface.spec.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' import type { SessionEvent, SurfaceEvent, SurfaceEventType } from '@deepseek-ai/dsh-session' -import { Session, SessionId } from '@deepseek-ai/dsh-session' +import { Session, SessionId, isSurfaceEligibleType, isSurfaceEvent } from '@deepseek-ai/dsh-session' import { CallId } from '@deepseek-ai/dsh-llm' /** Build a minimal session with turn boundaries and a single user message. */ @@ -279,3 +279,41 @@ describe('Session.append surface opts', () => { expect(event.surfaceOp).toBe('append') }) }) + +describe('surface type guards', () => { + it('isSurfaceEligibleType is true only for message-producing types', () => { + expect(isSurfaceEligibleType('user/message')).toBe(true) + expect(isSurfaceEligibleType('assistant/message')).toBe(true) + expect(isSurfaceEligibleType('tool/result')).toBe(true) + expect(isSurfaceEligibleType('context/message')).toBe(true) + expect(isSurfaceEligibleType('steering/message')).toBe(true) + expect(isSurfaceEligibleType('turn/start')).toBe(false) + expect(isSurfaceEligibleType('assistant/chunk')).toBe(false) + }) + + it('isSurfaceEvent narrows a fully-formed surface event', () => { + const s = surfaceSession() + const userMessage = s.events.find(e => e.type === 'user/message')! + expect(isSurfaceEvent(userMessage)).toBe(true) + }) + + it('isSurfaceEvent rejects a non-surface-eligible type', () => { + const s = surfaceSession() + const turnStart = s.events.find(e => e.type === 'turn/start')! + expect(isSurfaceEvent(turnStart)).toBe(false) + }) + + it('isSurfaceEvent rejects a surface-eligible type missing its surfaceOp marker', () => { + // A surface-eligible type whose mandatory surfaceOp is absent — the state a + // seed/load log can carry before the marker is validated. surfaceOp is + // optional on SessionEvent, so this is a representable runtime value. + const markerless: SessionEvent = { + type: 'user/message', + seq: 0, + time: 0, + data: { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, + } + expect(isSurfaceEligibleType(markerless.type)).toBe(true) + expect(isSurfaceEvent(markerless)).toBe(false) + }) +})