From c5a1c494e78278710a63257f0af7b64f7b6d9ce2 Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Wed, 17 Jun 2026 19:25:29 +0800 Subject: [PATCH 1/9] feature(session): session surface --- docs/adr/0019-session-surface.md | 63 ++++ docs/adr/README.md | 1 + packages/agent-loop/src/agent.ts | 4 +- packages/agent-loop/src/loop.ts | 14 +- packages/invariants/src/index.ts | 30 +- packages/invariants/tests/invariants.spec.ts | 112 ++++++ packages/session-persistence-sqlite/README.md | 2 +- .../session-persistence-sqlite/src/index.ts | 28 +- .../session-persistence-sqlite/src/schema.ts | 38 +- .../tests/sqlite.spec.ts | 130 ++++++- packages/session/README.md | 31 +- packages/session/src/index.ts | 128 +++++-- packages/session/src/repair.ts | 23 +- packages/session/src/surface.ts | 129 +++++++ packages/session/src/types.ts | 36 ++ packages/session/tests/repair.spec.ts | 32 ++ packages/session/tests/surface.spec.ts | 324 ++++++++++++++++++ 17 files changed, 1047 insertions(+), 78 deletions(-) create mode 100644 docs/adr/0019-session-surface.md create mode 100644 packages/session/src/surface.ts create mode 100644 packages/session/tests/surface.spec.ts diff --git a/docs/adr/0019-session-surface.md b/docs/adr/0019-session-surface.md new file mode 100644 index 0000000000..159db911b1 --- /dev/null +++ b/docs/adr/0019-session-surface.md @@ -0,0 +1,63 @@ +# ADR 0019: Session surface — a linked list over the event log for LLM message derivation + +Status: accepted (2026-06-17) + +## Context + +The `Session` event log is the single source of truth ([ADR 0003](0003-event-sourced-sessions.md)), but the only view over it was `deriveMessages()` — a linear scan that filtered and transformed raw events into `Message[]`. This creates problems for session-history-manipulating plugins (compaction, tool-call result pruning, etc.). Without a central mechanism, each plugin would need to wrap `agent/request` to rewrite the message list — a pattern that suffers from listener-ordering fragility, provides no durable record of what was changed, and forces repeated changes to the core `deriveMessages()` whenever a new manipulation is added. A central hub in the `session` package, with a provenance-recording mechanism and enough flexibility for future plugins to manipulate session history through a stable API, lays a solid foundation for plugin development. + +## Decision + +Add a **surface** — a derived, cached linked list of "surface nodes" (the subset of events that produce LLM messages) — maintained by `surfaceOp` markers in the event log. + +### Two new top-level fields on `SessionEvent` + +Every `SessionEvent` gains two optional fields (structural metadata, like `seq`/`time`): + +- **`sourceEventSeqs?: number[]`** — seq numbers of events that are provenance sources (e.g., the `assistant/chunk` seqs that built an `assistant/message`, or the surface nodes shadowed by a compaction marker). Provenance is a core design principle; without it, the replace-range operation cannot be validated on replay. +- **`surfaceOp?: SurfaceOp`** — how this event entered the surface. Absent for non-surface events. + +### SurfaceOp: two operations + +```ts +export type SurfaceOp = + | 'append' // normal tail append + | { op: 'replace'; start: number; end: number } // shadow [start, end] inclusive +``` + +1. **Append** — add a new node to the tail. Used by `user/message`, `assistant/message`, `tool/result`, `context/message`, `steering/message`. The loop passes `surfaceOp: 'append'` on all such appends, and `sourceEventSeqs` where applicable (e.g., `assistant/message` records its `assistant/chunk` sources; `tool/result` records its `tool/call` source). + +2. **Replace** — remove nodes from `start` through `end` (both inclusive) and insert a new node in their place. Both `start` and `end` must be valid surface node seqs in the current surface; `start === end` replaces a single node. The node's `sourceEventSeqs` must contain every shadowed surface node. The shadowed events remain in the log but are no longer on the surface. + +The both-ends-inclusive design was chosen over half-open `[start, endExclusive)` because the surface is a doubly-linked list — both ends are naturally named by node seqs, and single-node replacement (`start === end`) is a common case that reads naturally with inclusive semantics. + +### SurfaceManager: delta-based, not full rebuild + +A `SurfaceManager` class (private to `Session`) maintains the cached linked list. It tracks `_lastProcessedSeq` and processes only the **delta** (new events since the last access) rather than rescanning the entire log. Because the log is append-only, prior events never change — full rebuild is only needed after a wholesale log replacement (e.g., seeding). + +Why delta processing? The naive approach (a dirty flag + full rebuild on every access) would be O(N²) over a session's lifetime — every single-event append triggers a complete scan of all prior events. Delta processing is O(1) when no new events and O(new events) when new events arrive. + +`deriveMessages()` uses the surface when surface markers exist, falling back to the existing linear scan for sessions without markers (backward compatibility). + +### Persistence + +The new fields are serialized as top-level JSON properties. The JSONL backend requires zero changes — `JSON.stringify`/`JSON.parse` preserve everything transparently. The SQLite backend adds two nullable TEXT columns (`source_event_seqs`, `surface_op`) with an `ALTER TABLE` migration (SCHEMA_VERSION 1 → 2). The session format `version` stays at 1 — the new fields are optional and backward-compatible. + +### Crash recovery + +The `repair.ts` module synthesizes `tool/result` closers for orphaned tool calls after a crash. These closers carry `surfaceOp: 'append'` and `sourceEventSeqs` pointing to the orphaned `tool/call` event, so the rehydrated surface is valid. + +### Invariants + +The dev-mode invariants plugin validates: `sourceEventSeqs` references (non-empty, no duplicates, references earlier events, references known seqs) and `surfaceOp` (replace start ≤ end). + +## Consequences + +- **`packages/session`**: New `surface.ts` (`SurfaceManager`), new types (`SurfaceOp`, `SurfaceAppendOpts`), new fields on `SessionEvent`, modified `append()` (third optional `SurfaceAppendOpts` param), refactored `deriveMessages()` (surface path + legacy fallback), surface-aware `repair.ts`. +- **`packages/agent-loop`**: All surface-capable appends pass surface opts. Chunk seqs are collected for `assistant/message` provenance; `tool/call` seqs are captured for `tool/result` provenance. +- **`packages/session-persistence-sqlite`**: Schema migration v1 → v2 (two new nullable TEXT columns). +- **`packages/invariants`**: Surface-related validation rules. +- **`packages/session-persistence-jsonl`**: No changes required. +- **`packages/session-persistence`**: Abstract interface unchanged. + +The surface is the foundation for future compaction: a compaction plugin appends a new event (e.g., `compaction/marker`, added to `SessionEventMap` via declaration merging) with `surfaceOp: { op: 'replace', start, end }` and `sourceEventSeqs` covering the shadowed nodes. Replay preserves the compaction decision deterministically. diff --git a/docs/adr/README.md b/docs/adr/README.md index d9671c6652..8f3180979e 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -30,3 +30,4 @@ Do NOT write an ADR for: a mechanical or local choice (a variable name, a one-fi | [0016](0016-pnpm-over-yarn.md) | pnpm as the package manager instead of Yarn 4 | accepted | | [0017](0017-turn-enclosure-invariant.md) | Every session event is enclosed in a turn | accepted | | [0018](0018-session-persistence.md) | Session persistence as an abstract service over `SessionEvent` | accepted | +| [0019](0019-session-surface.md) | Session surface — a linked list over the event log for LLM message derivation | accepted | diff --git a/packages/agent-loop/src/agent.ts b/packages/agent-loop/src/agent.ts index 64576186c3..0735784616 100644 --- a/packages/agent-loop/src/agent.ts +++ b/packages/agent-loop/src/agent.ts @@ -78,7 +78,7 @@ export class LoopAgent implements Agent { // A turn is open in the LOG (decided from the log, not agent status — // status can be `running` with no turn open): the context/message is // turn-enclosed by that turn, so append it directly. - this.session.append('context/message', { content, source }) + this.session.append('context/message', { content, source }, { surfaceOp: 'append' }) return } // No turn open: wrap the injection in a one-shot turn so every event stays @@ -95,7 +95,7 @@ export class LoopAgent implements Agent { // can't happen for our fixed trigger — no turn was opened and none is owed.) try { this.session.append('turn/start', { turn, trigger: { kind: 'injection', source } }) - this.session.append('context/message', { content, source }) + this.session.append('context/message', { content, source }, { surfaceOp: 'append' }) } finally { // Close the turn if turn/start made it into the log. Contain a throwing // turn/end listener: Session.append pushes before notifying, so a throw diff --git a/packages/agent-loop/src/loop.ts b/packages/agent-loop/src/loop.ts index d2b1ed270b..f1bb879626 100644 --- a/packages/agent-loop/src/loop.ts +++ b/packages/agent-loop/src/loop.ts @@ -276,7 +276,7 @@ async function runTurn(ctx: Context, agent: LoopAgent, handle: LoopHandle, turn: // every event in the log is turn-enclosed. turn/end is now owed, so a throw // while appending these is caught below and the turn is still closed. for (const message of queued) { - session.append('user/message', { content: message.content, source: message.source }) + session.append('user/message', { content: message.content, source: message.source }, { surfaceOp: 'append' }) } ctx.emit('agent/turn-start', agent, turn) @@ -410,7 +410,7 @@ async function runTurn(ctx: Context, agent: LoopAgent, handle: LoopHandle, turn: function drainSteering(ctx: Context, agent: LoopAgent, turn: number): boolean { const messages = agent.inbox.drainSteering() for (const message of messages) { - agent.session.append('steering/message', { turn, content: message.content, source: message.source }) + agent.session.append('steering/message', { turn, content: message.content, source: message.source }, { surfaceOp: 'append' }) ctx.emit('agent/steering', agent, turn, message.content, message.source) } return messages.length > 0 @@ -446,10 +446,12 @@ async function runStep( // --- Model call (streaming-first; raw chunks are the replay record) --- const assembler = new BlockAssembler() + const chunkSeqs: number[] = [] for await (const chunk of ctx.llm.stream(request)) { /* v8 ignore next -- signal.reason always set by agent.abort() which provides a default */ if (signal.aborted) throw new Error(String(signal.reason ?? 'aborted')) - session.append('assistant/chunk', { turn, step, chunk }) + const chunkEvent = session.append('assistant/chunk', { turn, step, chunk }) + chunkSeqs.push(chunkEvent.seq) ctx.emit('agent/stream-chunk', agent, turn, step, chunk) assembler.push(chunk) } @@ -468,7 +470,7 @@ async function runStep( let message: Message = assembler.message() message = await ctx.waterfall('agent/step-result', agent, turn, step, message, () => Promise.resolve(message)) - session.append('assistant/message', { turn, step, content: message.content }) + session.append('assistant/message', { turn, step, content: message.content }, { surfaceOp: 'append', sourceEventSeqs: chunkSeqs }) if (assembler.usage) { session.append('usage', { turn, step, usage: assembler.usage }) } @@ -480,7 +482,7 @@ async function runStep( for (const call of toolCalls) { /* v8 ignore next -- signal.reason always set by agent.abort() which provides a default */ if (signal.aborted) throw new Error(String(signal.reason ?? 'aborted')) - session.append('tool/call', { turn, step, callId: call.id, name: call.name, arguments: call.arguments }) + const callEvent = session.append('tool/call', { turn, step, callId: call.id, name: call.name, arguments: call.arguments }) let parsedArguments: unknown try { parsedArguments = call.arguments ? JSON.parse(call.arguments) : {} @@ -506,7 +508,7 @@ async function runStep( content: result.content, isError: result.isError, ...result.error ? { error: result.error } : {}, - }) + }, { surfaceOp: 'append', sourceEventSeqs: [callEvent.seq] }) // signal CAN flip during the await above (abort() inside a tool); // the analyzer can't see through the await boundary. // signal can flip during the await above (abort() inside a tool); diff --git a/packages/invariants/src/index.ts b/packages/invariants/src/index.ts index ebb4decfd8..ac536eeceb 100644 --- a/packages/invariants/src/index.ts +++ b/packages/invariants/src/index.ts @@ -62,6 +62,8 @@ interface SessionTrace { * `step/end` — a result must arrive in the same step as its call. */ pendingCalls: Set + /** Every seq seen so far — validates `sourceEventSeqs` references. */ + knownSeqs: Set } /** @@ -105,6 +107,30 @@ function checkEvent(trace: SessionTrace, event: SessionEvent): void { } trace.lastSeq = event.seq + // --- Surface invariants --- + if (event.sourceEventSeqs !== undefined) { + if (event.sourceEventSeqs.length === 0) { + throw new InvariantError('sourceEventSeqs must not be empty when present') + } + const unique = new Set(event.sourceEventSeqs) + if (unique.size !== event.sourceEventSeqs.length) { + throw new InvariantError('sourceEventSeqs must not contain duplicates') + } + for (const ref of event.sourceEventSeqs) { + if (ref >= event.seq) { + throw new InvariantError(`sourceEventSeqs must reference earlier events: ${ref} >= current seq ${event.seq}`) + } + if (!trace.knownSeqs.has(ref)) { + throw new InvariantError(`sourceEventSeqs references unknown seq ${ref}`) + } + } + } + if (event.surfaceOp !== undefined && typeof event.surfaceOp !== 'string') { + if (event.surfaceOp.start > event.surfaceOp.end) { + throw new InvariantError(`surface replace: start ${event.surfaceOp.start} must be <= end ${event.surfaceOp.end}`) + } + } + // Boundary/step-scoped events have explicit cases; every OTHER event type — // including plugin-added (merge-extensible) SessionEventMap keys — is caught // by the `default` and must be turn-enclosed (ADR 0017). No assertNever: an @@ -185,6 +211,8 @@ function checkEvent(trace: SessionTrace, event: SessionEvent): void { break } } + // Track every seq seen — used above to validate sourceEventSeqs references. + trace.knownSeqs.add(event.seq) } /** Legal agent status transitions (the only state machine the loop guarantees). */ @@ -216,7 +244,7 @@ export function apply(ctx: Context, config: Config = {}): void { // (re-)apply seeds the baseline, so a reload never produces a false positive. const lastStatus = new WeakMap() - const freshTrace = (): SessionTrace => ({ lastSeq: -1, openTurn: null, openStep: null, pendingCalls: new Set() }) + const freshTrace = (): SessionTrace => ({ lastSeq: -1, openTurn: null, openStep: null, pendingCalls: new Set(), knownSeqs: new Set() }) /** Build (or rebuild) a session's trace by replaying its whole log; freeze it. */ const seedSession = (session: Session): SessionTrace => { diff --git a/packages/invariants/tests/invariants.spec.ts b/packages/invariants/tests/invariants.spec.ts index b96d172c22..69d63aed4e 100644 --- a/packages/invariants/tests/invariants.spec.ts +++ b/packages/invariants/tests/invariants.spec.ts @@ -387,3 +387,115 @@ describe('HMR safety', () => { expect(Object.isFrozen(session.events[0])).toBe(false) }) }) + +describe('surface invariants', () => { + it('accepts well-formed surface metadata', async () => { + const { ctx } = await setup() + const session = ctx.sessions.create() + // Events must be turn-enclosed and step-scoped events need an open step. + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + session.append('step/start', { turn: 1, step: 1 }) + expect(() => { + session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) + session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [1] }) + }).not.toThrow() + }) + + it('accepts replace surface op', async () => { + const { ctx } = await setup() + const session = ctx.sessions.create() + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + session.append('step/start', { turn: 1, step: 1 }) + session.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) + session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 2, end: 2 }, sourceEventSeqs: [2] }) + // no throw — well-formed replace op + }) + + it('rejects empty sourceEventSeqs', async () => { + const { ctx } = await setup() + const session = ctx.sessions.create() + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + expect(() => { + session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [] }) + }).toThrow(InvariantError) + }) + + it('rejects duplicate sourceEventSeqs', async () => { + const { ctx } = await setup() + const session = ctx.sessions.create() + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + session.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) + expect(() => { + session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [1, 1] }) + }).toThrow(/must not contain duplicates/) + }) + + it('rejects sourceEventSeqs referencing the event itself (self-reference)', async () => { + const { ctx } = await setup() + const session = ctx.sessions.create() + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) // seq 0 + // The next event is seq 1. Referencing its own seq fails on "must reference + // earlier events" (the check order is: earlier first, then unknown). + expect(() => { + session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [1] }) + }).toThrow(/must reference earlier/) + }) + + it('accepts sourceEventSeqs referencing a valid earlier event', async () => { + // Positive test: ref < current seq and ref is in knownSeqs → passes. + const { ctx } = await setup() + const session = ctx.sessions.create() + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + session.append('step/start', { turn: 1, step: 1 }) + // seqs so far: 0, 1. The next event at seq 2 references seq 1 → valid. + expect(() => { + session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [1] }) + }).not.toThrow() + }) + + it('rejects sourceEventSeqs referencing a far-future seq', async () => { + const { ctx } = await setup() + const session = ctx.sessions.create() + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + expect(() => { + session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [99] }) + }).toThrow(/must reference earlier/) + }) + + it('rejects sourceEventSeqs referencing unknown seq (gap in event log)', async () => { + // The unknown-seq check fires when a ref passes the "earlier" test but is + // not in knownSeqs — only possible with a gap in seqs. We create a gap by + // directly manipulating the private log array to skip a seq. + const { ctx } = await setup() + const session = ctx.sessions.create() + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + session.append('step/start', { turn: 1, step: 1 }) + // Push a fake event at seq 3 into the internal log, creating a gap at seq 2. + // The invariants plugin replays session.events on every append, so it sees + // this gap during trace reconstruction. + ;(session as unknown as { log: unknown[] }).log.push({ + type: 'assistant/chunk', + seq: 3, + time: Date.now(), + data: { turn: 1, step: 1, chunk: { type: 'text-delta', index: 0, text: 'x' } }, + }) + // Now the log has seqs 0, 1, 3 (gap at 2). Append at what session believes + // is seq 3 (log.length). Reference seq 2: passes earlier (2 < 3) but not + // in knownSeqs ({0, 1, 3} — gap at 2). + expect(() => { + session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [2] }) + }).toThrow(/unknown seq 2/) + }) + + it('rejects replace op with start > end', async () => { + const { ctx } = await setup() + const session = ctx.sessions.create() + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + session.append('step/start', { turn: 1, step: 1 }) + session.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 2 + // start > end is invalid (reversed order). + expect(() => { + session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 2, end: 1 }, sourceEventSeqs: [2] }) + }).toThrow(/must be <= end/) + }) +}) diff --git a/packages/session-persistence-sqlite/README.md b/packages/session-persistence-sqlite/README.md index 3a7a3c8163..dace03c529 100644 --- a/packages/session-persistence-sqlite/README.md +++ b/packages/session-persistence-sqlite/README.md @@ -6,7 +6,7 @@ A SQLite durable session-persistence backend — a second `SessionPersistence` i ## Storage model -Each `SessionEvent` maps 1:1 onto a row in an `events` table `(session_id, seq, type, time, data)` — `data` is the event payload as JSON text, so the row shape is the event verbatim (including `assistant/chunk`, keeping `seq` contiguous). Out-of-log metadata (`SessionMeta`) lives in a `sessions` row, including the mutable `SessionSummary` fields (`updatedAt`, `title`, `firstPrompt`) that `update()` rewrites without touching the event log. A `sessions` row is written only by the first `append` — its existence is the lazy-materialization signal (`has`/`list` report exactly the sessions that have a row), so no separate column is needed. +Each `SessionEvent` maps 1:1 onto a row in an `events` table `(session_id, seq, type, time, data, source_event_seqs, surface_op)` — `data` is the event payload as JSON text, so the row shape is the event verbatim (including `assistant/chunk`, keeping `seq` contiguous). The two `TEXT` columns `source_event_seqs` and `surface_op` are nullable; they store the event's optional surface-metadata fields (see [ADR 0019](../../docs/adr/0019-session-surface.md)). The schema migrates from v1 to v2 via `ALTER TABLE ADD COLUMN` — existing rows get NULL for both columns, which is correct for events written before surface support. Out-of-log metadata (`SessionMeta`) lives in a `sessions` row, including the mutable `SessionSummary` fields (`updatedAt`, `title`, `firstPrompt`) that `update()` rewrites without touching the event log. A `sessions` row is written only by the first `append` — its existence is the lazy-materialization signal (`has`/`list` report exactly the sessions that have a row), so no separate column is needed. The repo targets Node ≥ 24 (the root `engines` field), which includes the stable `node:sqlite` module. The database opens with `foreign_keys = ON` (so `ON DELETE CASCADE` drops a session's events with its row) and `journal_mode = WAL`. The table-layout version is stored in `PRAGMA user_version` and checked on open: a fresh database is stamped with the current `SCHEMA_VERSION`; a database written by a newer, incompatible build (higher `user_version`) is rejected rather than opened against an unknown layout. diff --git a/packages/session-persistence-sqlite/src/index.ts b/packages/session-persistence-sqlite/src/index.ts index bd4b4d9435..8a6cd8cb09 100644 --- a/packages/session-persistence-sqlite/src/index.ts +++ b/packages/session-persistence-sqlite/src/index.ts @@ -33,6 +33,18 @@ import { export { SCHEMA_VERSION } from './schema.ts' +/** + * Serialize an event's surface-metadata fields for SQL binding. Both fields are + * nullable TEXT columns — null when the event has no surface metadata (non-surface + * events, events written before surface support). + */ +function surfaceBindings(event: SessionEvent): [string | null, string | null] { + return [ + event.sourceEventSeqs ? JSON.stringify(event.sourceEventSeqs) : null, + event.surfaceOp !== undefined ? JSON.stringify(event.surfaceOp) : null, + ] +} + /** Plugin configuration. */ export interface Config { /** @@ -180,13 +192,14 @@ export class SessionPersistenceSqlite extends SessionPersistence { // durably closes the interrupted turn before returning, so by the time any // append runs the stored log is balanced and contiguous.) const insertEvent = this.db.prepare( - 'INSERT INTO events (session_id, seq, type, time, data) VALUES (?, ?, ?, ?, ?)', + 'INSERT INTO events (session_id, seq, type, time, data, source_event_seqs, surface_op) VALUES (?, ?, ?, ?, ?, ?, ?)', ) this.db.exec('BEGIN') try { if (!state.materialized) this.writeRow(state.meta) for (const event of events) { - insertEvent.run(id, event.seq, event.type, event.time, JSON.stringify(event.data)) + const [surfaceSeqs, surfaceOp] = surfaceBindings(event) + insertEvent.run(id, event.seq, event.type, event.time, JSON.stringify(event.data), surfaceSeqs, surfaceOp) } // Bump updatedAt on every append (the mutable summary lives in the row). const updatedAt = Date.now() @@ -220,7 +233,7 @@ export class SessionPersistenceSqlite extends SessionPersistence { // discarded (not unloadable); only a parse error / seq gap in the COMMITTED // region (at or before the last turn/end) throws (genuine corruption). const eventRows = this.db - .prepare('SELECT seq, type, time, data FROM events WHERE session_id = ? ORDER BY seq') + .prepare('SELECT seq, type, time, data, source_event_seqs, surface_op FROM events WHERE session_id = ? ORDER BY seq') .all(id) as unknown as EventRow[] const { preserved, tornFrom } = scanRows(eventRows) @@ -248,9 +261,12 @@ export class SessionPersistenceSqlite extends SessionPersistence { this.db.prepare('DELETE FROM events WHERE session_id = ? AND seq >= ?').run(id, tornFrom) } if (closers.length > 0) { - const insertEvent = this.db.prepare('INSERT INTO events (session_id, seq, type, time, data) VALUES (?, ?, ?, ?, ?)') + const insertEvent = this.db.prepare( + 'INSERT INTO events (session_id, seq, type, time, data, source_event_seqs, surface_op) VALUES (?, ?, ?, ?, ?, ?, ?)', + ) for (const event of closers) { - insertEvent.run(id, event.seq, event.type, event.time, JSON.stringify(event.data)) + const [surfaceSeqs, surfaceOp] = surfaceBindings(event) + insertEvent.run(id, event.seq, event.type, event.time, JSON.stringify(event.data), surfaceSeqs, surfaceOp) } } this.db.exec('COMMIT') @@ -497,7 +513,7 @@ export class SessionPersistenceSqlite extends SessionPersistence { /** The preserved events for a session id (torn tail excluded, turn NOT yet closed). */ private eventsFor(id: SessionId): SessionEvent[] { const rows = this.db - .prepare('SELECT seq, type, time, data FROM events WHERE session_id = ? ORDER BY seq') + .prepare('SELECT seq, type, time, data, source_event_seqs, surface_op FROM events WHERE session_id = ? ORDER BY seq') .all(id) as unknown as EventRow[] // Scan on seq+type columns, parsing `data` only for the preserved prefix (a // malformed torn tail must not throw here — same as loadCore). Returns the diff --git a/packages/session-persistence-sqlite/src/schema.ts b/packages/session-persistence-sqlite/src/schema.ts index 1dad51698b..84b357937a 100644 --- a/packages/session-persistence-sqlite/src/schema.ts +++ b/packages/session-persistence-sqlite/src/schema.ts @@ -8,14 +8,14 @@ */ import { DatabaseSync } from 'node:sqlite' -import type { SessionEvent, SessionId, SessionMeta } from '@deepseek-ai/dsh-session' +import type { SessionEvent, SessionId, SessionMeta, SurfaceOp } from '@deepseek-ai/dsh-session' /** * The on-disk schema version. Bumped only on a breaking change to the table * layout; orthogonal to a session's own `version` (which versions the EVENT * vocabulary, stored per session in the `sessions` row). */ -export const SCHEMA_VERSION = 1 +export const SCHEMA_VERSION = 2 /** * A row of the `sessions` table — the out-of-log metadata (`SessionMeta`). The @@ -41,6 +41,10 @@ export interface EventRow { type: string time: number data: string + /** JSON-encoded `number[]` — the event's sourceEventSeqs, or null. */ + source_event_seqs: string | null + /** JSON-encoded `SurfaceOp` — how the event entered the surface, or null. */ + surface_op: string | null } /** @@ -72,6 +76,13 @@ export function openDatabase(path: string): DatabaseSync { // constant (SCHEMA_VERSION is a trusted in-code number, not user input). db.exec(`PRAGMA user_version = ${SCHEMA_VERSION}`) } + if (onDisk === 1) { + // Migrate from v1 to v2: add surface-metadata columns (nullable — existing + // rows get NULL, which is correct for events written before surface existed). + db.exec('ALTER TABLE events ADD COLUMN source_event_seqs TEXT') + db.exec('ALTER TABLE events ADD COLUMN surface_op TEXT') + db.exec(`PRAGMA user_version = ${SCHEMA_VERSION}`) + } db.exec(` CREATE TABLE IF NOT EXISTS sessions ( id TEXT PRIMARY KEY, @@ -86,11 +97,13 @@ export function openDatabase(path: string): DatabaseSync { `) db.exec(` CREATE TABLE IF NOT EXISTS events ( - session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, - seq INTEGER NOT NULL, - type TEXT NOT NULL, - time INTEGER NOT NULL, - data TEXT NOT NULL, + session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, + seq INTEGER NOT NULL, + type TEXT NOT NULL, + time INTEGER NOT NULL, + data TEXT NOT NULL, + source_event_seqs TEXT, + surface_op TEXT, PRIMARY KEY (session_id, seq) ) STRICT `) @@ -113,12 +126,19 @@ export function rowToMeta(row: SessionRow): SessionMeta { /** Reconstruct a {@link SessionEvent} from an `events` row (parses `data`). */ export function rowToEvent(row: EventRow): SessionEvent { - return { - type: row.type, + const event = { + type: row.type as SessionEvent['type'], seq: row.seq, time: row.time, data: JSON.parse(row.data) as SessionEvent['data'], } as SessionEvent + if (row.source_event_seqs !== null) { + event.sourceEventSeqs = JSON.parse(row.source_event_seqs) as number[] + } + if (row.surface_op !== null) { + event.surfaceOp = JSON.parse(row.surface_op) as SurfaceOp + } + return event } /** diff --git a/packages/session-persistence-sqlite/tests/sqlite.spec.ts b/packages/session-persistence-sqlite/tests/sqlite.spec.ts index 44d788ab72..61b166f257 100644 --- a/packages/session-persistence-sqlite/tests/sqlite.spec.ts +++ b/packages/session-persistence-sqlite/tests/sqlite.spec.ts @@ -1,12 +1,13 @@ import { afterEach, describe, expect, it } from 'vitest' import { Context } from 'cordis' +import { DatabaseSync } from 'node:sqlite' import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import SessionStore, { SessionId } from '@deepseek-ai/dsh-session' import type { Session, SessionEvent, SessionMeta } from '@deepseek-ai/dsh-session' import SessionPersistenceSqlite, { SCHEMA_VERSION } from '@deepseek-ai/dsh-session-persistence-sqlite' -import { openDatabase, scanRows, type EventRow } from '../src/schema.ts' +import { openDatabase, rowToEvent, scanRows, type EventRow } from '../src/schema.ts' import { runPersistenceContract, meta, oneTurnLog } from '../../session-persistence/tests/contract.ts' const dirs: string[] = [] @@ -42,7 +43,7 @@ describe('scanRows', () => { // scanRows works off EventRows (data is a JSON string column); build them from // SessionEvents so the unit tests read in terms of the event vocabulary. const rows = (events: SessionEvent[]): EventRow[] => - events.map(e => ({ seq: e.seq, type: e.type, time: e.time, data: JSON.stringify(e.data) })) + events.map(e => ({ seq: e.seq, type: e.type, time: e.time, data: JSON.stringify(e.data), source_event_seqs: null, surface_op: null })) it('preserves the full log when it ends exactly on a turn/end (no torn tail)', () => { const { preserved, tornFrom } = scanRows(rows(oneTurnLog())) @@ -91,8 +92,8 @@ describe('scanRows', () => { it('throws on an unparsable row inside the committed region', () => { const withCorruptCommitted: EventRow[] = [ - { seq: 0, type: 'turn/start', time: 1, data: '{not json' }, // corrupt, sits before a turn/end - { seq: 1, type: 'turn/end', time: 2, data: JSON.stringify({ turn: 1, reason: { kind: 'completed' } }) }, + { seq: 0, type: 'turn/start', time: 1, data: '{not json', source_event_seqs: null, surface_op: null }, // corrupt, sits before a turn/end + { seq: 1, type: 'turn/end', time: 2, data: JSON.stringify({ turn: 1, reason: { kind: 'completed' } }), source_event_seqs: null, surface_op: null }, ] expect(() => scanRows(withCorruptCommitted)).toThrow(/unparsable committed event/) }) @@ -100,7 +101,7 @@ describe('scanRows', () => { it('tolerates an unparsable torn-tail row after the last turn/end', () => { const withCorruptTail: EventRow[] = [ ...rows(oneTurnLog()), - { seq: 6, type: 'turn/start', time: 7, data: '{not json' }, // torn fragment, no committed turn/end after + { seq: 6, type: 'turn/start', time: 7, data: '{not json', source_event_seqs: null, surface_op: null }, // torn fragment, no committed turn/end after ] const { preserved, tornFrom } = scanRows(withCorruptTail) expect(preserved).toEqual(oneTurnLog()) @@ -343,7 +344,7 @@ describe('SessionPersistenceSqlite: durability and crash semantics', () => { }) it('exposes the schema version constant', () => { - expect(SCHEMA_VERSION).toBe(1) + expect(SCHEMA_VERSION).toBe(2) }) }) @@ -749,4 +750,121 @@ describe('SessionPersistenceSqlite: edge cases', () => { await expect(ctx.parallel('session/flush', session)).rejects.toThrow(/id collision/) await ctx.fiber.dispose() }) + + it('migrates a v1 database to v2 (adds surface columns)', async () => { + const path = await freshDbPath() + // Manually create a v1 database with the OLD schema (no surface columns). + const db = new DatabaseSync(path) + db.exec('PRAGMA user_version = 1') + db.exec(` + CREATE TABLE sessions ( + id TEXT PRIMARY KEY, + version INTEGER NOT NULL, + created_at INTEGER NOT NULL, + cwd TEXT, + parent_session TEXT, + updated_at INTEGER NOT NULL, + title TEXT, + first_prompt TEXT + ) STRICT + `) + db.exec(` + CREATE TABLE events ( + session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, + seq INTEGER NOT NULL, + type TEXT NOT NULL, + time INTEGER NOT NULL, + data TEXT NOT NULL, + PRIMARY KEY (session_id, seq) + ) STRICT + `) + db.close() + // Re-open with v2 code: migration adds the surface columns and stamps v2. + const db2 = openDatabase(path) + const version = (db2.prepare('PRAGMA user_version').get() as { user_version: number }).user_version + expect(version).toBe(2) + const info = db2.prepare("PRAGMA table_info('events')").all() as Array<{ name: string }> + const names = info.map(c => c.name) + expect(names).toContain('source_event_seqs') + expect(names).toContain('surface_op') + db2.close() + }) +}) + +describe('surface field round-trip', () => { + it('rowToEvent parses surface fields from EventRow columns', () => { + const row: EventRow = { + seq: 0, type: 'assistant/message', time: 1, + data: JSON.stringify({ turn: 1, step: 1, content: [] }), + source_event_seqs: JSON.stringify([3, 5]), + surface_op: JSON.stringify('append'), + } + const event = rowToEvent(row) + expect(event.sourceEventSeqs).toEqual([3, 5]) + expect(event.surfaceOp).toBe('append') + }) + + it('rowToEvent handles replace surfaceOp object', () => { + const row: EventRow = { + seq: 0, type: 'assistant/message', time: 1, + data: JSON.stringify({ turn: 1, step: 1, content: [] }), + source_event_seqs: JSON.stringify([0, 1]), + surface_op: JSON.stringify({ op: 'replace', start: 0, end: 1 }), + } + const event = rowToEvent(row) + expect(event.sourceEventSeqs).toEqual([0, 1]) + expect(event.surfaceOp).toEqual({ op: 'replace', start: 0, end: 1 }) + }) + + it('scanRows with surface columns reconstructs events with surface fields', () => { + const rows: EventRow[] = [ + { seq: 0, type: 'user/message', time: 1, + data: JSON.stringify({ content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }), + source_event_seqs: null, surface_op: '{"op":"replace","start":0,"end":0}' }, + { seq: 1, type: 'turn/end', time: 2, + data: JSON.stringify({ turn: 1, reason: { kind: 'completed' } }), + source_event_seqs: null, surface_op: null }, + ] + const { preserved } = scanRows(rows) + expect(preserved).toHaveLength(2) + expect(preserved[0]!.surfaceOp).toEqual({ op: 'replace', start: 0, end: 0 }) + expect(preserved[0]!.sourceEventSeqs).toBeUndefined() + expect(preserved[1]!.surfaceOp).toBeUndefined() + }) + + it('append and load round-trips surface fields through SQLite', async () => { + const ctx = new Context() + await ctx.plugin(SessionStore) + const fiber = await ctx.plugin(SessionPersistenceSqlite, { path: ':memory:' }) + const session = ctx.sessions.create('roundtrip-surface') + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) + session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append', sourceEventSeqs: [0] }) + session.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) + await ctx.parallel('session/flush', session) + const loaded = await ctx.sessionPersistence.load(SessionId('roundtrip-surface')) + expect(loaded.events).toHaveLength(4) + const um = loaded.events[1]! + expect(um.surfaceOp).toBe('append') + expect(um.sourceEventSeqs).toBeUndefined() + const am = loaded.events[2]! + expect(am.surfaceOp).toBe('append') + expect(am.sourceEventSeqs).toEqual([0]) + await fiber.dispose() + }) + + it('persists events with surfaceOp but no sourceEventSeqs (covers null branch in surfaceBindings)', async () => { + const ctx = new Context() + await ctx.plugin(SessionStore) + const fiber = await ctx.plugin(SessionPersistenceSqlite, { path: ':memory:' }) + const session = ctx.sessions.create('surface-noseq') + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + session.append('steering/message', { turn: 1, content: [], source: { kind: 'user' } }, { surfaceOp: 'append' }) + session.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) + await ctx.parallel('session/flush', session) + const loaded = await ctx.sessionPersistence.load(SessionId('surface-noseq')) + expect(loaded.events[1]!.surfaceOp).toBe('append') + expect(loaded.events[1]!.sourceEventSeqs).toBeUndefined() + await fiber.dispose() + }) }) diff --git a/packages/session/README.md b/packages/session/README.md index 7cf443fa71..b7787a768c 100644 --- a/packages/session/README.md +++ b/packages/session/README.md @@ -1,6 +1,6 @@ # dsh-session -Event-sourced session log and in-memory store. A `Session` is the append-only source of truth for an agent's whole interaction history — the LLM message history is *derived* from it. +Event-sourced session log and in-memory store. A `Session` is the append-only source of truth for an agent's whole interaction history — the LLM message history is *derived* from it. A **surface** layer (a linked list of message-producing events) is maintained on top of the raw log for efficient derivation and compaction. ## Service: `SessionStore` (ctx key: `sessions`) @@ -24,16 +24,17 @@ Creates and holds event-sourced `Session` instances. Persistence is intentionall Plain class (not a Cordis Service). Create via `ctx.sessions.create()`. -- `session.append(type, data): SessionEvent` — synchronous, never blocks on I/O. **Throws** if `data` is not losslessly JSON-serializable (BigInt, function, symbol, undefined, non-finite number, circular ref, or an exotic object like Map/Set/Date) — the event log is the durable source of truth, so this invariant is enforced at the source (exported as `isJsonValue` for backends to reuse on their replay/fork entry points). -- `session.deriveMessages(): Message[]` — derive the LLM message history from the event log. Raw `assistant/chunk` events are skipped; `context/message` and `steering/message` render as tagged synthetic user messages. +- `session.append(type, data, opts?): SessionEvent` — synchronous, never blocks on I/O. **Throws** if `data` is not losslessly JSON-serializable (BigInt, function, symbol, undefined, non-finite number, circular ref, or an exotic object like Map/Set/Date) — the event log is the durable source of truth, so this invariant is enforced at the source (exported as `isJsonValue` for backends to reuse on their replay/fork entry points). An optional third parameter `opts: SurfaceAppendOpts` carries surface metadata: `surfaceOp` controls how the event enters the surface linked list, and `sourceEventSeqs` records provenance (the seq numbers of events this one derives from). +- `session.deriveMessages(): Message[]` — derive the LLM message history. If any event in the log carries `surfaceOp`, derivation walks the surface linked list (skipping non-surface events). Otherwise, falls back to a linear scan of the raw log (legacy sessions without surface markers). +- `session.surface: SurfaceManager` — the derived surface, lazily rebuilt from `surfaceOp` markers in the log. Processes only new events (delta) on each access — the log is append-only, so prior events never change. - `session.events`, `session.seq`, `session.id` - `session.header: SessionHeader` — immutable creation metadata (`version`, `id`, `createdAt`, optional `cwd`/`parentSession`). Kept out of the event log (a storage concern, not replayable state); a minimal v1 header is synthesized for bare `Session` construction. -### Metadata types (`types.ts`) +### Surface types -- `SessionHeader` — immutable, written once: `{ version, id, createdAt, cwd?, parentSession? }`. -- `SessionSummary` — mutable, updateable without touching the log: `{ updatedAt, title?, firstPrompt? }`. -- `SessionMeta = SessionHeader & SessionSummary` — owned here (beside `SessionId`) because `Session.header` is typed by it; persistence backends re-export these rather than own them (which would force a package cycle). +- `SurfaceOp` — how a surface node entered the linked list: `'append'` (normal tail append) or `{ op: 'replace', start, end }` (replace nodes from `start` through `end` inclusive — both must be valid surface node seqs; `start === end` replaces a single node). Used by compaction to shadow old nodes without deleting them. +- `SurfaceAppendOpts` — `{ surfaceOp?: SurfaceOp; sourceEventSeqs?: number[] }`, the optional third parameter to `session.append()`. +- `SurfaceNode` — `{ seq: number; prev: number | null; next: number | null }`, one node in the surface linked list. ### Session event vocabulary (`types.ts`) @@ -43,11 +44,23 @@ Merge-extensible via `SessionEventMap` — a compaction plugin adds `compaction/ Also defines `TurnTriggerMap` and `TurnEndReasonMap` (merge-extensible sum types for typed turn boundaries — `kind`-tagged instead of strings). +Every `SessionEvent` carries two optional top-level fields (structural metadata): + +- `sourceEventSeqs?: number[]` — seq numbers of provenance sources (e.g., the `assistant/chunk` seqs behind an `assistant/message`, or the shadowed nodes behind a compaction marker). +- `surfaceOp?: SurfaceOp` — how this event entered the surface. Absent for non-surface events (boundaries, chunks, usage, errors). + +### Metadata types (`types.ts`) + +- `SessionHeader` — immutable, written once: `{ version, id, createdAt, cwd?, parentSession? }`. +- `SessionSummary` — mutable, updateable without touching the log: `{ updatedAt, title?, firstPrompt? }`. +- `SessionMeta = SessionHeader & SessionSummary` — owned here (beside `SessionId`) because `Session.header` is typed by it; persistence backends re-export these rather than own them (which would force a package cycle). + ### Extension points - Persistence plugins: subscribe to `session/event` (write-behind) and drain on `session/flush` (awaited) and fiber dispose. A durable backend reads the log and reloads it into a live session; the metadata seam (`SessionHeader`/`SessionSummary`/`SessionMeta`, `session.header`) is what such a backend stores beside the log. -- Replay/fork: `ctx.sessions.create(id, { seed })` seeds a new session with an existing event log. +- Replay/fork: `ctx.sessions.create(id, { seed })` seeds a new session with an existing event log. The surface rebuilds deterministically from `surfaceOp` markers in the seeded events. +- Compaction: a future plugin appends a new event with `surfaceOp: { op: 'replace', start, end }` to shadow old surface nodes. ### What is NOT here (TODO) -- **Session branching/tree** (pi-style entry tree) — defered unless needed beyond seed-based forking. +- **Session branching/tree** (pi-style entry tree) — deferred unless needed beyond seed-based forking. diff --git a/packages/session/src/index.ts b/packages/session/src/index.ts index 34eefb24a3..d0d4d44aed 100644 --- a/packages/session/src/index.ts +++ b/packages/session/src/index.ts @@ -10,12 +10,14 @@ import { Context, Service } from 'cordis' import { isAbsolute } from 'node:path' import type { ContentBlock, Message, MessageSource } from '@deepseek-ai/dsh-llm' import { SessionId } from './types.ts' -import type { CreateSessionOptions, SessionEvent, SessionEventMap, SessionEventType, SessionHeader } from './types.ts' +import type { CreateSessionOptions, SessionEvent, SessionEventMap, SessionEventType, SessionHeader, SurfaceAppendOpts } from './types.ts' import { isJsonValue } from './json.ts' +import { SurfaceManager } from './surface.ts' export * from './types.ts' export { isJsonValue } from './json.ts' export { interruptedTurnClosers } from './repair.ts' +export type { SurfaceNode } from './surface.ts' declare module 'cordis' { interface Context { @@ -65,6 +67,21 @@ export class Session { /** Set by the store so appends are observable; undefined when detached. */ onAppend: ((event: SessionEvent) => void) | undefined + /** + * Derived surface — a cached linked list of message-producing events. + * Lazily rebuilt from `surfaceOp` markers in the log; processes only new + * events (delta) on each access — the log is append-only, so prior events + * never change. + * `append`. Undefined until first accessed (including after fork/seed). + */ + private _surface: SurfaceManager | undefined + + /** The surface linked list over this session's event log. */ + get surface(): SurfaceManager { + if (!this._surface) this._surface = new SurfaceManager(this.log) + return this._surface + } + /** * Immutable creation metadata (format version, cwd, lineage). Supplied by * the store via `ctx.sessions.create()`. When a `Session` is constructed @@ -117,6 +134,11 @@ export class Session { * `onAppend`. The hot path never blocks on I/O — persistence plugins buffer * asynchronously. * + * @param type - The event type (key of {@link SessionEventMap}). + * @param data - The event payload; must be JSON-serializable. + * @param opts - Optional surface metadata: `surfaceOp` controls how the + * event enters the surface linked list; `sourceEventSeqs` records + * provenance (the seq numbers of events this one derives from). * @throws if `data` is not losslessly JSON-serializable (BigInt, function, * symbol, undefined, non-finite number, circular ref, or an exotic object * like Map/Set/Date). The event log is the durable source of truth, so this @@ -125,7 +147,7 @@ export class Session { * throw surfaces at the buggy caller's append site, not asynchronously in a * backend flush. */ - append(type: T, data: SessionEventMap[T]): SessionEvent { + append(type: T, data: SessionEventMap[T], opts?: SurfaceAppendOpts): SessionEvent { if (!isJsonValue(data)) { throw new Error(`session event "${type}" carries non-JSON-serializable data`) } @@ -138,14 +160,29 @@ export class Session { // validated. structuredClone is safe because serializability was just // checked. The returned event carries the SAME snapshot, so a caller reading // back `event.data` sees the logged value, not its own mutable input. - const event = { type, seq: this.log.length, time: Date.now(), data: structuredClone(data) } as SessionEvent + // + // Surface metadata is snapshot separately: sourceEventSeqs (number[] — + // primitives, so array spread is a complete copy) and surfaceOp (a string + // primitive, or cloned if it's a replace object). + const event = { + type, + seq: this.log.length, + time: Date.now(), + data: structuredClone(data), + ...opts?.sourceEventSeqs !== undefined ? { sourceEventSeqs: [...opts.sourceEventSeqs] } : {}, + ...opts?.surfaceOp !== undefined ? { + surfaceOp: typeof opts.surfaceOp === 'string' ? opts.surfaceOp : structuredClone(opts.surfaceOp), + } : {}, + } as SessionEvent this.log.push(event) this.onAppend?.(event) return event } /** - * Derive the LLM message history from the event log. + * Derive the LLM message history from the session surface (when surface + * markers exist) or from a linear scan of the raw event log (legacy sessions + * without surface markers). * * - `user/message` → user message * - `assistant/message` → assistant message (chunks are skipped — they are @@ -163,43 +200,62 @@ export class Session { * negligible next to a model call. */ deriveMessages(): Message[] { + if (this.surface.hasSurface) { + const messages: Message[] = [] + for (const node of this.surface.nodes) { + // Surface nodes are built from this.log — node.seq is always a valid + // index by construction. The non-null assertion expresses that invariant. + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const msg = this._deriveOneMessage(this.log[node.seq]!) + if (msg) messages.push(msg) + } + return messages + } + // Legacy path: linear scan for sessions without surface markers. const messages: Message[] = [] for (const event of this.log) { - // Intentionally non-exhaustive: only message-producing events derive - // history; turn/step boundaries, chunks, usage, and errors are - // trace/replay data. - // eslint-disable-next-line @typescript-eslint/switch-exhaustiveness-check - switch (event.type) { - case 'user/message': { - messages.push({ role: 'user', content: structuredClone(event.data.content) }) - break - } - case 'assistant/message': { - messages.push({ role: 'assistant', content: structuredClone(event.data.content) }) - break - } - case 'tool/result': { - const { callId, content, isError } = event.data - messages.push({ - role: 'user', - content: [{ type: 'tool-result', toolCallId: callId, content: structuredClone(content), isError }], - }) - break - } - case 'context/message': { - const { content, source } = event.data - messages.push({ role: 'user', content: renderTagged('context', structuredClone(content), source) }) - break - } - case 'steering/message': { - const { content, source } = event.data - messages.push({ role: 'user', content: renderTagged('steering', structuredClone(content), source) }) - break - } - } + const msg = this._deriveOneMessage(event) + if (msg) messages.push(msg) } return messages } + + /** + * Derive a single LLM message from one event, or null if the event type + * does not produce a message. Extracted so both the surface path and the + * legacy linear-scan path share the same derivation rules. + */ + private _deriveOneMessage(event: SessionEvent): Message | null { + // Intentionally non-exhaustive: only message-producing events derive + // history; turn/step boundaries, chunks, usage, and errors are + // trace/replay data. + + switch (event.type) { + case 'user/message': { + return { role: 'user', content: structuredClone(event.data.content) } + } + case 'assistant/message': { + return { role: 'assistant', content: structuredClone(event.data.content) } + } + case 'tool/result': { + const { callId, content, isError } = event.data + return { + role: 'user', + content: [{ type: 'tool-result', toolCallId: callId, content: structuredClone(content), isError }], + } + } + case 'context/message': { + const { content, source } = event.data + return { role: 'user', content: renderTagged('context', structuredClone(content), source) } + } + case 'steering/message': { + const { content, source } = event.data + return { role: 'user', content: renderTagged('steering', structuredClone(content), source) } + } + default: + return null + } + } } /** diff --git a/packages/session/src/repair.ts b/packages/session/src/repair.ts index 6a3f60a681..d313af12c3 100644 --- a/packages/session/src/repair.ts +++ b/packages/session/src/repair.ts @@ -61,7 +61,12 @@ export function interruptedTurnClosers(events: readonly SessionEvent[]): Session // call is "pending" until its matching tool/result arrives. Reset at every // turn boundary so a committed earlier turn (already balanced) never leaks a // phantom pending call into the interrupted-turn repair. - const pendingCalls = new Map() + // Track pending tool calls with their callSeq (the seq of the `tool/call` + // event, captured for surface sourceEventSeqs provenance on the synthetic + // result). CallSeq is set from `tool/call` events; the assistant/message + // block scan may register a call first (it appears earlier in the log), and + // the later `tool/call` event fills in the seq. + const pendingCalls = new Map() for (const event of events) { switch (event.type) { case 'turn/start': @@ -87,6 +92,18 @@ export function interruptedTurnClosers(events: readonly SessionEvent[]): Session if (block.type === 'tool-call') pendingCalls.set(block.id, { step: event.data.step }) } break + case 'tool/call': + // Capture the tool/call event seq for surface provenance on the + // synthesized tool/result. The entry may already exist (registered by + // the assistant/message above) or may be new (if the assistant/message + // came from a prior step that was already closed). + { + const entry = pendingCalls.get(event.data.callId) + if (entry) { + entry.callSeq = event.seq + } + } + break case 'tool/result': pendingCalls.delete(event.data.callId) break @@ -112,7 +129,7 @@ export function interruptedTurnClosers(events: readonly SessionEvent[]): Session // crash, so deriveMessages() yields a valid provider transcript on resume (a // dangling assistant tool-call is rejected by every provider). Insertion // order follows the Map (insertion = log order of the assistant messages). - for (const [callId, { step }] of pendingCalls) { + for (const [callId, { step, callSeq }] of pendingCalls) { closers.push({ type: 'tool/result', seq: seq++, @@ -125,6 +142,8 @@ export function interruptedTurnClosers(events: readonly SessionEvent[]): Session isError: true, error: { name: 'InterruptedError', code: 'interrupted' }, }, + surfaceOp: 'append', + ...callSeq !== undefined ? { sourceEventSeqs: [callSeq] } : {}, }) } diff --git a/packages/session/src/surface.ts b/packages/session/src/surface.ts new file mode 100644 index 0000000000..a09dbd001f --- /dev/null +++ b/packages/session/src/surface.ts @@ -0,0 +1,129 @@ +/** + * Surface layer on top of the session event log: a derived, cached linked list + * of events that produce LLM messages. Rebuilt deterministically from + * `surfaceOp` markers in the log — the log is the source of truth; the surface + * is a view. + * + * @module @deepseek-ai/dsh-session/surface + */ + +import type { SessionEvent, SurfaceOp } from './types.ts' + +/** One node in the surface linked list. */ +export interface SurfaceNode { + /** The event seq of this surface node. */ + seq: number + /** The previous surface node's seq, or null if this is the head. */ + prev: number | null + /** The next surface node's seq, or null if this is the tail. */ + next: number | null +} + +/** + * Maintains a cached linked list of surface nodes, rebuilt lazily from + * `surfaceOp` markers in the event log. Because the log is append-only, it + * processes only the delta since the last rebuild — new events are folded + * into the existing surface in O(new events) rather than rescanning the + * whole log. + */ +export class SurfaceManager { + /** Surface nodes in linked-list order (head to tail). Empty until first access. */ + private _nodes: SurfaceNode[] = [] + /** Map from event seq → node for O(1) lookup during replacements. */ + private _nodeBySeq = new Map() + /** The last processed seq. -1 forces a full rebuild on first access. */ + private _lastProcessedSeq = -1 + + constructor(private log: readonly SessionEvent[]) {} + + /** + * Reset to unprocessed state. Call after the log has been replaced + * wholesale (e.g. after Session seed). Not needed for normal appends — + * those are picked up incrementally. + */ + invalidate(): void { + this._lastProcessedSeq = -1 + this._nodes = [] + this._nodeBySeq.clear() + } + + /** The surface nodes in linked-list order (head to tail). */ + get nodes(): readonly SurfaceNode[] { + if (this._lastProcessedSeq < this.log.length - 1) this._processDelta() + return this._nodes + } + + /** Whether any event in the log carries `surfaceOp` markers. */ + get hasSurface(): boolean { + if (this._nodes.length > 0) return true + // Never processed anything — scan the whole log. + if (this._lastProcessedSeq === -1) return this.log.some(e => e.surfaceOp !== undefined) + // Processed up to _lastProcessedSeq without finding surface nodes; check + // only new events. + for (let i = this._lastProcessedSeq + 1; i < this.log.length; i++) { + if (this.log[i]?.surfaceOp !== undefined) return true + } + return false + } + + /** + * Process events from `_lastProcessedSeq + 1` through the end of the log, + * folding new surface markers into the existing linked list. + */ + private _processDelta(): void { + for (let i = this._lastProcessedSeq + 1; i < this.log.length; i++) { + const event = this.log[i] + if (event === undefined || event.surfaceOp === undefined) continue + + if (event.surfaceOp === 'append') { + const tail = this._nodes.length > 0 ? this._nodes[this._nodes.length - 1] : undefined + const node: SurfaceNode = { seq: event.seq, prev: tail?.seq ?? null, next: null } + if (tail) tail.next = event.seq + this._nodes.push(node) + this._nodeBySeq.set(event.seq, node) + } else { + this._replace(this._nodes, this._nodeBySeq, event.seq, event.surfaceOp) + } + } + this._lastProcessedSeq = this.log.length - 1 + } + + /** Apply a replace operation to the in-progress surface. */ + private _replace( + nodes: SurfaceNode[], + nodeBySeq: Map, + newSeq: number, + op: Extract, + ): void { + const startIdx = nodes.findIndex(n => n.seq === op.start) + if (startIdx === -1) { + throw new Error(`surface replace: start seq ${op.start} not found in surface`) + } + const endIdx = nodes.findIndex(n => n.seq === op.end) + if (endIdx === -1) { + throw new Error(`surface replace: end seq ${op.end} not found in surface`) + } + if (startIdx > endIdx) { + throw new Error(`surface replace: start seq ${op.start} (index ${startIdx}) is after end seq ${op.end} (index ${endIdx})`) + } + + // Remove shadowed nodes from `[startIdx, endIdx]` inclusive. + const count = endIdx - startIdx + 1 + const removed = nodes.splice(startIdx, count) + for (const r of removed) nodeBySeq.delete(r.seq) + + // Insert the new node where the removed range was. + const prevNode = startIdx > 0 ? nodes[startIdx - 1] : undefined + const nextNode = startIdx < nodes.length ? nodes[startIdx] : undefined + + const newNode: SurfaceNode = { + seq: newSeq, + prev: prevNode?.seq ?? null, + next: nextNode?.seq ?? null, + } + if (prevNode) prevNode.next = newSeq + if (nextNode) nextNode.prev = newSeq + nodes.splice(startIdx, 0, newNode) + nodeBySeq.set(newSeq, newNode) + } +} diff --git a/packages/session/src/types.ts b/packages/session/src/types.ts index d8ffaf9dc8..9f2cd87643 100644 --- a/packages/session/src/types.ts +++ b/packages/session/src/types.ts @@ -160,6 +160,34 @@ export interface SessionEventMap { export type SessionEventType = keyof SessionEventMap +/** + * How a session event entered the surface linked list. Absent for non-surface + * events (boundaries, chunks, usage, errors). + * + * - `'append'`: added to the tail — normal path for user/assistant/tool/context + * messages. + * - `{ op: 'replace', start, end }`: replaces surface nodes from `start` + * (inclusive) through `end` (inclusive) with this node. Both must exist as + * surface nodes in the current surface. `start === end` replaces a single + * node. The node's {@link SessionEvent.sourceEventSeqs} must include every + * shadowed surface node. Used by compaction and possible other manipulations. + */ +export type SurfaceOp = + | 'append' + | { op: 'replace'; start: number; end: number } + +/** + * Optional surface metadata passed to {@link Session.append}. + * `surfaceOp` controls how the event enters the surface linked list; + * `sourceEventSeqs` records the seq numbers of events that are provenance + * sources of this one (e.g. the `assistant/chunk` seqs behind an + * `assistant/message`, or the shadowed nodes behind a compaction replacement). + */ +export interface SurfaceAppendOpts { + surfaceOp?: SurfaceOp + sourceEventSeqs?: number[] +} + /** * One immutable entry in the session log. * @@ -174,5 +202,13 @@ export type SessionEvent = { /** Unix epoch milliseconds. */ time: number data: SessionEventMap[K] + /** + * Seq numbers of events that are provenance sources of this event + * (e.g. the `assistant/chunk` seqs that built an `assistant/message`, + * or the surface nodes shadowed by a compaction marker). + */ + sourceEventSeqs?: number[] + /** How this event entered the surface; absent for non-surface events. */ + surfaceOp?: SurfaceOp } }[T] diff --git a/packages/session/tests/repair.spec.ts b/packages/session/tests/repair.spec.ts index 893015b218..cf6fb2b51c 100644 --- a/packages/session/tests/repair.spec.ts +++ b/packages/session/tests/repair.spec.ts @@ -122,4 +122,36 @@ describe('interruptedTurnClosers', () => { const result = closers[0]! expect(result.type === 'tool/result' && result.data.callId).toBe('call-b') }) + + it('synthesized tool/result carries surfaceOp and sourceEventSeqs when tool/call was logged', () => { + const events: SessionEvent[] = [ + userTurnStart(1, 0), + { type: 'step/start', seq: 1, time: 1, data: { turn: 1, step: 1 } }, + { type: 'assistant/message', seq: 2, time: 2, data: { turn: 1, step: 1, content: [ + { type: 'tool-call', id: CallId('call-1'), name: 'bash', arguments: '{}' }, + ] } }, + { type: 'tool/call', seq: 3, time: 3, data: { turn: 1, step: 1, callId: CallId('call-1'), name: 'bash', arguments: '{}' } }, + ] + const closers = interruptedTurnClosers(events) + expect(closers.map(e => e.type)).toEqual(['tool/result', 'step/end', 'turn/end']) + const result = closers[0]! + expect(result.surfaceOp).toBe('append') + expect(result.sourceEventSeqs).toEqual([3]) + }) + + it('handles tool/call without a matching assistant/message entry gracefully', () => { + // A tool/call event exists in the log but no assistant/message registered + // the callId in pendingCalls (e.g., a plugin appended it directly, or the + // assistant/message from a prior step didn't have this call). The repair + // should still close the turn — it just won't synthesize a result for this + // call (there's nothing to answer). + const events: SessionEvent[] = [ + userTurnStart(1, 0), + { type: 'step/start', seq: 1, time: 1, data: { turn: 1, step: 1 } }, + { type: 'tool/call', seq: 2, time: 2, data: { turn: 1, step: 1, callId: CallId('orphan'), name: 'bash', arguments: '{}' } }, + ] + const closers = interruptedTurnClosers(events) + // No pending calls → no synthetic tool/result, just step/end + turn/end. + expect(closers.map(e => e.type)).toEqual(['step/end', 'turn/end']) + }) }) diff --git a/packages/session/tests/surface.spec.ts b/packages/session/tests/surface.spec.ts new file mode 100644 index 0000000000..d8f503c5e5 --- /dev/null +++ b/packages/session/tests/surface.spec.ts @@ -0,0 +1,324 @@ +import { describe, expect, it } from 'vitest' +import type { SessionEvent } from '@deepseek-ai/dsh-session' +import { Session, SessionId } from '@deepseek-ai/dsh-session' +import { CallId } from '@deepseek-ai/dsh-llm' + +/** Build a minimal session with turn boundaries and a single user message. */ +function surfaceSession(): Session { + const s = new Session(SessionId('ss')) + s.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + s.append('user/message', { content: [{ type: 'text', text: 'hello' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) + s.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 'hi' }] }, { surfaceOp: 'append' }) + s.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) + return s +} + +describe('SurfaceManager', () => { + it('rebuilds a linked list from surfaceOp: append markers', () => { + const s = surfaceSession() + const nodes = s.surface.nodes + // Only the user/message and assistant/message carry surfaceOp: 'append'. + // The turn boundaries do not have surface markers. + expect(nodes.length).toBe(2) + expect(nodes[0]!.seq).toBe(1) // user/message (turn/start is seq 0) + expect(nodes[0]!.prev).toBeNull() + expect(nodes[0]!.next).toBe(2) // assistant/message (seq 2) + expect(nodes[1]!.seq).toBe(2) + expect(nodes[1]!.prev).toBe(1) + expect(nodes[1]!.next).toBeNull() + }) + + it('hasSurface returns false when no events have surfaceOp', () => { + const s = new Session(SessionId('nosurface')) + s.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + s.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }) + s.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) + expect(s.surface.hasSurface).toBe(false) + }) + + it('hasSurface returns true when any event has surfaceOp', () => { + const s = surfaceSession() + expect(s.surface.hasSurface).toBe(true) + }) + + it('hasSurface detects surface markers that arrive after initial processing', () => { + // Start with no surface markers. Access nodes first to set _lastProcessedSeq + // (via delta processing), keeping _nodes empty. Then append a mix of non-surface + // and surface events, and verify hasSurface detects via the delta-only check. + const s = new Session(SessionId('late')) + s.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + s.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }) + // Access nodes to trigger processing: sets _lastProcessedSeq = 1, _nodes = []. + expect(s.surface.nodes.length).toBe(0) + // Append non-surface events first (exercises the loop-continue branch), then + // a surface event (exercises the return-true branch). + s.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) + s.append('turn/start', { turn: 2, trigger: { kind: 'continuation' } }) + s.append('assistant/message', { turn: 2, step: 1, content: [] }, { surfaceOp: 'append' }) + // hasSurface checks only new seqs [2, 3, 4]; skips 2 and 3 (non-surface), + // finds surfaceOp on seq 4 and returns true. + expect(s.surface.hasSurface).toBe(true) + }) + + it('invalidate resets to full rebuild', () => { + const s = surfaceSession() + expect(s.surface.nodes.length).toBe(2) + // After invalidate, the surface should rebuild from scratch on next access. + ;(s.surface).invalidate() + expect(s.surface.nodes.length).toBe(2) // same result, but rebuilt + }) + + it('empty surface yields empty nodes', () => { + const s = new Session(SessionId('empty')) + // Only turn boundaries, no surface nodes. + s.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + s.append('step/start', { turn: 1, step: 1 }) + s.append('step/end', { turn: 1, step: 1 }) + s.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) + expect(s.surface.nodes.length).toBe(0) + expect(s.surface.hasSurface).toBe(false) + // deriveMessages returns empty array + expect(s.deriveMessages()).toEqual([]) + }) + + it('picks up new events incrementally (delta processing)', () => { + const s = surfaceSession() + expect(s.surface.nodes.length).toBe(2) + // Append another surface node + s.append('tool/result', { turn: 1, step: 1, callId: CallId('c1'), content: [{ type: 'text', text: 'ok' }], isError: false }, { surfaceOp: 'append' }) + expect(s.surface.nodes.length).toBe(3) + expect(s.surface.nodes[2]!.seq).toBe(4) // seq 4: after turn/end at seq 3 + expect(s.surface.nodes[2]!.prev).toBe(2) + expect(s.surface.nodes[1]!.next).toBe(4) + }) + + it('replays identically from a seeded log with surface markers', () => { + const original = surfaceSession() + original.append('tool/result', { turn: 1, step: 1, callId: CallId('c1'), content: [{ type: 'text', text: 'ok' }], isError: false }, { surfaceOp: 'append' }) + const replayed = new Session(SessionId('replay'), [...original.events]) + // Surface rebuilds from the seeded log's markers. + expect(replayed.surface.nodes.map(n => n.seq)).toEqual([1, 2, 4]) + expect(replayed.deriveMessages()).toEqual(original.deriveMessages()) + }) + + it('rebuild with replace operation splices out shadowed nodes', () => { + const s = surfaceSession() + // seq: 0=turn/start, 1=user, 2=assistant, 3=turn/end + // Surface nodes: seq 1 (user), seq 2 (assistant). + // Replace both with a compaction marker. Both 1 and 2 are valid surface seqs. + s.append('assistant/message', + { turn: 2, step: 1, content: [{ type: 'text', text: 'summary' }] }, + { surfaceOp: { op: 'replace', start: 1, end: 2 }, sourceEventSeqs: [1, 2] }, + ) + // Now the surface should have just the compaction node. + expect(s.surface.nodes.length).toBe(1) + expect(s.surface.nodes[0]!.seq).toBe(4) // seq of the compaction marker + expect(s.surface.nodes[0]!.prev).toBeNull() + expect(s.surface.nodes[0]!.next).toBeNull() + }) + + it('replace with both ends at real nodes splices only the range', () => { + const s = new Session(SessionId('range')) + s.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 0 + s.append('user/message', { content: [{ type: 'text', text: 'b' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 1 + s.append('user/message', { content: [{ type: 'text', text: 'c' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 2 + // Replace seq 0 through 1 inclusive: shadow a and b, keep c. + s.append('assistant/message', + { turn: 1, step: 1, content: [{ type: 'text', text: 'summary' }] }, + { surfaceOp: { op: 'replace', start: 0, end: 1 }, sourceEventSeqs: [0, 1] }, + ) // seq 3 + expect(s.surface.nodes.map(n => n.seq)).toEqual([3, 2]) + // Links: 3 ↔ 2 + expect(s.surface.nodes[0]!.prev).toBeNull() + expect(s.surface.nodes[0]!.next).toBe(2) + expect(s.surface.nodes[1]!.prev).toBe(3) + expect(s.surface.nodes[1]!.next).toBeNull() + }) + + it('single-node replacement (start === end)', () => { + const s = new Session(SessionId('single')) + s.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 0 + s.append('user/message', { content: [{ type: 'text', text: 'b' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 1 + // Replace only seq 1 (single node). + s.append('assistant/message', + { turn: 1, step: 1, content: [{ type: 'text', text: 'x' }] }, + { surfaceOp: { op: 'replace', start: 1, end: 1 }, sourceEventSeqs: [1] }, + ) // seq 2 + expect(s.surface.nodes.map(n => n.seq)).toEqual([0, 2]) + expect(s.surface.nodes[0]!.next).toBe(2) + expect(s.surface.nodes[1]!.prev).toBe(0) + }) + + it('throws when replace start is not found', () => { + const s = new Session(SessionId('bad-start')) + s.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 0 + s.append('assistant/message', + { turn: 1, step: 1, content: [{ type: 'text', text: 'y' }] }, + { surfaceOp: { op: 'replace', start: 5, end: 0 }, sourceEventSeqs: [5, 0] }, + ) + expect(() => s.surface.nodes).toThrow(/surface replace: start seq 5 not found/) + }) + + it('throws when replace end is not found', () => { + const s = new Session(SessionId('bad-end')) + s.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 0 + s.append('assistant/message', + { turn: 1, step: 1, content: [{ type: 'text', text: 'y' }] }, + { surfaceOp: { op: 'replace', start: 0, end: 99 }, sourceEventSeqs: [0] }, + ) + expect(() => s.surface.nodes).toThrow(/surface replace: end seq 99 not found/) + }) + + it('throws when start is after end', () => { + const s = new Session(SessionId('reversed')) + s.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 0 + s.append('user/message', { content: [{ type: 'text', text: 'b' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 1 + // start=1, end=0 would be reversed order. + s.append('assistant/message', + { turn: 1, step: 1, content: [{ type: 'text', text: 'y' }] }, + { surfaceOp: { op: 'replace', start: 1, end: 0 }, sourceEventSeqs: [1, 0] }, + ) + expect(() => s.surface.nodes).toThrow(/start seq 1.*after end seq 0/) + }) + + it('sourceEventSeqs is snapshot so caller mutation does not affect logged event', () => { + const s = new Session(SessionId('immutable')) + const sources = [10, 20] + s.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 'h' }] }, { surfaceOp: 'append', sourceEventSeqs: sources }) + // Mutate caller's array after append. + sources.push(30) + sources[0] = 99 + const logged = s.events[0]! + expect(logged.sourceEventSeqs).toEqual([10, 20]) + }) + + it('replace starting at non-head position links to previous node correctly', () => { + const s = new Session(SessionId('mid-replace')) + s.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 0 + s.append('user/message', { content: [{ type: 'text', text: 'b' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 1 + s.append('user/message', { content: [{ type: 'text', text: 'c' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 2 + // Replace the middle node (seq 1) only, keeping seq 0 and seq 2. + s.append('assistant/message', + { turn: 1, step: 1, content: [{ type: 'text', text: 'x' }] }, + { surfaceOp: { op: 'replace', start: 1, end: 1 }, sourceEventSeqs: [1] }, + ) // seq 3 + expect(s.surface.nodes.map(n => n.seq)).toEqual([0, 3, 2]) + // Links: 0 → 3 → 2 + expect(s.surface.nodes[0]!.prev).toBeNull() + expect(s.surface.nodes[0]!.next).toBe(3) + expect(s.surface.nodes[1]!.prev).toBe(0) + expect(s.surface.nodes[1]!.next).toBe(2) + expect(s.surface.nodes[2]!.prev).toBe(3) + expect(s.surface.nodes[2]!.next).toBeNull() + }) + + it('surfaceOp replace object is snapshot so caller mutation is isolated', () => { + const s = new Session(SessionId('immutable-op')) + s.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) + const op = { op: 'replace' as const, start: 0, end: 0 } + s.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 's' }] }, { surfaceOp: op, sourceEventSeqs: [0] }) + // Mutate caller's object after append. + op.start = 99 + const logged = s.events[1]! + expect(logged.surfaceOp).toEqual({ op: 'replace', start: 0, end: 0 }) + }) +}) + +describe('deriveMessages with surface', () => { + it('uses the surface path when surface markers are present', () => { + const s = surfaceSession() + const messages = s.deriveMessages() + expect(messages).toHaveLength(2) + expect(messages[0]!.role).toBe('user') + expect(messages[0]!.content[0]).toMatchObject({ type: 'text', text: 'hello' }) + expect(messages[1]!.role).toBe('assistant') + expect(messages[1]!.content[0]).toMatchObject({ type: 'text', text: 'hi' }) + }) + + it('falls back to linear scan when no surface markers exist', () => { + const s = new Session(SessionId('legacy')) + s.append('user/message', { content: [{ type: 'text', text: 'q' }], source: { kind: 'user' } }) + s.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 'a' }] }) + const messages = s.deriveMessages() + expect(messages).toHaveLength(2) + expect(messages[0]!.role).toBe('user') + expect(messages[1]!.role).toBe('assistant') + }) + + it('surface path skips non-surface events (chunks, boundaries)', () => { + const s = new Session(SessionId('filter')) + s.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + s.append('assistant/chunk', { turn: 1, step: 1, chunk: { type: 'text-delta', index: 0, text: 'h' } }) + s.append('assistant/chunk', { turn: 1, step: 1, chunk: { type: 'text-delta', index: 1, text: 'i' } }) + s.append('user/message', { content: [{ type: 'text', text: 'hello' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) + s.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 'hi' }] }, { surfaceOp: 'append' }) + s.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) + // Chunks and boundaries are NOT in the surface, so only 2 messages. + expect(s.deriveMessages()).toHaveLength(2) + }) + + it('deriveMessages via surface respects replace (shadowed nodes are excluded)', () => { + const s = new Session(SessionId('compacted')) + s.append('user/message', { content: [{ type: 'text', text: 'original' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) + s.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 'compacted' }] }, { surfaceOp: { op: 'replace', start: 0, end: 0 }, sourceEventSeqs: [0] }) + // Only the compaction node is visible. + const messages = s.deriveMessages() + expect(messages).toHaveLength(1) + expect(messages[0]!.content[0]).toMatchObject({ type: 'text', text: 'compacted' }) + }) + + it('context/message and steering/message appear on surface', () => { + const s = new Session(SessionId('ctx')) + s.append('context/message', { content: [{ type: 'text', text: 'file changed' }], source: { kind: 'plugin', plugin: 'watcher' } }, { surfaceOp: 'append' }) + s.append('steering/message', { turn: 1, content: [{ type: 'text', text: 'focus' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) + const messages = s.deriveMessages() + expect(messages).toHaveLength(2) + expect(messages[0]!.content[0]).toMatchObject({ type: 'text', text: '' }) + expect(messages[1]!.content[0]).toMatchObject({ type: 'text', text: '' }) + }) +}) + +describe('Session.append surface opts', () => { + it('records sourceEventSeqs and surfaceOp on the event', () => { + const s = new Session(SessionId('opts')) + const event = s.append('assistant/message', + { turn: 1, step: 1, content: [{ type: 'text', text: 'h' }] }, + { surfaceOp: 'append', sourceEventSeqs: [3, 5, 7] }, + ) + expect(event.sourceEventSeqs).toEqual([3, 5, 7]) + expect(event.surfaceOp).toBe('append') + // The logged event matches the returned event. + expect(s.events[0]!.sourceEventSeqs).toEqual([3, 5, 7]) + expect(s.events[0]!.surfaceOp).toBe('append') + }) + + it('deriveMessages skips surface nodes whose event type is not message-producing', () => { + // A surface node with a type not handled by _deriveOneMessage (e.g., 'usage' + // placed on surface) should be skipped — the null-check in the surface + // derivation path is exercised. + const seed: SessionEvent[] = [ + { type: 'turn/start', seq: 0, time: 1, data: { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } } }, + { type: 'step/start', seq: 1, time: 2, data: { turn: 1, step: 1 } }, + { type: 'usage', seq: 2, time: 3, data: { turn: 1, step: 1, usage: { inputTokens: 0, outputTokens: 0 } }, surfaceOp: 'append' as const }, + { type: 'step/end', seq: 3, time: 4, data: { turn: 1, step: 1 } }, + { type: 'turn/end', seq: 4, time: 5, data: { turn: 1, reason: { kind: 'completed' } } }, + ] + const s = new Session(SessionId('nomessage'), seed) + // The usage event is on the surface but _deriveOneMessage returns null for it. + expect(s.deriveMessages()).toHaveLength(0) + }) + + it('append without surface opts produces an event without surface fields', () => { + const s = new Session(SessionId('noopts')) + s.append('user/message', { content: [{ type: 'text', text: 'x' }], source: { kind: 'user' } }) + expect(s.events[0]!.sourceEventSeqs).toBeUndefined() + expect(s.events[0]!.surfaceOp).toBeUndefined() + }) + + it('surfaceOp primitives are not cloned (they are immutable)', () => { + const s = new Session(SessionId('prim')) + const event = s.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append' }) + // The string 'append' is a primitive — identity-preserving is fine. + expect(event.surfaceOp).toBe('append') + }) +}) From e5d82631942a55916d6199b8cac522a44db51db7 Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Thu, 18 Jun 2026 09:37:19 +0800 Subject: [PATCH 2/9] fix broken cross-links --- docs/rfc/implemented/2026-06-18-session-surface.md | 6 +++--- packages/session-persistence-sqlite/README.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/rfc/implemented/2026-06-18-session-surface.md b/docs/rfc/implemented/2026-06-18-session-surface.md index 159db911b1..528718b968 100644 --- a/docs/rfc/implemented/2026-06-18-session-surface.md +++ b/docs/rfc/implemented/2026-06-18-session-surface.md @@ -1,10 +1,10 @@ -# ADR 0019: Session surface — a linked list over the event log for LLM message derivation +# RFC: Session surface — a linked list over the event log for LLM message derivation -Status: accepted (2026-06-17) +Status: implemented (accepted 2026-06-18) ## Context -The `Session` event log is the single source of truth ([ADR 0003](0003-event-sourced-sessions.md)), but the only view over it was `deriveMessages()` — a linear scan that filtered and transformed raw events into `Message[]`. This creates problems for session-history-manipulating plugins (compaction, tool-call result pruning, etc.). Without a central mechanism, each plugin would need to wrap `agent/request` to rewrite the message list — a pattern that suffers from listener-ordering fragility, provides no durable record of what was changed, and forces repeated changes to the core `deriveMessages()` whenever a new manipulation is added. A central hub in the `session` package, with a provenance-recording mechanism and enough flexibility for future plugins to manipulate session history through a stable API, lays a solid foundation for plugin development. +The `Session` event log is the single source of truth ([event-sourced sessions](../implemented/2026-06-11-event-sourced-sessions.md)), but the only view over it was `deriveMessages()` — a linear scan that filtered and transformed raw events into `Message[]`. This creates problems for session-history-manipulating plugins (compaction, tool-call result pruning, etc.). Without a central mechanism, each plugin would need to wrap `agent/request` to rewrite the message list — a pattern that suffers from listener-ordering fragility, provides no durable record of what was changed, and forces repeated changes to the core `deriveMessages()` whenever a new manipulation is added. A central hub in the `session` package, with a provenance-recording mechanism and enough flexibility for future plugins to manipulate session history through a stable API, lays a solid foundation for plugin development. ## Decision diff --git a/packages/session-persistence-sqlite/README.md b/packages/session-persistence-sqlite/README.md index cec1e9700f..6eb8568e46 100644 --- a/packages/session-persistence-sqlite/README.md +++ b/packages/session-persistence-sqlite/README.md @@ -6,7 +6,7 @@ A SQLite durable session-persistence backend — a second `SessionPersistence` i ## Storage model -Each `SessionEvent` maps 1:1 onto a row in an `events` table `(session_id, seq, type, time, data, source_event_seqs, surface_op)` — `data` is the event payload as JSON text, so the row shape is the event verbatim (including `assistant/chunk`, keeping `seq` contiguous). The two `TEXT` columns `source_event_seqs` and `surface_op` are nullable; they store the event's optional surface-metadata fields (see [ADR 0019](../../docs/adr/0019-session-surface.md)). The schema migrates from v1 to v2 via `ALTER TABLE ADD COLUMN` — existing rows get NULL for both columns, which is correct for events written before surface support. Out-of-log metadata (`SessionMeta`) lives in a `sessions` row, including the mutable `SessionSummary` fields (`updatedAt`, `title`, `firstPrompt`) that `update()` rewrites without touching the event log. A `sessions` row is written only by the first `append` — its existence is the lazy-materialization signal (`has`/`list` report exactly the sessions that have a row), so no separate column is needed. +Each `SessionEvent` maps 1:1 onto a row in an `events` table `(session_id, seq, type, time, data, source_event_seqs, surface_op)` — `data` is the event payload as JSON text, so the row shape is the event verbatim (including `assistant/chunk`, keeping `seq` contiguous). The two `TEXT` columns `source_event_seqs` and `surface_op` are nullable; they store the event's optional surface-metadata fields (see [session surface](../../docs/rfc/implemented/2026-06-18-session-surface.md)). The schema migrates from v1 to v2 via `ALTER TABLE ADD COLUMN` — existing rows get NULL for both columns, which is correct for events written before surface support. Out-of-log metadata (`SessionMeta`) lives in a `sessions` row, including the mutable `SessionSummary` fields (`updatedAt`, `title`, `firstPrompt`) that `update()` rewrites without touching the event log. A `sessions` row is written only by the first `append` — its existence is the lazy-materialization signal (`has`/`list` report exactly the sessions that have a row), so no separate column is needed. The repo targets Node ≥ 24 (the root `engines` field), which includes the stable `node:sqlite` module. The database opens with `foreign_keys = ON` (so `ON DELETE CASCADE` drops a session's events with its row) and `journal_mode = WAL`. The table-layout version is stored in `PRAGMA user_version` and checked on open: a fresh database is stamped with the current `SCHEMA_VERSION`; a database written by a newer, incompatible build (higher `user_version`) is rejected rather than opened against an unknown layout. From 0279cf09d678ec0298715f61329a2e9b968f8e3f Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Thu, 18 Jun 2026 13:26:50 +0800 Subject: [PATCH 3/9] feat(session): enforce SurfaceEvent type --- packages/invariants/src/index.ts | 35 ++++++++++++---- packages/invariants/tests/invariants.spec.ts | 26 +++++++++++- .../session-persistence-sqlite/src/index.ts | 7 ++-- .../session-persistence-sqlite/src/schema.ts | 16 +++---- .../tests/sqlite.spec.ts | 28 ++++++------- packages/session/src/index.ts | 36 ++++++++++++---- packages/session/src/surface.ts | 42 ++++++++++++++++--- packages/session/src/types.ts | 40 ++++++++++++++++-- packages/session/tests/repair.spec.ts | 6 +-- packages/session/tests/surface.spec.ts | 16 +++---- 10 files changed, 189 insertions(+), 63 deletions(-) diff --git a/packages/invariants/src/index.ts b/packages/invariants/src/index.ts index 7655be46d6..97cb0af680 100644 --- a/packages/invariants/src/index.ts +++ b/packages/invariants/src/index.ts @@ -22,7 +22,7 @@ import type { Context } from 'cordis' import { HarnessError } from '@deepseek-ai/dsh-llm' import type { Agent, AgentStatus } from '@deepseek-ai/dsh-agent' -import type { Session, SessionEvent } from '@deepseek-ai/dsh-session' +import type { Session, SessionEvent, SurfaceEventType } from '@deepseek-ai/dsh-session' export const name = 'invariants' export const inject = ['sessions'] @@ -108,15 +108,32 @@ function checkEvent(trace: SessionTrace, event: SessionEvent): void { trace.lastSeq = event.seq // --- Surface invariants --- - if (event.sourceEventSeqs !== undefined) { - if (event.sourceEventSeqs.length === 0) { + // Surface metadata (sourceEventSeqs, surfaceOp) is only valid on + // surface-eligible event types. The compiler enforces this at append() + // call sites; this runtime check catches casts and persisted data. + const SURFACE_TYPES = new Set(['user/message', 'assistant/message', 'tool/result', 'context/message', 'steering/message']) + // Cast to surface-eligible event type so we can access surfaceOp and + // sourceEventSeqs (optional on SessionEvent, mandatory on SurfaceEvent). + // SurfaceEvent's mandatory surfaceOp is too strict here — we need to + // CHECK whether surface metadata is present, not assume it. + const se = event as SessionEvent + if (!SURFACE_TYPES.has(event.type)) { + if (se.sourceEventSeqs !== undefined) { + throw new InvariantError(`${event.type} cannot carry sourceEventSeqs (non-surface event)`) + } + if (se.surfaceOp !== undefined) { + throw new InvariantError(`${event.type} cannot carry surfaceOp (non-surface event)`) + } + } + if (se.sourceEventSeqs !== undefined) { + if (se.sourceEventSeqs.length === 0) { throw new InvariantError('sourceEventSeqs must not be empty when present') } - const unique = new Set(event.sourceEventSeqs) - if (unique.size !== event.sourceEventSeqs.length) { + const unique = new Set(se.sourceEventSeqs) + if (unique.size !== se.sourceEventSeqs.length) { throw new InvariantError('sourceEventSeqs must not contain duplicates') } - for (const ref of event.sourceEventSeqs) { + for (const ref of se.sourceEventSeqs) { if (ref >= event.seq) { throw new InvariantError(`sourceEventSeqs must reference earlier events: ${ref} >= current seq ${event.seq}`) } @@ -125,9 +142,9 @@ function checkEvent(trace: SessionTrace, event: SessionEvent): void { } } } - if (event.surfaceOp !== undefined && typeof event.surfaceOp !== 'string') { - if (event.surfaceOp.start > event.surfaceOp.end) { - throw new InvariantError(`surface replace: start ${event.surfaceOp.start} must be <= end ${event.surfaceOp.end}`) + if (se.surfaceOp !== undefined && typeof se.surfaceOp !== 'string') { + if (se.surfaceOp.start > se.surfaceOp.end) { + throw new InvariantError(`surface replace: start ${se.surfaceOp.start} must be <= end ${se.surfaceOp.end}`) } } diff --git a/packages/invariants/tests/invariants.spec.ts b/packages/invariants/tests/invariants.spec.ts index 8e4c5edebe..28fd27163f 100644 --- a/packages/invariants/tests/invariants.spec.ts +++ b/packages/invariants/tests/invariants.spec.ts @@ -105,7 +105,11 @@ describe('session-log invariants', () => { expect(() => session.append('error', { turn: 1, step: 1, message: 'boom' })) .toThrow(/outside any open turn/) // A PLUGIN-added (merge-extensible) event type is caught by the default too. - expect(() => session.append('compaction/marker' as never, { foo: 'bar' } as never)) + // Cast through `any`: 'compaction/marker' is not in SessionEventType (it's + // merge-extensible), so the typed append() won't accept it. The test verifies + // the runtime default-branch turn-enclosure check. + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-return + expect(() => (session.append as any)('compaction/marker', { foo: 'bar' })) .toThrow(/outside any open turn/) }) @@ -498,4 +502,24 @@ describe('surface invariants', () => { session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 2, end: 1 }, sourceEventSeqs: [2] }) }).toThrow(/must be <= end/) }) + + it('rejects sourceEventSeqs on a non-surface event', async () => { + const { ctx } = await setup() + const session = ctx.sessions.create() + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + // Type system prevents surface metadata on non-surface events; this test + // exercises the runtime guard against casts or persisted-data bypass. + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-return + expect(() => (session.append as any)('turn/end', { turn: 1, reason: { kind: 'completed' } }, { sourceEventSeqs: [0] })) + .toThrow(/cannot carry sourceEventSeqs/) + }) + + it('rejects surfaceOp on a non-surface event', async () => { + const { ctx } = await setup() + const session = ctx.sessions.create() + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-return + expect(() => (session.append as any)('turn/end', { turn: 1, reason: { kind: 'completed' } }, { surfaceOp: 'append' })) + .toThrow(/cannot carry surfaceOp/) + }) }) diff --git a/packages/session-persistence-sqlite/src/index.ts b/packages/session-persistence-sqlite/src/index.ts index 3475f8b4dd..de3e96fda5 100644 --- a/packages/session-persistence-sqlite/src/index.ts +++ b/packages/session-persistence-sqlite/src/index.ts @@ -26,7 +26,7 @@ import { mkdir } from 'node:fs/promises' import { dirname, resolve } from 'node:path' import { SessionPersistence } from '@deepseek-ai/dsh-session-persistence' import { isJsonValue, interruptedTurnClosers } from '@deepseek-ai/dsh-session' -import type { Session, SessionEvent, SessionId, SessionMeta, SessionSummary } from '@deepseek-ai/dsh-session' +import type { Session, SessionEvent, SurfaceEventType, SessionId, SessionMeta, SessionSummary } from '@deepseek-ai/dsh-session' import { openDatabase, rowToMeta, scanRows, type EventRow, type SessionRow, } from './schema.ts' @@ -39,9 +39,10 @@ export { SCHEMA_VERSION } from './schema.ts' * events, events written before surface support). */ function surfaceBindings(event: SessionEvent): [string | null, string | null] { + const se = event as SessionEvent return [ - event.sourceEventSeqs ? JSON.stringify(event.sourceEventSeqs) : null, - event.surfaceOp !== undefined ? JSON.stringify(event.surfaceOp) : null, + se.sourceEventSeqs ? JSON.stringify(se.sourceEventSeqs) : null, + se.surfaceOp !== undefined ? JSON.stringify(se.surfaceOp) : null, ] } diff --git a/packages/session-persistence-sqlite/src/schema.ts b/packages/session-persistence-sqlite/src/schema.ts index 68ff4905ae..974fd77778 100644 --- a/packages/session-persistence-sqlite/src/schema.ts +++ b/packages/session-persistence-sqlite/src/schema.ts @@ -126,19 +126,19 @@ export function rowToMeta(row: SessionRow): SessionMeta { /** Reconstruct a {@link SessionEvent} from an `events` row (parses `data`). */ export function rowToEvent(row: EventRow): SessionEvent { - const event = { + // Surface-metadata fields are conditional on the event type in the type + // system; spread them so each variant gets only the fields it declares. + const surfaceFields = { + ...row.source_event_seqs !== null ? { sourceEventSeqs: JSON.parse(row.source_event_seqs) as number[] } : {}, + ...row.surface_op !== null ? { surfaceOp: JSON.parse(row.surface_op) as SurfaceOp } : {}, + } + return { type: row.type as SessionEvent['type'], seq: row.seq, time: row.time, data: JSON.parse(row.data) as SessionEvent['data'], + ...surfaceFields, } as SessionEvent - if (row.source_event_seqs !== null) { - event.sourceEventSeqs = JSON.parse(row.source_event_seqs) as number[] - } - if (row.surface_op !== null) { - event.surfaceOp = JSON.parse(row.surface_op) as SurfaceOp - } - return event } /** diff --git a/packages/session-persistence-sqlite/tests/sqlite.spec.ts b/packages/session-persistence-sqlite/tests/sqlite.spec.ts index 61b166f257..252e9c4660 100644 --- a/packages/session-persistence-sqlite/tests/sqlite.spec.ts +++ b/packages/session-persistence-sqlite/tests/sqlite.spec.ts @@ -5,7 +5,7 @@ import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import SessionStore, { SessionId } from '@deepseek-ai/dsh-session' -import type { Session, SessionEvent, SessionMeta } from '@deepseek-ai/dsh-session' +import type { Session, SessionEvent, SurfaceEvent, SurfaceEventType, SessionMeta } from '@deepseek-ai/dsh-session' import SessionPersistenceSqlite, { SCHEMA_VERSION } from '@deepseek-ai/dsh-session-persistence-sqlite' import { openDatabase, rowToEvent, scanRows, type EventRow } from '../src/schema.ts' import { runPersistenceContract, meta, oneTurnLog } from '../../session-persistence/tests/contract.ts' @@ -800,8 +800,8 @@ describe('surface field round-trip', () => { surface_op: JSON.stringify('append'), } const event = rowToEvent(row) - expect(event.sourceEventSeqs).toEqual([3, 5]) - expect(event.surfaceOp).toBe('append') + expect((event as SurfaceEvent).sourceEventSeqs).toEqual([3, 5]) + expect((event as SurfaceEvent).surfaceOp).toBe('append') }) it('rowToEvent handles replace surfaceOp object', () => { @@ -812,8 +812,8 @@ describe('surface field round-trip', () => { surface_op: JSON.stringify({ op: 'replace', start: 0, end: 1 }), } const event = rowToEvent(row) - expect(event.sourceEventSeqs).toEqual([0, 1]) - expect(event.surfaceOp).toEqual({ op: 'replace', start: 0, end: 1 }) + expect((event as SurfaceEvent).sourceEventSeqs).toEqual([0, 1]) + expect((event as SurfaceEvent).surfaceOp).toEqual({ op: 'replace', start: 0, end: 1 }) }) it('scanRows with surface columns reconstructs events with surface fields', () => { @@ -827,9 +827,9 @@ describe('surface field round-trip', () => { ] const { preserved } = scanRows(rows) expect(preserved).toHaveLength(2) - expect(preserved[0]!.surfaceOp).toEqual({ op: 'replace', start: 0, end: 0 }) - expect(preserved[0]!.sourceEventSeqs).toBeUndefined() - expect(preserved[1]!.surfaceOp).toBeUndefined() + expect((preserved[0]! as SurfaceEvent).surfaceOp).toEqual({ op: 'replace', start: 0, end: 0 }) + expect((preserved[0]! as SurfaceEvent).sourceEventSeqs).toBeUndefined() + expect((preserved[1] as SessionEvent).surfaceOp).toBeUndefined() }) it('append and load round-trips surface fields through SQLite', async () => { @@ -845,11 +845,11 @@ describe('surface field round-trip', () => { const loaded = await ctx.sessionPersistence.load(SessionId('roundtrip-surface')) expect(loaded.events).toHaveLength(4) const um = loaded.events[1]! - expect(um.surfaceOp).toBe('append') - expect(um.sourceEventSeqs).toBeUndefined() + expect((um as SurfaceEvent).surfaceOp).toBe('append') + expect((um as SurfaceEvent).sourceEventSeqs).toBeUndefined() const am = loaded.events[2]! - expect(am.surfaceOp).toBe('append') - expect(am.sourceEventSeqs).toEqual([0]) + expect((am as SurfaceEvent).surfaceOp).toBe('append') + expect((am as SurfaceEvent).sourceEventSeqs).toEqual([0]) await fiber.dispose() }) @@ -863,8 +863,8 @@ describe('surface field round-trip', () => { session.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) await ctx.parallel('session/flush', session) const loaded = await ctx.sessionPersistence.load(SessionId('surface-noseq')) - expect(loaded.events[1]!.surfaceOp).toBe('append') - expect(loaded.events[1]!.sourceEventSeqs).toBeUndefined() + expect((loaded.events[1]! as SurfaceEvent).surfaceOp).toBe('append') + expect((loaded.events[1]! as SurfaceEvent).sourceEventSeqs).toBeUndefined() await fiber.dispose() }) }) diff --git a/packages/session/src/index.ts b/packages/session/src/index.ts index d0d4d44aed..e04bda5e7d 100644 --- a/packages/session/src/index.ts +++ b/packages/session/src/index.ts @@ -10,7 +10,7 @@ import { Context, Service } from 'cordis' import { isAbsolute } from 'node:path' import type { ContentBlock, Message, MessageSource } from '@deepseek-ai/dsh-llm' import { SessionId } from './types.ts' -import type { CreateSessionOptions, SessionEvent, SessionEventMap, SessionEventType, SessionHeader, SurfaceAppendOpts } from './types.ts' +import type { CreateSessionOptions, SessionEvent, SessionEventMap, SessionEventType, SessionHeader, SurfaceAppendOpts, SurfaceEventType } from './types.ts' import { isJsonValue } from './json.ts' import { SurfaceManager } from './surface.ts' @@ -18,6 +18,7 @@ export * from './types.ts' export { isJsonValue } from './json.ts' export { interruptedTurnClosers } from './repair.ts' export type { SurfaceNode } from './surface.ts' +export { isSurfaceEvent } from './surface.ts' declare module 'cordis' { interface Context { @@ -138,7 +139,9 @@ export class Session { * @param data - The event payload; must be JSON-serializable. * @param opts - Optional surface metadata: `surfaceOp` controls how the * event enters the surface linked list; `sourceEventSeqs` records - * provenance (the seq numbers of events this one derives from). + * provenance (the seq numbers of events this one derives from). Only + * accepted for {@link SurfaceEventType} events — the compiler rejects + * surface opts for non-surface types like `turn/start` or `assistant/chunk`. * @throws if `data` is not losslessly JSON-serializable (BigInt, function, * symbol, undefined, non-finite number, circular ref, or an exotic object * like Map/Set/Date). The event log is the durable source of truth, so this @@ -147,7 +150,11 @@ export class Session { * throw surfaces at the buggy caller's append site, not asynchronously in a * backend flush. */ - append(type: T, data: SessionEventMap[T], opts?: SurfaceAppendOpts): SessionEvent { + append( + type: T, + data: SessionEventMap[T], + ...opts: T extends SurfaceEventType ? [opts?: SurfaceAppendOpts] : [] + ): SessionEvent { if (!isJsonValue(data)) { throw new Error(`session event "${type}" carries non-JSON-serializable data`) } @@ -164,18 +171,25 @@ export class Session { // Surface metadata is snapshot separately: sourceEventSeqs (number[] — // primitives, so array spread is a complete copy) and surfaceOp (a string // primitive, or cloned if it's a replace object). + const surfaceOpts: SurfaceAppendOpts | undefined = opts[0] + // Build the event shape with conditional surface fields via spreading. + // The result is cast through `unknown` because the conditional spreads + // produce an intersection type that the assignability checker can't + // narrow to a specific discriminated-union member when T is generic. + // This is a safe internal boundary: data was validated above, and + // surface metadata was snapshot from primitive/clone-safe values. const event = { type, seq: this.log.length, time: Date.now(), data: structuredClone(data), - ...opts?.sourceEventSeqs !== undefined ? { sourceEventSeqs: [...opts.sourceEventSeqs] } : {}, - ...opts?.surfaceOp !== undefined ? { - surfaceOp: typeof opts.surfaceOp === 'string' ? opts.surfaceOp : structuredClone(opts.surfaceOp), + ...surfaceOpts?.sourceEventSeqs !== undefined ? { sourceEventSeqs: [...surfaceOpts.sourceEventSeqs] } : {}, + ...surfaceOpts?.surfaceOp !== undefined ? { + surfaceOp: typeof surfaceOpts.surfaceOp === 'string' ? surfaceOpts.surfaceOp : structuredClone(surfaceOpts.surfaceOp), } : {}, - } as SessionEvent - this.log.push(event) - this.onAppend?.(event) + } as unknown as SessionEvent + this.log.push(event as unknown as SessionEvent) + this.onAppend?.(event as unknown as SessionEvent) return event } @@ -207,6 +221,10 @@ export class Session { // index by construction. The non-null assertion expresses that invariant. // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const msg = this._deriveOneMessage(this.log[node.seq]!) + // isSurfaceEvent guarantees only the five surface-eligible types + // enter the surface, and all five produce messages → msg is never + // null. Defensive guard retained for interface contract clarity. + /* v8 ignore next */ if (msg) messages.push(msg) } return messages diff --git a/packages/session/src/surface.ts b/packages/session/src/surface.ts index a09dbd001f..eaa8baeb4a 100644 --- a/packages/session/src/surface.ts +++ b/packages/session/src/surface.ts @@ -7,7 +7,33 @@ * @module @deepseek-ai/dsh-session/surface */ -import type { SessionEvent, SurfaceOp } from './types.ts' +import type { SessionEvent, SurfaceEvent, SurfaceEventType, SurfaceOp } from './types.ts' + +/** + * The set of event type strings that are eligible for the surface linked list. + * Mirrors the {@link SurfaceEventType} union; kept as a runtime set so the + * type guard can check membership without a chain of string comparisons. + */ +const SURFACE_EVENT_TYPES = new Set([ + 'user/message', + 'assistant/message', + 'tool/result', + 'context/message', + 'steering/message', +]) + +/** + * Narrow a {@link SessionEvent} to {@link SurfaceEvent}: checks that the + * event's `type` is surface-eligible AND that `surfaceOp` is present. + * The narrowed type has mandatory {@link SurfaceOp}. + */ +export function isSurfaceEvent(event: SessionEvent): event is SurfaceEvent { + if (!SURFACE_EVENT_TYPES.has(event.type)) return false + // surfaceOp is optional on SessionEvent (even for surface-eligible types) + // but mandatory on SurfaceEvent — this check is the narrowing gate. + if ((event as SessionEvent).surfaceOp === undefined) return false + return true +} /** One node in the surface linked list. */ export interface SurfaceNode { @@ -57,11 +83,12 @@ export class SurfaceManager { get hasSurface(): boolean { if (this._nodes.length > 0) return true // Never processed anything — scan the whole log. - if (this._lastProcessedSeq === -1) return this.log.some(e => e.surfaceOp !== undefined) + if (this._lastProcessedSeq === -1) return this.log.some(e => isSurfaceEvent(e)) // Processed up to _lastProcessedSeq without finding surface nodes; check // only new events. for (let i = this._lastProcessedSeq + 1; i < this.log.length; i++) { - if (this.log[i]?.surfaceOp !== undefined) return true + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + if (isSurfaceEvent(this.log[i]!)) return true } return false } @@ -72,8 +99,13 @@ export class SurfaceManager { */ private _processDelta(): void { for (let i = this._lastProcessedSeq + 1; i < this.log.length; i++) { - const event = this.log[i] - if (event === undefined || event.surfaceOp === undefined) continue + // Index is bounded by i < this.log.length — never undefined. + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const event = this.log[i]! + // isSurfaceEvent checks event.type first (is it a surface-eligible type?) + // then checks that surfaceOp is present. Only after both pass do we treat + // it as a SurfaceEvent with mandatory surfaceOp. + if (!isSurfaceEvent(event)) continue if (event.surfaceOp === 'append') { const tail = this._nodes.length > 0 ? this._nodes[this._nodes.length - 1] : undefined diff --git a/packages/session/src/types.ts b/packages/session/src/types.ts index d7e9b19259..be3ed61af2 100644 --- a/packages/session/src/types.ts +++ b/packages/session/src/types.ts @@ -175,8 +175,31 @@ export interface SessionEventMap { export type SessionEventType = keyof SessionEventMap /** - * How a session event entered the surface linked list. Absent for non-surface - * events (boundaries, chunks, usage, errors). + * The subset of {@link SessionEventType} values whose events produce LLM + * messages and are eligible to appear on the surface linked list. Only these + * event types may carry {@link SurfaceOp} and {@link SessionEvent.sourceEventSeqs}. + */ +export type SurfaceEventType = + | 'user/message' + | 'assistant/message' + | 'tool/result' + | 'context/message' + | 'steering/message' + +/** + * A {@link SessionEvent} that is **on** the surface linked list — its + * `surfaceOp` is guaranteed present (mandatory), narrowed from a + * surface-eligible {@link SessionEvent} by checking both `type` and + * `surfaceOp` at runtime. + * + * Use the `isSurfaceEvent` type guard (in `surface.ts`) to narrow a + * `SessionEvent` to this type. + */ +export type SurfaceEvent = SessionEvent & { surfaceOp: SurfaceOp } + +/** + * How a session event entered the surface linked list. Only valid on + * {@link SurfaceEventType} events. * * - `'append'`: added to the tail — normal path for user/assistant/tool/context * messages. @@ -196,6 +219,9 @@ export type SurfaceOp = * `sourceEventSeqs` records the seq numbers of events that are provenance * sources of this one (e.g. the `assistant/chunk` seqs behind an * `assistant/message`, or the shadowed nodes behind a compaction replacement). + * + * Only accepted for {@link SurfaceEventType} events — non-surface event types + * (`turn/start`, `assistant/chunk`, `error`, …) cannot carry surface metadata. */ export interface SurfaceAppendOpts { surfaceOp?: SurfaceOp @@ -207,6 +233,13 @@ export interface SurfaceAppendOpts { * * A proper discriminated union over `type` (not independent `type`/`data` * unions), so `switch (event.type)` narrows `event.data` without casts. + * + * The {@link sourceEventSeqs} and {@link surfaceOp} fields are conditional: + * they only exist on {@link SurfaceEventType} variants (`user/message`, + * `assistant/message`, `tool/result`, `context/message`, `steering/message`). + * Non-surface events (boundary markers, chunks, usage, errors) never carry + * surface metadata — the compiler enforces this at `Session.append()` + * call sites. */ export type SessionEvent = { [K in SessionEventType]: { @@ -216,6 +249,7 @@ export type SessionEvent = { /** Unix epoch milliseconds. */ time: number data: SessionEventMap[K] + } & (K extends SurfaceEventType ? { /** * Seq numbers of events that are provenance sources of this event * (e.g. the `assistant/chunk` seqs that built an `assistant/message`, @@ -224,5 +258,5 @@ export type SessionEvent = { sourceEventSeqs?: number[] /** How this event entered the surface; absent for non-surface events. */ surfaceOp?: SurfaceOp - } + } : object) }[T] diff --git a/packages/session/tests/repair.spec.ts b/packages/session/tests/repair.spec.ts index cf6fb2b51c..0b7dee9f2b 100644 --- a/packages/session/tests/repair.spec.ts +++ b/packages/session/tests/repair.spec.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest' import { CallId } from '@deepseek-ai/dsh-llm' import { interruptedTurnClosers } from '../src/index.ts' -import type { SessionEvent } from '../src/index.ts' +import type { SessionEvent, SurfaceEvent } from '../src/index.ts' /** * Unit coverage for the crash-recovery closer synthesis. The persistence @@ -135,8 +135,8 @@ describe('interruptedTurnClosers', () => { const closers = interruptedTurnClosers(events) expect(closers.map(e => e.type)).toEqual(['tool/result', 'step/end', 'turn/end']) const result = closers[0]! - expect(result.surfaceOp).toBe('append') - expect(result.sourceEventSeqs).toEqual([3]) + expect((result as SurfaceEvent).surfaceOp).toBe('append') + expect((result as SurfaceEvent).sourceEventSeqs).toEqual([3]) }) it('handles tool/call without a matching assistant/message entry gracefully', () => { diff --git a/packages/session/tests/surface.spec.ts b/packages/session/tests/surface.spec.ts index d8f503c5e5..51ddcbcd44 100644 --- a/packages/session/tests/surface.spec.ts +++ b/packages/session/tests/surface.spec.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest' -import type { SessionEvent } from '@deepseek-ai/dsh-session' +import type { SessionEvent, SurfaceEvent, SurfaceEventType } from '@deepseek-ai/dsh-session' import { Session, SessionId } from '@deepseek-ai/dsh-session' import { CallId } from '@deepseek-ai/dsh-llm' @@ -188,7 +188,7 @@ describe('SurfaceManager', () => { // Mutate caller's array after append. sources.push(30) sources[0] = 99 - const logged = s.events[0]! + const logged = s.events[0]! as SurfaceEvent expect(logged.sourceEventSeqs).toEqual([10, 20]) }) @@ -219,7 +219,7 @@ describe('SurfaceManager', () => { s.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 's' }] }, { surfaceOp: op, sourceEventSeqs: [0] }) // Mutate caller's object after append. op.start = 99 - const logged = s.events[1]! + const logged = s.events[1]! as SurfaceEvent expect(logged.surfaceOp).toEqual({ op: 'replace', start: 0, end: 0 }) }) }) @@ -288,8 +288,8 @@ describe('Session.append surface opts', () => { expect(event.sourceEventSeqs).toEqual([3, 5, 7]) expect(event.surfaceOp).toBe('append') // The logged event matches the returned event. - expect(s.events[0]!.sourceEventSeqs).toEqual([3, 5, 7]) - expect(s.events[0]!.surfaceOp).toBe('append') + expect((s.events[0]! as SurfaceEvent).sourceEventSeqs).toEqual([3, 5, 7]) + expect((s.events[0]! as SurfaceEvent).surfaceOp).toBe('append') }) it('deriveMessages skips surface nodes whose event type is not message-producing', () => { @@ -299,7 +299,7 @@ describe('Session.append surface opts', () => { const seed: SessionEvent[] = [ { type: 'turn/start', seq: 0, time: 1, data: { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } } }, { type: 'step/start', seq: 1, time: 2, data: { turn: 1, step: 1 } }, - { type: 'usage', seq: 2, time: 3, data: { turn: 1, step: 1, usage: { inputTokens: 0, outputTokens: 0 } }, surfaceOp: 'append' as const }, + { type: 'usage', seq: 2, time: 3, data: { turn: 1, step: 1, usage: { inputTokens: 0, outputTokens: 0 } }, surfaceOp: 'append' as const } as SessionEvent, { type: 'step/end', seq: 3, time: 4, data: { turn: 1, step: 1 } }, { type: 'turn/end', seq: 4, time: 5, data: { turn: 1, reason: { kind: 'completed' } } }, ] @@ -311,8 +311,8 @@ describe('Session.append surface opts', () => { it('append without surface opts produces an event without surface fields', () => { const s = new Session(SessionId('noopts')) s.append('user/message', { content: [{ type: 'text', text: 'x' }], source: { kind: 'user' } }) - expect(s.events[0]!.sourceEventSeqs).toBeUndefined() - expect(s.events[0]!.surfaceOp).toBeUndefined() + expect((s.events[0] as SessionEvent).sourceEventSeqs).toBeUndefined() + expect((s.events[0] as SessionEvent).surfaceOp).toBeUndefined() }) it('surfaceOp primitives are not cloned (they are immutable)', () => { From 30805e1983d76d0846fcbdb49d0c2b06edb1a7b9 Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Tue, 23 Jun 2026 09:48:15 +0800 Subject: [PATCH 4/9] fix(sqlite): bump SCHEMA_VERSION to 3 for the new surface columns --- .../session-persistence-sqlite/src/schema.ts | 2 +- .../session-persistence-sqlite/tests/sqlite.spec.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/session-persistence/session-persistence-sqlite/src/schema.ts b/packages/session-persistence/session-persistence-sqlite/src/schema.ts index 6012916457..b747b7900c 100644 --- a/packages/session-persistence/session-persistence-sqlite/src/schema.ts +++ b/packages/session-persistence/session-persistence-sqlite/src/schema.ts @@ -15,7 +15,7 @@ import type { SessionEvent, SessionId, SessionHeader, SurfaceOp } from '@deepsee * layout; orthogonal to a session's own `version` (which versions the EVENT * vocabulary, stored per session in the `sessions` row). */ -export const SCHEMA_VERSION = 2 +export const SCHEMA_VERSION = 3 /** * A row of the `sessions` table — the out-of-log metadata ({@link SessionHeader}). diff --git a/packages/session-persistence/session-persistence-sqlite/tests/sqlite.spec.ts b/packages/session-persistence/session-persistence-sqlite/tests/sqlite.spec.ts index 86d72275f6..0b8b83cd4e 100644 --- a/packages/session-persistence/session-persistence-sqlite/tests/sqlite.spec.ts +++ b/packages/session-persistence/session-persistence-sqlite/tests/sqlite.spec.ts @@ -315,7 +315,7 @@ describe('SessionPersistenceSqlite: durability and crash semantics', () => { }) it('exposes the schema version constant', () => { - expect(SCHEMA_VERSION).toBe(2) + expect(SCHEMA_VERSION).toBe(3) }) }) From 09d497d5c5b2e7d321cded16110dd34bae2fff96 Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Tue, 23 Jun 2026 09:54:58 +0800 Subject: [PATCH 5/9] docs(rfc): clarify compaction rides the replace op on an existing event type --- docs/rfc/implemented/architecture/2026-06-18-session-surface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/rfc/implemented/architecture/2026-06-18-session-surface.md b/docs/rfc/implemented/architecture/2026-06-18-session-surface.md index 42c3872db6..644d526373 100644 --- a/docs/rfc/implemented/architecture/2026-06-18-session-surface.md +++ b/docs/rfc/implemented/architecture/2026-06-18-session-surface.md @@ -60,4 +60,4 @@ The dev-mode invariants plugin validates: `sourceEventSeqs` references (non-empt - **`packages/session-persistence/session-persistence-jsonl`**: No changes required. - **`packages/session-persistence/session-persistence`**: Abstract interface unchanged. -The surface is the foundation for future compaction: a compaction plugin appends a new event (e.g., `compaction/marker`, added to `SessionEventMap` via declaration merging) with `surfaceOp: { op: 'replace', start, end }` and `sourceEventSeqs` covering the shadowed nodes. Replay preserves the compaction decision deterministically. +The surface is the foundation for future history manipulation. A compaction or tool-result-prune plugin appends one of the existing message-producing event types (a `user/message` carrying the summary, say) with `surfaceOp: { op: 'replace', start, end }` and `sourceEventSeqs` covering the shadowed nodes — the new node takes the range's place on the surface while the plugin's own trace events (e.g. `compaction/start`, `compaction/end`) stay off it. Replay preserves the decision deterministically. From 6089e226bc2423229eea942ae7e253cc847799f8 Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Tue, 23 Jun 2026 13:05:59 +0800 Subject: [PATCH 6/9] refactor(session): make surface the sole derivation path, drop legacy fallback --- docs/cordis-catalog/events-and-services.md | 2 +- .../2026-06-18-session-surface.md | 2 +- packages/core/session/README.md | 6 +- packages/core/session/src/index.ts | 59 +++++++++---------- packages/core/session/src/surface.ts | 14 ----- packages/core/session/src/types.ts | 12 ++-- packages/core/session/tests/session.spec.ts | 32 +++++----- packages/core/session/tests/surface.spec.ts | 47 +-------------- .../tests/jsonl.spec.ts | 8 +-- .../tests/coordinator-contract.ts | 14 ++--- .../invariants/tests/invariants.spec.ts | 36 +++++------ 11 files changed, 87 insertions(+), 145 deletions(-) diff --git a/docs/cordis-catalog/events-and-services.md b/docs/cordis-catalog/events-and-services.md index 1ee86d6a5e..f8c6b12c0d 100644 --- a/docs/cordis-catalog/events-and-services.md +++ b/docs/cordis-catalog/events-and-services.md @@ -390,7 +390,7 @@ get(id: SessionId): Session | undefined list(): Session[] ``` -Source: [`packages/core/session/src/index.ts:303`](../../packages/core/session/src/index.ts) +Source: [`packages/core/session/src/index.ts:300`](../../packages/core/session/src/index.ts) ### `ctx.systemPrompt` — `SystemPrompt` diff --git a/docs/rfc/implemented/architecture/2026-06-18-session-surface.md b/docs/rfc/implemented/architecture/2026-06-18-session-surface.md index 644d526373..ae2475c786 100644 --- a/docs/rfc/implemented/architecture/2026-06-18-session-surface.md +++ b/docs/rfc/implemented/architecture/2026-06-18-session-surface.md @@ -53,7 +53,7 @@ The dev-mode invariants plugin validates: `sourceEventSeqs` references (non-empt ## Consequences -- **`packages/core/session`**: New `surface.ts` (`SurfaceManager`), new types (`SurfaceOp`, `SurfaceAppendOpts`), new fields on `SessionEvent`, modified `append()` (third optional `SurfaceAppendOpts` param), refactored `deriveMessages()` (surface path + legacy fallback), surface-aware `repair.ts`. +- **`packages/core/session`**: New `surface.ts` (`SurfaceManager`), new types (`SurfaceOp`, `SurfaceIntent`), new fields on `SessionEvent`, modified `append()` (third required `SurfaceIntent` param), refactored `deriveMessages()` (surface path + legacy fallback), surface-aware `repair.ts`. - **`packages/core/agent-loop`**: All surface-capable appends pass surface opts. Chunk seqs are collected for `assistant/message` provenance; `tool/call` seqs are captured for `tool/result` provenance. - **`packages/session-persistence/session-persistence-sqlite`**: Two new nullable TEXT columns (`source_event_seqs`, `surface_op`) on the `events` table; `SCHEMA_VERSION` bumped (bump-and-reject, no migration). - **`packages/support/invariants`**: Surface-related validation rules. diff --git a/packages/core/session/README.md b/packages/core/session/README.md index 206022e306..08323eff0c 100644 --- a/packages/core/session/README.md +++ b/packages/core/session/README.md @@ -34,8 +34,8 @@ Creates and holds event-sourced `Session` instances. Persistence is intentionall Plain class (not a Cordis Service). Create via `ctx.sessions.create()`. -- `session.append(type, data, opts?): SessionEvent` — synchronous, never blocks on I/O. **Throws** if `data` is not losslessly JSON-serializable (BigInt, function, symbol, undefined, non-finite number, circular ref, or an exotic object like Map/Set/Date) — the event log is the durable source of truth, so this invariant is enforced at the source (exported as `isJsonValue` for backends to reuse on their replay/fork entry points). An optional third parameter `opts: SurfaceAppendOpts` carries surface metadata: `surfaceOp` controls how the event enters the surface linked list, and `sourceEventSeqs` records provenance (the seq numbers of events this one derives from). -- `session.deriveMessages(): Message[]` — derive the LLM message history. If any event in the log carries `surfaceOp`, derivation walks the surface linked list (skipping non-surface events). Otherwise, falls back to a linear scan of the raw log (legacy sessions without surface markers). +- `session.append(type, data, opts?): SessionEvent` — synchronous, never blocks on I/O. **Throws** if `data` is not losslessly JSON-serializable (BigInt, function, symbol, undefined, non-finite number, circular ref, or an exotic object like Map/Set/Date) — the event log is the durable source of truth, so this invariant is enforced at the source (exported as `isJsonValue` for backends to reuse on their replay/fork entry points). A third parameter `opts: SurfaceIntent` carries surface metadata: `surfaceOp` controls how the event enters the surface linked list, and `sourceEventSeqs` records provenance (the seq numbers of events this one derives from). It is **required** for the five `SurfaceEventType` events (every message-producing event must declare how it joins the surface) and rejected by the compiler for non-surface types. +- `session.deriveMessages(): Message[]` — derive the LLM message history by walking the surface linked list (skipping non-surface events like chunks and boundaries; a `replace` shadows the nodes it covers). The surface is the single source of derived history — there is no raw-log fallback. - `session.surface: SurfaceManager` — the derived surface, lazily rebuilt from `surfaceOp` markers in the log. Processes only new events (delta) on each access — the log is append-only, so prior events never change. - `session.events`, `session.seq`, `session.id` - `session.header: SessionHeader` — immutable creation metadata (`version`, `id`, `createdAt`, optional `cwd`/`parentSession`). Kept out of the event log (a storage concern, not replayable state); a minimal header (stamped with the current `SESSION_FORMAT_VERSION`) is synthesized for bare `Session` construction. @@ -43,7 +43,7 @@ Plain class (not a Cordis Service). Create via `ctx.sessions.create()`. ### Surface types - `SurfaceOp` — how a surface node entered the linked list: `'append'` (normal tail append) or `{ op: 'replace', start, end }` (replace nodes from `start` through `end` inclusive — both must be valid surface node seqs; `start === end` replaces a single node). Used by compaction to shadow old nodes without deleting them. -- `SurfaceAppendOpts` — `{ surfaceOp?: SurfaceOp; sourceEventSeqs?: number[] }`, the optional third parameter to `session.append()`. +- `SurfaceIntent` — `{ surfaceOp: SurfaceOp; sourceEventSeqs?: number[] }`, the required third parameter to `session.append()` for surface-eligible types. - `SurfaceNode` — `{ seq: number; prev: number | null; next: number | null }`, one node in the surface linked list. ### Session event vocabulary (`types.ts`) diff --git a/packages/core/session/src/index.ts b/packages/core/session/src/index.ts index 23e65cebda..5df4db58e9 100644 --- a/packages/core/session/src/index.ts +++ b/packages/core/session/src/index.ts @@ -10,7 +10,7 @@ import { Context, Service } from 'cordis' import { isAbsolute } from 'node:path' import type { ContentBlock, Message, MessageSource } from '@deepseek-ai/dsh-llm' import { SESSION_FORMAT_VERSION, SessionId } from './types.ts' -import type { CreateSessionOptions, SessionEvent, SessionEventMap, SessionEventType, SessionHeader, SurfaceAppendOpts, SurfaceEventType } from './types.ts' +import type { CreateSessionOptions, SessionEvent, SessionEventMap, SessionEventType, SessionHeader, SurfaceIntent, SurfaceEventType } from './types.ts' import { isJsonValue } from './json.ts' import { SurfaceManager } from './surface.ts' @@ -149,11 +149,13 @@ export class Session { * * @param type - The event type (key of {@link SessionEventMap}). * @param data - The event payload; must be JSON-serializable. - * @param opts - Optional surface metadata: `surfaceOp` controls how the - * event enters the surface linked list; `sourceEventSeqs` records - * provenance (the seq numbers of events this one derives from). Only - * accepted for {@link SurfaceEventType} events — the compiler rejects - * surface opts for non-surface types like `turn/start` or `assistant/chunk`. + * @param opts - Surface metadata: `surfaceOp` controls how the event enters + * the surface linked list; `sourceEventSeqs` records provenance (the seq + * numbers of events this one derives from). REQUIRED for + * {@link SurfaceEventType} events (every message-producing event must + * declare how it joins the surface, the sole source of derived history) and + * rejected by the compiler for non-surface types like `turn/start` or + * `assistant/chunk`. * @throws if `data` is not losslessly JSON-serializable (BigInt, function, * symbol, undefined, non-finite number, circular ref, or an exotic object * like Map/Set/Date). The event log is the durable source of truth, so this @@ -165,7 +167,7 @@ export class Session { append( type: T, data: SessionEventMap[T], - ...opts: T extends SurfaceEventType ? [opts?: SurfaceAppendOpts] : [] + ...opts: T extends SurfaceEventType ? [opts: SurfaceIntent] : [] ): SessionEvent { if (!isJsonValue(data)) { throw new Error(`session event "${type}" carries non-JSON-serializable data`) @@ -183,7 +185,7 @@ export class Session { // Surface metadata is snapshot separately: sourceEventSeqs (number[] — // primitives, so array spread is a complete copy) and surfaceOp (a string // primitive, or cloned if it's a replace object). - const surfaceOpts: SurfaceAppendOpts | undefined = opts[0] + const surfaceOpts: SurfaceIntent | undefined = opts[0] // Build the event shape with conditional surface fields via spreading. // The result is cast through `unknown` because the conditional spreads // produce an intersection type that the assignability checker can't @@ -206,9 +208,12 @@ export class Session { } /** - * Derive the LLM message history from the session surface (when surface - * markers exist) or from a linear scan of the raw event log (legacy sessions - * without surface markers). + * Derive the LLM message history by walking the session surface — the linked + * list of message-producing events maintained by `surfaceOp` markers. The + * surface is the single source of derived history: every message-producing + * append records its `surfaceOp`, so a raw event with no marker (a chunk, a + * turn boundary) is correctly absent, and a compaction `replace` deletes the + * shadowed nodes from the derivation. * * - `user/message` → user message * - `assistant/message` → assistant message (chunks are skipped — they are @@ -229,33 +234,24 @@ export class Session { * negligible next to a model call. */ deriveMessages(): Message[] { - if (this.surface.hasSurface) { - const messages: Message[] = [] - for (const node of this.surface.nodes) { - // Surface nodes are built from this.log — node.seq is always a valid - // index by construction. The non-null assertion expresses that invariant. - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const msg = this._deriveOneMessage(this.log[node.seq]!) - // A surface node is one of the five message-producing types, but an - // empty-content assistant/message (a max-tokens step that hosts only - // usage) derives to null and must not enter the transcript. - if (msg) messages.push(msg) - } - return messages - } - // Legacy path: linear scan for sessions without surface markers. const messages: Message[] = [] - for (const event of this.log) { - const msg = this._deriveOneMessage(event) + for (const node of this.surface.nodes) { + // Surface nodes are built from this.log — node.seq is always a valid + // index by construction. The non-null assertion expresses that invariant. + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const msg = this._deriveOneMessage(this.log[node.seq]!) + // A surface node is one of the five message-producing types, but an + // empty-content assistant/message (a max-tokens step that hosts only + // usage) derives to null and must not enter the transcript. if (msg) messages.push(msg) } return messages } /** - * Derive a single LLM message from one event, or null if the event type - * does not produce a message. Extracted so both the surface path and the - * legacy linear-scan path share the same derivation rules. + * Derive a single LLM message from one surface event, or null if it produces + * no message (an empty-content assistant/message that exists only to host + * usage). */ private _deriveOneMessage(event: SessionEvent): Message | null { // Intentionally non-exhaustive: only message-producing events derive @@ -288,6 +284,7 @@ export class Session { const { content, source } = event.data return { role: 'user', content: renderTagged('steering', structuredClone(content), source) } } + /* v8 ignore next 2 -- unreachable: only surface nodes (the 5 message-producing types) reach here */ default: return null } diff --git a/packages/core/session/src/surface.ts b/packages/core/session/src/surface.ts index eaa8baeb4a..7ed1743af3 100644 --- a/packages/core/session/src/surface.ts +++ b/packages/core/session/src/surface.ts @@ -79,20 +79,6 @@ export class SurfaceManager { return this._nodes } - /** Whether any event in the log carries `surfaceOp` markers. */ - get hasSurface(): boolean { - if (this._nodes.length > 0) return true - // Never processed anything — scan the whole log. - if (this._lastProcessedSeq === -1) return this.log.some(e => isSurfaceEvent(e)) - // Processed up to _lastProcessedSeq without finding surface nodes; check - // only new events. - for (let i = this._lastProcessedSeq + 1; i < this.log.length; i++) { - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - if (isSurfaceEvent(this.log[i]!)) return true - } - return false - } - /** * Process events from `_lastProcessedSeq + 1` through the end of the log, * folding new surface markers into the existing linked list. diff --git a/packages/core/session/src/types.ts b/packages/core/session/src/types.ts index 1fcd150d37..44e8e35fdf 100644 --- a/packages/core/session/src/types.ts +++ b/packages/core/session/src/types.ts @@ -222,17 +222,19 @@ export type SurfaceOp = | { op: 'replace'; start: number; end: number } /** - * Optional surface metadata passed to {@link Session.append}. + * Surface metadata passed to {@link Session.append}. * `surfaceOp` controls how the event enters the surface linked list; * `sourceEventSeqs` records the seq numbers of events that are provenance * sources of this one (e.g. the `assistant/chunk` seqs behind an * `assistant/message`, or the shadowed nodes behind a compaction replacement). * - * Only accepted for {@link SurfaceEventType} events — non-surface event types - * (`turn/start`, `assistant/chunk`, `error`, …) cannot carry surface metadata. + * Required for {@link SurfaceEventType} events — every message-producing event + * MUST declare how it enters the surface, because the surface is the sole + * source of derived history. Non-surface event types (`turn/start`, + * `assistant/chunk`, `error`, …) cannot carry surface metadata. */ -export interface SurfaceAppendOpts { - surfaceOp?: SurfaceOp +export interface SurfaceIntent { + surfaceOp: SurfaceOp sourceEventSeqs?: number[] } diff --git a/packages/core/session/tests/session.spec.ts b/packages/core/session/tests/session.spec.ts index f095bdfb40..46f96742cc 100644 --- a/packages/core/session/tests/session.spec.ts +++ b/packages/core/session/tests/session.spec.ts @@ -7,7 +7,7 @@ describe('Session', () => { it('derives message history from the event log', () => { const session = new Session(SessionId('s1')) session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) - session.append('user/message', { content: [{ type: 'text', text: 'hello' }], source: { kind: 'user' } }) + session.append('user/message', { content: [{ type: 'text', text: 'hello' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) session.append('assistant/chunk', { turn: 1, step: 1, chunk: { type: 'text-delta', index: 0, text: 'hi' } }) session.append('assistant/message', { turn: 1, step: 1, @@ -15,8 +15,8 @@ describe('Session', () => { { type: 'text', text: 'let me check' }, { type: 'tool-call', id: CallId('c1'), name: 'echo', arguments: '{}' }, ], - }) - session.append('tool/result', { turn: 1, step: 1, callId: CallId('c1'), content: [{ type: 'text', text: 'ok' }], isError: false }) + }, { surfaceOp: 'append' }) + session.append('tool/result', { turn: 1, step: 1, callId: CallId('c1'), content: [{ type: 'text', text: 'ok' }], isError: false }, { surfaceOp: 'append' }) session.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) const messages = session.deriveMessages() @@ -44,12 +44,12 @@ describe('Session', () => { session.append('context/message', { content: [{ type: 'text', text: 'file changed: a.ts' }], source: { kind: 'plugin', plugin: 'watcher' }, - }) + }, { surfaceOp: 'append' }) session.append('steering/message', { turn: 1, content: [{ type: 'text', text: 'focus on tests' }], source: { kind: 'user' }, - }) + }, { surfaceOp: 'append' }) const [contextMessage, steeringMessage] = session.deriveMessages() expect(contextMessage!.role).toBe('user') @@ -60,8 +60,8 @@ describe('Session', () => { it('replays identically from a seeded event log', () => { const original = new Session(SessionId('s3')) - original.append('user/message', { content: [{ type: 'text', text: 'q' }], source: { kind: 'user' } }) - original.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 'a' }] }) + original.append('user/message', { content: [{ type: 'text', text: 'q' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) + original.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 'a' }] }, { surfaceOp: 'append' }) const replayed = new Session(SessionId('s3-replay'), [...original.events]) expect(replayed.deriveMessages()).toEqual(original.deriveMessages()) @@ -70,11 +70,11 @@ describe('Session', () => { it('isolates the log from mutation through a derived message (append-only contract)', () => { const session = new Session(SessionId('s4')) - session.append('user/message', { content: [{ type: 'text', text: 'original' }], source: { kind: 'user' } }) + session.append('user/message', { content: [{ type: 'text', text: 'original' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) session.append('tool/result', { turn: 1, step: 1, callId: CallId('c1'), content: [{ type: 'text', text: 'tool out' }], isError: false, - }) + }, { surfaceOp: 'append' }) const before = structuredClone(session.events) // A request middleware / adapter mutates the messages it was handed. @@ -95,7 +95,7 @@ describe('Session', () => { it('rejects non-JSON-serializable event data at the source (incl. sparse arrays)', () => { const session = new Session(SessionId('s5')) - const bad = (extra: unknown) => () => session.append('user/message', { content: [{ type: 'text', text: 'x' }], source: { kind: 'user' }, extra } as never) + const bad = (extra: unknown) => () => session.append('user/message', { content: [{ type: 'text', text: 'x' }], source: { kind: 'user' }, extra } as never, { surfaceOp: 'append' }) expect(bad(1n)).toThrow(/non-JSON-serializable/) expect(bad(() => 0)).toThrow(/non-JSON-serializable/) expect(bad(Symbol('s'))).toThrow(/non-JSON-serializable/) @@ -122,7 +122,7 @@ describe('Session', () => { it('accepts dense arrays and nested plain objects', () => { const session = new Session(SessionId('s6')) - expect(() => session.append('user/message', { content: [{ type: 'text', text: 'x' }], source: { kind: 'user' }, extra: [1, 2, [3, { a: null, b: true }]] } as never)).not.toThrow() + expect(() => session.append('user/message', { content: [{ type: 'text', text: 'x' }], source: { kind: 'user' }, extra: [1, 2, [3, { a: null, b: true }]] } as never, { surfaceOp: 'append' })).not.toThrow() expect(session.events).toHaveLength(1) }) @@ -174,7 +174,7 @@ describe('Session', () => { it('snapshots append data: mutating the passed object after append does not affect session.events', () => { const session = new Session(SessionId('append-snapshot')) const data = { content: [{ type: 'text' as const, text: 'original' }], source: { kind: 'user' as const } } - const event = session.append('user/message', data) + const event = session.append('user/message', data, { surfaceOp: 'append' }) // Mutate the caller's object after append returns. A shared reference would // make session.events diverge from the value that passed validation. data.content[0]!.text = 'HACKED' @@ -201,7 +201,7 @@ describe('SessionStore', () => { const session = ctx.sessions.create() expect(created).toEqual([session]) - session.append('user/message', { content: [{ type: 'text', text: 'x' }], source: { kind: 'user' } }) + session.append('user/message', { content: [{ type: 'text', text: 'x' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) expect(events).toHaveLength(1) expect(events[0]![0]).toBe(session) expect(events[0]![1].type).toBe('user/message') @@ -216,7 +216,7 @@ describe('SessionStore', () => { const a = ctx.sessions.create(SessionId('fixed')) expect(() => ctx.sessions.create(SessionId('fixed'))).toThrow('already exists') - a.append('user/message', { content: [{ type: 'text', text: 'q' }], source: { kind: 'user' } }) + a.append('user/message', { content: [{ type: 'text', text: 'q' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) const forked = ctx.sessions.create(SessionId('fork'), { seed: [...a.events] }) expect(forked.deriveMessages()).toEqual(a.deriveMessages()) }) @@ -309,7 +309,7 @@ describe('SessionStore', () => { await fiber.dispose() expect(ctx.sessions.get(SessionId('scoped'))).toBeUndefined() - session.append('user/message', { content: [{ type: 'text', text: 'late' }], source: { kind: 'user' } }) + session.append('user/message', { content: [{ type: 'text', text: 'late' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) expect(observed).toBe(0) }) @@ -332,7 +332,7 @@ describe('SessionStore', () => { ctx.on('session/event', (_session, event) => void events.push(event)) const session = ctx.sessions.create(SessionId('fixed')) expect(ctx.sessions.get(SessionId('fixed'))).toBe(session) - session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }) + session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) expect(events).toHaveLength(1) }) }) diff --git a/packages/core/session/tests/surface.spec.ts b/packages/core/session/tests/surface.spec.ts index b4e1022494..a7f4c13dad 100644 --- a/packages/core/session/tests/surface.spec.ts +++ b/packages/core/session/tests/surface.spec.ts @@ -28,38 +28,6 @@ describe('SurfaceManager', () => { expect(nodes[1]!.next).toBeNull() }) - it('hasSurface returns false when no events have surfaceOp', () => { - const s = new Session(SessionId('nosurface')) - s.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) - s.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }) - s.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) - expect(s.surface.hasSurface).toBe(false) - }) - - it('hasSurface returns true when any event has surfaceOp', () => { - const s = surfaceSession() - expect(s.surface.hasSurface).toBe(true) - }) - - it('hasSurface detects surface markers that arrive after initial processing', () => { - // Start with no surface markers. Access nodes first to set _lastProcessedSeq - // (via delta processing), keeping _nodes empty. Then append a mix of non-surface - // and surface events, and verify hasSurface detects via the delta-only check. - const s = new Session(SessionId('late')) - s.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) - s.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }) - // Access nodes to trigger processing: sets _lastProcessedSeq = 1, _nodes = []. - expect(s.surface.nodes.length).toBe(0) - // Append non-surface events first (exercises the loop-continue branch), then - // a surface event (exercises the return-true branch). - s.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) - s.append('turn/start', { turn: 2, trigger: { kind: 'continuation' } }) - s.append('assistant/message', { turn: 2, step: 1, content: [] }, { surfaceOp: 'append' }) - // hasSurface checks only new seqs [2, 3, 4]; skips 2 and 3 (non-surface), - // finds surfaceOp on seq 4 and returns true. - expect(s.surface.hasSurface).toBe(true) - }) - it('invalidate resets to full rebuild', () => { const s = surfaceSession() expect(s.surface.nodes.length).toBe(2) @@ -76,7 +44,6 @@ describe('SurfaceManager', () => { s.append('step/end', { turn: 1, step: 1 }) s.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) expect(s.surface.nodes.length).toBe(0) - expect(s.surface.hasSurface).toBe(false) // deriveMessages returns empty array expect(s.deriveMessages()).toEqual([]) }) @@ -235,16 +202,6 @@ describe('deriveMessages with surface', () => { expect(messages[1]!.content[0]).toMatchObject({ type: 'text', text: 'hi' }) }) - it('falls back to linear scan when no surface markers exist', () => { - const s = new Session(SessionId('legacy')) - s.append('user/message', { content: [{ type: 'text', text: 'q' }], source: { kind: 'user' } }) - s.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 'a' }] }) - const messages = s.deriveMessages() - expect(messages).toHaveLength(2) - expect(messages[0]!.role).toBe('user') - expect(messages[1]!.role).toBe('assistant') - }) - it('surface path skips non-surface events (chunks, boundaries)', () => { const s = new Session(SessionId('filter')) s.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) @@ -308,9 +265,9 @@ describe('Session.append surface opts', () => { expect(s.deriveMessages()).toHaveLength(0) }) - it('append without surface opts produces an event without surface fields', () => { + it('a non-surface event carries no surface fields', () => { const s = new Session(SessionId('noopts')) - s.append('user/message', { content: [{ type: 'text', text: 'x' }], source: { kind: 'user' } }) + s.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) expect((s.events[0] as SessionEvent).sourceEventSeqs).toBeUndefined() expect((s.events[0] as SessionEvent).surfaceOp).toBeUndefined() }) diff --git a/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts b/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts index 1df70f9c9a..7df0fd5180 100644 --- a/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts +++ b/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts @@ -276,8 +276,8 @@ describe('SessionPersistenceJsonl: write path (session/event → flush)', () => const a = ctx.sessions.create(SessionId('sa')) const b = ctx.sessions.create(SessionId('sb')) - a.append('user/message', { content: [{ type: 'text', text: 'A' }], source: { kind: 'user' } }) - b.append('user/message', { content: [{ type: 'text', text: 'B' }], source: { kind: 'user' } }) + a.append('user/message', { content: [{ type: 'text', text: 'A' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) + b.append('user/message', { content: [{ type: 'text', text: 'B' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) a.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) b.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) await ctx.parallel('session/flush', a) @@ -647,7 +647,7 @@ describe('SessionPersistenceJsonl: edge cases', () => { await ctx2.plugin(SessionPersistenceJsonl, { root }) const session = ctx2.sessions.create(SessionId('flush-fail')) // A full turn lands in the write-behind buffer. - session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }) + session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) session.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) // Make the durable materialize fail on the next flush. const backend = ctx2.sessionPersistence as unknown as { materialize: (...args: unknown[]) => Promise } @@ -695,7 +695,7 @@ describe('SessionPersistenceJsonl: edge cases', () => { // can never diverge from the live log. The throw surfaces at the caller's // append site, not asynchronously in a backend flush. expect(() => { - session.append('user/message', { content: [{ type: 'text', text: 'bad' }], source: { kind: 'user' }, bad: 1n } as never) + session.append('user/message', { content: [{ type: 'text', text: 'bad' }], source: { kind: 'user' }, bad: 1n } as never, { surfaceOp: 'append' }) }).toThrow(/non-JSON-serializable/) // The bad event was rejected, so the log stayed empty. expect(session.events.length).toBe(0) diff --git a/packages/session-persistence/session-persistence/tests/coordinator-contract.ts b/packages/session-persistence/session-persistence/tests/coordinator-contract.ts index 431d02b4cb..71cd296fb0 100644 --- a/packages/session-persistence/session-persistence/tests/coordinator-contract.ts +++ b/packages/session-persistence/session-persistence/tests/coordinator-contract.ts @@ -128,7 +128,7 @@ export function runCoordinatorContract(name: string, makeFixture: () => Promise< const { ctx, fiber } = await freshCtx(fix) try { const session = ctx.sessions.create(SessionId('mutate'), { meta: { cwd: WORK } }) - const ev = session.append('user/message', { content: [{ type: 'text', text: 'original' }], source: { kind: 'user' } }) + const ev = session.append('user/message', { content: [{ type: 'text', text: 'original' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // Mutate the live event object AFTER it was buffered by session/event. ;(ev.data as { content: { type: 'text'; text: string }[] }).content[0]!.text = 'HACKED' session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) @@ -232,7 +232,7 @@ export function runCoordinatorContract(name: string, makeFixture: () => Promise< await ctx.plugin(SessionStore) // A session exists BEFORE the persistence plugin is applied. const session = ctx.sessions.create(SessionId('pre-existing'), { meta: { cwd: WORK } }) - session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }) + session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) session.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) const fiber = await fix.mount(ctx) @@ -253,7 +253,7 @@ export function runCoordinatorContract(name: string, makeFixture: () => Promise< await ctx.plugin(SessionStore) const fiber = await fix.mount(ctx) const session = await liveSessionInFiber(ctx, 'drain', WORK) - session.append('user/message', { content: [{ type: 'text', text: 'buffered' }], source: { kind: 'user' } }) + session.append('user/message', { content: [{ type: 'text', text: 'buffered' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) session.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) // No explicit flush — dispose must drain. await fiber.dispose() @@ -279,7 +279,7 @@ export function runCoordinatorContract(name: string, makeFixture: () => Promise< // Backend instance 1 materializes the session. const backend1 = await fix.mount(ctx) session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) - session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }) + session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) session.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) await ctx.parallel('session/flush', session) @@ -290,7 +290,7 @@ export function runCoordinatorContract(name: string, makeFixture: () => Promise< await backend1.dispose() await fix.mount(ctx) session.append('turn/start', { turn: 2, trigger: { kind: 'message', source: { kind: 'user' } } }) - session.append('user/message', { content: [{ type: 'text', text: 'again' }], source: { kind: 'user' } }) + session.append('user/message', { content: [{ type: 'text', text: 'again' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) session.append('turn/end', { turn: 2, reason: { kind: 'completed' } }) await expect(ctx.parallel('session/flush', session)).resolves.not.toThrow() @@ -452,7 +452,7 @@ export function runCoordinatorContract(name: string, makeFixture: () => Promise< const { ctx, fiber } = await freshCtx(fix) try { const session = ctx.sessions.create(SessionId('idem'), { meta: { cwd: WORK } }) - session.append('user/message', { content: [{ type: 'text', text: 'x' }], source: { kind: 'user' } }) + session.append('user/message', { content: [{ type: 'text', text: 'x' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) session.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) await ctx.parallel('session/flush', session) // Re-emit session/created for the SAME live session (idempotent initFor). @@ -704,7 +704,7 @@ export function runCoordinatorContract(name: string, makeFixture: () => Promise< // async onCreated init has necessarily set state (exercises the // state-undefined cursor path). const session = ctx.sessions.create(SessionId('flush-nostate'), { meta: { cwd: WORK } }) - session.append('user/message', { content: [{ type: 'text', text: 'q' }], source: { kind: 'user' } }) + session.append('user/message', { content: [{ type: 'text', text: 'q' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) session.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) await ctx.parallel('session/flush', session) const loaded = await ctx.sessionPersistence.load(SessionId('flush-nostate')) diff --git a/packages/support/invariants/tests/invariants.spec.ts b/packages/support/invariants/tests/invariants.spec.ts index f66b6a8929..2eaae73169 100644 --- a/packages/support/invariants/tests/invariants.spec.ts +++ b/packages/support/invariants/tests/invariants.spec.ts @@ -25,12 +25,12 @@ describe('session-log invariants', () => { const session = ctx.sessions.create() expect(() => { session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) - session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }) + session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) session.append('step/start', { turn: 1, step: 1 }) session.append('assistant/chunk', { turn: 1, step: 1, chunk: { type: 'text-delta', index: 0, text: 'h' } }) - session.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'tool-call', id: CallId('c1'), name: 'echo', arguments: '{}' }] }) + session.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'tool-call', id: CallId('c1'), name: 'echo', arguments: '{}' }] }, { surfaceOp: 'append' }) session.append('tool/call', { turn: 1, step: 1, callId: CallId('c1'), name: 'echo', arguments: '{}' }) - session.append('tool/result', { turn: 1, step: 1, callId: CallId('c1'), content: [{ type: 'text', text: 'ok' }], isError: false }) + session.append('tool/result', { turn: 1, step: 1, callId: CallId('c1'), content: [{ type: 'text', text: 'ok' }], isError: false }, { surfaceOp: 'append' }) session.append('step/end', { turn: 1, step: 1 }) session.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) }).not.toThrow() @@ -89,9 +89,9 @@ describe('session-log invariants', () => { const { ctx } = await setup({ freeze: false }) const session = ctx.sessions.create() // No turn open: every message-bearing event must be turn-enclosed (the turn-enclosure RFC). - expect(() => session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } })) + expect(() => session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' })) .toThrow(/outside any open turn/) - expect(() => session.append('context/message', { content: [{ type: 'text', text: 'ctx' }], source: { kind: 'user' } })) + expect(() => session.append('context/message', { content: [{ type: 'text', text: 'ctx' }], source: { kind: 'user' } }, { surfaceOp: 'append' })) .toThrow(/outside any open turn/) }) @@ -100,7 +100,7 @@ describe('session-log invariants', () => { const session = ctx.sessions.create() // steering/message is turn-scoped: outside a turn it would land past the // commit boundary and be dropped on resume (the turn-enclosure RFC). - expect(() => session.append('steering/message', { turn: 1, content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } })) + expect(() => session.append('steering/message', { turn: 1, content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }, { surfaceOp: 'append' })) .toThrow(/outside any open turn/) // A PLUGIN-added (merge-extensible) event type is caught by the default too. // Cast through `any`: 'compaction/marker' is not in SessionEventType (it's @@ -115,7 +115,7 @@ describe('session-log invariants', () => { const { ctx } = await setup({ freeze: false }) const session = ctx.sessions.create() session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) - expect(() => session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } })) + expect(() => session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' })) .not.toThrow() }) @@ -124,7 +124,7 @@ describe('session-log invariants', () => { const session = ctx.sessions.create() session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) session.append('step/start', { turn: 1, step: 1 }) - expect(() => session.append('tool/result', { turn: 1, step: 1, callId: CallId('ghost'), content: [], isError: false })) + expect(() => session.append('tool/result', { turn: 1, step: 1, callId: CallId('ghost'), content: [], isError: false }, { surfaceOp: 'append' })) .toThrow(/no prior tool\/call/) }) @@ -136,7 +136,7 @@ describe('session-log invariants', () => { session.append('step/start', { turn: 1, step: 1 }) session.append('assistant/message', { turn: 1, step: 1, content: [ { type: 'tool-call', id: CallId('crashed'), name: 'bash', arguments: '{}' }, - ] }) + ] }, { surfaceOp: 'append' }) session.append('tool/result', { turn: 1, step: 1, @@ -144,7 +144,7 @@ describe('session-log invariants', () => { content: [{ type: 'text', text: 'interrupted' }], isError: true, error: { name: 'InterruptedError', code: 'interrupted' }, - }) + }, { surfaceOp: 'append' }) session.append('step/end', { turn: 1, step: 1 }) session.append('turn/end', { turn: 1, reason: { kind: 'interrupted' } }) }).not.toThrow() @@ -190,10 +190,10 @@ describe('session-log invariants', () => { expect(() => { session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) session.append('step/start', { turn: 1, step: 1 }) - session.append('assistant/message', { turn: 1, step: 1, content: [] }) + session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: 'append' }) session.append('step/end', { turn: 1, step: 1 }) session.append('step/start', { turn: 1, step: 2 }) - session.append('assistant/message', { turn: 1, step: 2, content: [] }) + session.append('assistant/message', { turn: 1, step: 2, content: [] }, { surfaceOp: 'append' }) session.append('step/end', { turn: 1, step: 2 }) session.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) session.append('turn/start', { turn: 2, trigger: { kind: 'message', source: { kind: 'user' } } }) @@ -246,7 +246,7 @@ describe('session-log invariants', () => { // step ends with the call unresolved — pendingCalls is cleared. session.append('step/end', { turn: 1, step: 1 }) session.append('step/start', { turn: 1, step: 2 }) - expect(() => session.append('tool/result', { turn: 1, step: 2, callId: CallId('c1'), content: [], isError: false })) + expect(() => session.append('tool/result', { turn: 1, step: 2, callId: CallId('c1'), content: [], isError: false }, { surfaceOp: 'append' })) .toThrow(/no prior tool\/call in this step/) }) @@ -255,7 +255,7 @@ describe('session-log invariants', () => { const session = ctx.sessions.create() session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) session.append('step/start', { turn: 1, step: 1 }) - expect(() => session.append('assistant/message', { turn: 1, step: 2, content: [] })) + expect(() => session.append('assistant/message', { turn: 1, step: 2, content: [] }, { surfaceOp: 'append' })) .toThrow(/open is turn 1\/step 1/) }) }) @@ -287,7 +287,7 @@ describe('dev-freeze', () => { const { ctx } = await setup() // freeze defaults true const session = ctx.sessions.create() session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) - const event = session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }) + const event = session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) expect(Object.isFrozen(event)).toBe(true) expect(Object.isFrozen(event.data)).toBe(true) expect(Object.isFrozen(event.data.content)).toBe(true) @@ -298,7 +298,7 @@ describe('dev-freeze', () => { const { ctx } = await setup({ freeze: false }) const session = ctx.sessions.create() session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) - const event = session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }) + const event = session.append('user/message', { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) expect(Object.isFrozen(event)).toBe(false) }) @@ -324,7 +324,7 @@ describe('dev-freeze', () => { // the caller's input — read the event back and assert on its data. const innerContent: { type: 'text'; text: string }[] = [{ type: 'text', text: 'inner' }] const block = Object.freeze({ type: 'tool-result' as const, toolCallId: CallId('c1'), content: innerContent, isError: false }) - const event = session.append('user/message', { content: [block], source: { kind: 'user' } }) + const event = session.append('user/message', { content: [block], source: { kind: 'user' } }, { surfaceOp: 'append' }) const logged = event.data.content[0] as { content: { type: 'text'; text: string }[] } expect(Object.isFrozen(logged.content)).toBe(true) expect(Object.isFrozen(logged.content[0])).toBe(true) @@ -424,7 +424,7 @@ describe('HMR safety', () => { const spy = vi.fn() ctx.on('session/event', spy) const session = ctx.sessions.create() - session.append('user/message', { content: [{ type: 'text', text: 'x' }], source: { kind: 'user' } }) + session.append('user/message', { content: [{ type: 'text', text: 'x' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // our own spy fires, proving events still flow — but the plugin's frozen. expect(spy).toHaveBeenCalledOnce() expect(Object.isFrozen(session.events[0])).toBe(false) From 1ec8c40d0dcef27e689e6d81412ae464d29c91da Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Tue, 23 Jun 2026 13:26:45 +0800 Subject: [PATCH 7/9] refactor(surface): use nodeBySeq map for lookup in _replace, drop dead params --- docs/core-data-structures/session.md | 46 +++++++++++++++++++++++++++- packages/core/session/src/surface.ts | 28 ++++++++--------- scripts/type-equiv.manifest.json | 4 +++ 3 files changed, 63 insertions(+), 15 deletions(-) diff --git a/docs/core-data-structures/session.md b/docs/core-data-structures/session.md index 62f6d062a4..2618db43c9 100644 --- a/docs/core-data-structures/session.md +++ b/docs/core-data-structures/session.md @@ -66,7 +66,51 @@ type SessionEvent = { `SessionEventType = keyof SessionEventMap`. Because `SessionEventMap` is merge-extensible, switches over `SessionEvent` must NOT use `assertNever` — a plugin-added variant is a valid unknown value; handle the known cases and fall through `default`. -The five message-producing types (`SurfaceEventType` — `user/message`, `assistant/message`, `tool/result`, `context/message`, `steering/message`) additionally carry two optional surface fields: `surfaceOp` (how the event enters the derived surface linked list — `'append'` or a `{ op: 'replace', start, end }` shadow) and `sourceEventSeqs` (provenance). See the [session surface RFC](../rfc/implemented/architecture/2026-06-18-session-surface.md). +## Surface types + +The five message-producing types (`SurfaceEventType` — `user/message`, `assistant/message`, `tool/result`, `context/message`, `steering/message`) carry surface metadata declaring how they join the derived surface linked list. See the [session surface RFC](../rfc/implemented/architecture/2026-06-18-session-surface.md). + +### `SurfaceEventType` — the message-producing subset of event types + +```ts type-equiv +export type SurfaceEventType = + | 'user/message' + | 'assistant/message' + | 'tool/result' + | 'context/message' + | 'steering/message' +``` + +### `SurfaceOp` — how an event entered the surface + +```ts type-equiv +export type SurfaceOp = + | 'append' + | { op: 'replace'; start: number; end: number } +``` + +`'append'` is the normal tail-append path. `replace` shadows surface nodes from `start` through `end` inclusive (both must be valid surface node seqs; `start === end` replaces a single node) and inserts the new node in their place. + +### `SurfaceIntent` — the parameter to `session.append()` + +```ts type-equiv +export interface SurfaceIntent { + surfaceOp: SurfaceOp + sourceEventSeqs?: number[] +} +``` + +Required for `SurfaceEventType` events — every message-producing event must declare how it joins the surface, the sole source of derived history. Non-surface types reject it at compile time. + +### `SurfaceNode` — a node in the surface linked list + +```ts type-equiv +export interface SurfaceNode { + seq: number + prev: number | null + next: number | null +} +``` ## Derived history: `deriveMessages()` diff --git a/packages/core/session/src/surface.ts b/packages/core/session/src/surface.ts index 7ed1743af3..3615668674 100644 --- a/packages/core/session/src/surface.ts +++ b/packages/core/session/src/surface.ts @@ -55,7 +55,7 @@ export interface SurfaceNode { export class SurfaceManager { /** Surface nodes in linked-list order (head to tail). Empty until first access. */ private _nodes: SurfaceNode[] = [] - /** Map from event seq → node for O(1) lookup during replacements. */ + /** Map from event seq → node. */ private _nodeBySeq = new Map() /** The last processed seq. -1 forces a full rebuild on first access. */ private _lastProcessedSeq = -1 @@ -100,7 +100,7 @@ export class SurfaceManager { this._nodes.push(node) this._nodeBySeq.set(event.seq, node) } else { - this._replace(this._nodes, this._nodeBySeq, event.seq, event.surfaceOp) + this._replace(event.seq, event.surfaceOp) } } this._lastProcessedSeq = this.log.length - 1 @@ -108,31 +108,31 @@ export class SurfaceManager { /** Apply a replace operation to the in-progress surface. */ private _replace( - nodes: SurfaceNode[], - nodeBySeq: Map, newSeq: number, op: Extract, ): void { - const startIdx = nodes.findIndex(n => n.seq === op.start) - if (startIdx === -1) { + const startNode = this._nodeBySeq.get(op.start) + if (!startNode) { throw new Error(`surface replace: start seq ${op.start} not found in surface`) } - const endIdx = nodes.findIndex(n => n.seq === op.end) - if (endIdx === -1) { + const endNode = this._nodeBySeq.get(op.end) + if (!endNode) { throw new Error(`surface replace: end seq ${op.end} not found in surface`) } + const startIdx = this._nodes.indexOf(startNode) + const endIdx = this._nodes.indexOf(endNode) if (startIdx > endIdx) { throw new Error(`surface replace: start seq ${op.start} (index ${startIdx}) is after end seq ${op.end} (index ${endIdx})`) } // Remove shadowed nodes from `[startIdx, endIdx]` inclusive. const count = endIdx - startIdx + 1 - const removed = nodes.splice(startIdx, count) - for (const r of removed) nodeBySeq.delete(r.seq) + const removed = this._nodes.splice(startIdx, count) + for (const r of removed) this._nodeBySeq.delete(r.seq) // Insert the new node where the removed range was. - const prevNode = startIdx > 0 ? nodes[startIdx - 1] : undefined - const nextNode = startIdx < nodes.length ? nodes[startIdx] : undefined + const prevNode = startIdx > 0 ? this._nodes[startIdx - 1] : undefined + const nextNode = startIdx < this._nodes.length ? this._nodes[startIdx] : undefined const newNode: SurfaceNode = { seq: newSeq, @@ -141,7 +141,7 @@ export class SurfaceManager { } if (prevNode) prevNode.next = newSeq if (nextNode) nextNode.prev = newSeq - nodes.splice(startIdx, 0, newNode) - nodeBySeq.set(newSeq, newNode) + this._nodes.splice(startIdx, 0, newNode) + this._nodeBySeq.set(newSeq, newNode) } } diff --git a/scripts/type-equiv.manifest.json b/scripts/type-equiv.manifest.json index f46c4fca6b..30165e2388 100644 --- a/scripts/type-equiv.manifest.json +++ b/scripts/type-equiv.manifest.json @@ -19,6 +19,10 @@ { "doc": "docs/core-data-structures/session.md", "symbol": "SessionEvent", "source": "packages/core/session/src/types.ts" }, { "doc": "docs/core-data-structures/session.md", "symbol": "TurnTriggerMap", "source": "packages/core/session/src/types.ts" }, { "doc": "docs/core-data-structures/session.md", "symbol": "TurnEndReasonMap", "source": "packages/core/session/src/types.ts" }, + { "doc": "docs/core-data-structures/session.md", "symbol": "SurfaceEventType", "source": "packages/core/session/src/types.ts" }, + { "doc": "docs/core-data-structures/session.md", "symbol": "SurfaceOp", "source": "packages/core/session/src/types.ts" }, + { "doc": "docs/core-data-structures/session.md", "symbol": "SurfaceIntent", "source": "packages/core/session/src/types.ts" }, + { "doc": "docs/core-data-structures/session.md", "symbol": "SurfaceNode", "source": "packages/core/session/src/surface.ts" }, { "doc": "docs/core-data-structures/persistence.md", "symbol": "SessionHeader", "source": "packages/core/session/src/types.ts" }, { "doc": "docs/core-data-structures/persistence.md", "symbol": "CreateSessionOptions", "source": "packages/core/session/src/types.ts" }, From 358ae02c5610e923e7a1e6405da29f9fd1fe9f71 Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Tue, 23 Jun 2026 13:37:13 +0800 Subject: [PATCH 8/9] =?UTF-8?q?feat(invariants):=20enforce=20replace=20pro?= =?UTF-8?q?venance=20=E2=80=94=20sourceEventSeqs=20must=20cover=20every=20?= =?UTF-8?q?shadowed=20surface=20node?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2026-06-18-session-surface.md | 2 +- packages/support/invariants/src/index.ts | 44 ++++++++- .../invariants/tests/invariants.spec.ts | 90 +++++++++++++++++++ 3 files changed, 132 insertions(+), 4 deletions(-) diff --git a/docs/rfc/implemented/architecture/2026-06-18-session-surface.md b/docs/rfc/implemented/architecture/2026-06-18-session-surface.md index ae2475c786..0ee163cbcf 100644 --- a/docs/rfc/implemented/architecture/2026-06-18-session-surface.md +++ b/docs/rfc/implemented/architecture/2026-06-18-session-surface.md @@ -49,7 +49,7 @@ The `repair.ts` module synthesizes `tool/result` closers for orphaned tool calls ### Invariants -The dev-mode invariants plugin validates: `sourceEventSeqs` references (non-empty, no duplicates, references earlier events, references known seqs) and `surfaceOp` (replace start ≤ end). +The dev-mode invariants plugin validates: `sourceEventSeqs` references (non-empty, no duplicates, references earlier events, references known seqs) and `surfaceOp` (replace `start ≤ end`, both endpoints are on the tracked surface, the range is non-reversed in surface position, and `sourceEventSeqs` includes every node the range shadows). ## Consequences diff --git a/packages/support/invariants/src/index.ts b/packages/support/invariants/src/index.ts index 492db4bdd0..b2372e28db 100644 --- a/packages/support/invariants/src/index.ts +++ b/packages/support/invariants/src/index.ts @@ -69,6 +69,13 @@ interface SessionTrace { pendingCalls: Set /** Every seq seen so far — validates `sourceEventSeqs` references. */ knownSeqs: Set + /** + * The seqs currently on the surface linked list, in linked-list order + * (head to tail). A replace reorders this relative to seq order (the new + * node takes the replaced range's position), so range validation is + * positional, not by seq comparison. + */ + surface: number[] } /** @@ -147,9 +154,39 @@ function checkEvent(trace: SessionTrace, event: SessionEvent): void { } } } - if (se.surfaceOp !== undefined && typeof se.surfaceOp !== 'string') { - if (se.surfaceOp.start > se.surfaceOp.end) { - throw new InvariantError(`surface replace: start ${se.surfaceOp.start} must be <= end ${se.surfaceOp.end}`) + // Fold this event into the tracked surface linked list, validating the + // replace contract as we go. `append` adds a tail node; `replace` shadows a + // positional range — every shadowed node must appear in sourceEventSeqs. + if (se.surfaceOp !== undefined) { + if (se.surfaceOp === 'append') { + trace.surface.push(event.seq) + } else { + const { start, end } = se.surfaceOp + if (start > end) { + throw new InvariantError(`surface replace: start ${start} must be <= end ${end}`) + } + const startIdx = trace.surface.indexOf(start) + if (startIdx === -1) { + throw new InvariantError(`surface replace: start seq ${start} is not on the surface`) + } + const endIdx = trace.surface.indexOf(end) + if (endIdx === -1) { + throw new InvariantError(`surface replace: end seq ${end} is not on the surface`) + } + if (startIdx > endIdx) { + throw new InvariantError(`surface replace: start seq ${start} (pos ${startIdx}) is after end seq ${end} (pos ${endIdx}) on the surface`) + } + // Every node the replace shadows (surface positions [startIdx, endIdx] + // inclusive) must appear in sourceEventSeqs — the provenance contract. + const shadowed = trace.surface.slice(startIdx, endIdx + 1) + const recorded = new Set(se.sourceEventSeqs ?? []) + const missing = shadowed.filter(seq => !recorded.has(seq)) + if (missing.length > 0) { + throw new InvariantError(`surface replace: sourceEventSeqs must include every shadowed surface node; missing ${missing.join(', ')}`) + } + // Apply the replace to the tracked surface: the new node takes the + // range's position so order stays in sync for later replaces. + trace.surface.splice(startIdx, shadowed.length, event.seq) } } @@ -288,6 +325,7 @@ export function apply(ctx: Context, config: Config = {}): void { nextStep: 1, pendingCalls: new Set(), knownSeqs: new Set(), + surface: [], }) /** Build (or rebuild) a session's trace by replaying its whole log; freeze it. */ diff --git a/packages/support/invariants/tests/invariants.spec.ts b/packages/support/invariants/tests/invariants.spec.ts index 2eaae73169..21f6356aee 100644 --- a/packages/support/invariants/tests/invariants.spec.ts +++ b/packages/support/invariants/tests/invariants.spec.ts @@ -542,6 +542,96 @@ describe('surface invariants', () => { }).toThrow(/must be <= end/) }) + it('rejects a replace whose sourceEventSeqs omits a shadowed surface node', async () => { + const { ctx } = await setup() + const session = ctx.sessions.create() + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + session.append('step/start', { turn: 1, step: 1 }) + session.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 2 + session.append('user/message', { content: [{ type: 'text', text: 'b' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 3 + // Replace shadows surface nodes [2, 3] but records provenance for only [2]. + expect(() => { + session.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 'sum' }] }, { surfaceOp: { op: 'replace', start: 2, end: 3 }, sourceEventSeqs: [2] }) + }).toThrow(/must include every shadowed surface node; missing 3/) + }) + + it('accepts a replace whose sourceEventSeqs covers every shadowed surface node', async () => { + const { ctx } = await setup() + const session = ctx.sessions.create() + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + session.append('step/start', { turn: 1, step: 1 }) + session.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 2 + session.append('user/message', { content: [{ type: 'text', text: 'b' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 3 + expect(() => { + session.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 'sum' }] }, { surfaceOp: { op: 'replace', start: 2, end: 3 }, sourceEventSeqs: [2, 3] }) + }).not.toThrow() + }) + + it('rejects a replace naming a start seq that is not on the surface', async () => { + const { ctx } = await setup() + const session = ctx.sessions.create() + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + session.append('step/start', { turn: 1, step: 1 }) + session.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 2 + // seq 1 (step/start) is a real earlier event but never entered the surface. + expect(() => { + session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 1, end: 2 }, sourceEventSeqs: [1, 2] }) + }).toThrow(/start seq 1 is not on the surface/) + }) + + it('rejects a replace naming an end seq that is not on the surface', async () => { + const { ctx } = await setup() + const session = ctx.sessions.create() + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + session.append('step/start', { turn: 1, step: 1 }) + session.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 2 + // start (2) is on the surface but end (99) never entered it. + expect(() => { + session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 2, end: 99 }, sourceEventSeqs: [2] }) + }).toThrow(/end seq 99 is not on the surface/) + }) + + it('rejects a replace whose range is reversed in surface position after a prior replace reordered it', async () => { + const { ctx } = await setup() + const session = ctx.sessions.create() + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + session.append('step/start', { turn: 1, step: 1 }) + session.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 2 + session.append('user/message', { content: [{ type: 'text', text: 'b' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 3 + // Replace node 2 (position 0) with seq 4 — surface is now [4, 3], so seq 4 + // precedes seq 3 in linked-list order even though 4 > 3 numerically. + session.append('assistant/message', { turn: 1, step: 1, content: [{ type: 'text', text: 's' }] }, { surfaceOp: { op: 'replace', start: 2, end: 2 }, sourceEventSeqs: [2] }) // seq 4 + // A replace with start=3, end=4 passes the seq check (3 <= 4) but is + // reversed positionally (3 is at pos 1, 4 is at pos 0). + expect(() => { + session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 3, end: 4 }, sourceEventSeqs: [3, 4] }) // seq 5 + }).toThrow(/is after end seq 4 .* on the surface/) + }) + + it('rejects a replace that omits sourceEventSeqs entirely', async () => { + const { ctx } = await setup() + const session = ctx.sessions.create() + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + session.append('step/start', { turn: 1, step: 1 }) + session.append('user/message', { content: [{ type: 'text', text: 'a' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) // seq 2 + // A replace with no sourceEventSeqs records no provenance for the node it shadows. + expect(() => { + session.append('assistant/message', { turn: 1, step: 1, content: [] }, { surfaceOp: { op: 'replace', start: 2, end: 2 } }) + }).toThrow(/must include every shadowed surface node; missing 2/) + }) + + it('catches an incomplete-provenance replace on the load/seed path', async () => { + const { ctx } = await setup({ freeze: false }) + const badSeed = [ + { type: 'turn/start' as const, seq: 0, time: 0, data: { turn: 1, trigger: { kind: 'message' as const, source: { kind: 'user' as const } } } }, + { type: 'step/start' as const, seq: 1, time: 0, data: { turn: 1, step: 1 } }, + { type: 'user/message' as const, seq: 2, time: 0, data: { content: [{ type: 'text' as const, text: 'a' }], source: { kind: 'user' as const } }, surfaceOp: 'append' as const }, + { type: 'user/message' as const, seq: 3, time: 0, data: { content: [{ type: 'text' as const, text: 'b' }], source: { kind: 'user' as const } }, surfaceOp: 'append' as const }, + { type: 'assistant/message' as const, seq: 4, time: 0, data: { turn: 1, step: 1, content: [{ type: 'text' as const, text: 'sum' }] }, surfaceOp: { op: 'replace' as const, start: 2, end: 3 }, sourceEventSeqs: [2] }, + ] + expect(() => ctx.sessions.create(undefined, { seed: badSeed })).toThrow(/must include every shadowed surface node; missing 3/) + }) + it('rejects sourceEventSeqs on a non-surface event', async () => { const { ctx } = await setup() const session = ctx.sessions.create() From 828c3f85c9d61a5bcea7ccb894feb9896343de33 Mon Sep 17 00:00:00 2001 From: Hypatia May Date: Wed, 24 Jun 2026 17:45:48 +0800 Subject: [PATCH 9/9] fix review findings: skip collided SCHEMA_VERSION 3; reject marker-less surface events MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P1: both merge parents shipped SCHEMA_VERSION=3 for different layouts (surface columns vs seed_length), so an on-disk 3 was ambiguous and wrongly accepted. Bump to 4 (merged layout) so the version check rejects both sibling v3s. P2: a surface-eligible event with no surfaceOp lands in the log but vanishes from deriveMessages() (surface is the sole derivation path). The typed append overload enforces the marker only when the type arg is a literal; it collapses to optional when widened to the union (a caller iterating raw events). Guard at runtime in both append() and the seed constructor — no backward-compat for surface-less logs. Shared seed fixtures carry surfaceOp explicitly and the appendLog helper forwards it verbatim (no synthesized default). Exports isSurfaceEligibleType. Regression tests for all three, each verified to fail on the unfixed code. Gates: typecheck, test (1115), snapshot (14), doc-sync, lint, build, hygiene green. --- docs/cordis-catalog/events-and-services.md | 2 +- .../2026-06-18-session-surface.md | 4 ++- ...6-06-22-fork-child-replay-seed-boundary.md | 2 +- packages/core/session/README.md | 5 +-- packages/core/session/src/index.ts | 27 +++++++++++++-- packages/core/session/src/surface.ts | 12 +++++++ .../core/session/tests/properties.spec.ts | 27 ++++++++++----- packages/core/session/tests/session.spec.ts | 33 +++++++++++++++++-- .../tests/jsonl.spec.ts | 6 ++-- .../session-persistence-sqlite/src/schema.ts | 14 +++++--- .../tests/sqlite.spec.ts | 32 +++++++++++++++--- .../session-persistence/tests/contract.ts | 32 ++++++++++++++++-- .../tests/coordinator-contract.ts | 4 +-- .../invariants/tests/invariants.spec.ts | 2 +- 14 files changed, 166 insertions(+), 36 deletions(-) diff --git a/docs/cordis-catalog/events-and-services.md b/docs/cordis-catalog/events-and-services.md index 737a4cc8ef..bac283b1f8 100644 --- a/docs/cordis-catalog/events-and-services.md +++ b/docs/cordis-catalog/events-and-services.md @@ -412,7 +412,7 @@ get(id: SessionId): Session | undefined list(): Session[] ``` -Source: [`packages/core/session/src/index.ts:300`](../../packages/core/session/src/index.ts) +Source: [`packages/core/session/src/index.ts:321`](../../packages/core/session/src/index.ts) ### `ctx.subagents` — `SubagentService` diff --git a/docs/rfc/implemented/architecture/2026-06-18-session-surface.md b/docs/rfc/implemented/architecture/2026-06-18-session-surface.md index 0ee163cbcf..3f4419b5ca 100644 --- a/docs/rfc/implemented/architecture/2026-06-18-session-surface.md +++ b/docs/rfc/implemented/architecture/2026-06-18-session-surface.md @@ -51,9 +51,11 @@ The `repair.ts` module synthesizes `tool/result` closers for orphaned tool calls The dev-mode invariants plugin validates: `sourceEventSeqs` references (non-empty, no duplicates, references earlier events, references known seqs) and `surfaceOp` (replace `start ≤ end`, both endpoints are on the tracked surface, the range is non-reversed in surface position, and `sourceEventSeqs` includes every node the range shadows). +Because the surface is the SOLE derivation path, a surface-eligible event that carries no `surfaceOp` marker is invisible to `deriveMessages()` — it would land in the log yet silently drop from history on resume/fork. `append`'s typed overload makes the marker mandatory for `SurfaceEventType` events at compile time, but only when the type argument is a SPECIFIC literal; when it widens to the `SessionEventType` union (a caller iterating raw events, e.g. `for (const e of log) append(e.type, e.data)`) the conditional rest collapses to optional and the compiler stops enforcing it. The marker requirement is therefore ALSO checked at runtime in two places: `append` itself throws on a marker-less surface-eligible event (covering the union-widening loophole), and the `Session` seed constructor re-checks the same invariant (alongside its seq-contiguity and JSON-serializability checks) so a seed/load/fork — which arrives as raw `SessionEvent[]`, bypassing `append` — is REJECTED rather than constructing a session that resumes with missing history. (No backward-compat path for surface-less logs: per the pre-release stance there is no persisted user data to preserve, so such a log is rejected, not upgraded.) + ## Consequences -- **`packages/core/session`**: New `surface.ts` (`SurfaceManager`), new types (`SurfaceOp`, `SurfaceIntent`), new fields on `SessionEvent`, modified `append()` (third required `SurfaceIntent` param), refactored `deriveMessages()` (surface path + legacy fallback), surface-aware `repair.ts`. +- **`packages/core/session`**: New `surface.ts` (`SurfaceManager`), new types (`SurfaceOp`, `SurfaceIntent`), new fields on `SessionEvent`, modified `append()` (third required `SurfaceIntent` param), refactored `deriveMessages()` (walks the surface as the sole derivation path), surface-aware `repair.ts`. The seed constructor rejects a surface-eligible seed event missing its `surfaceOp` marker (see § Invariants). - **`packages/core/agent-loop`**: All surface-capable appends pass surface opts. Chunk seqs are collected for `assistant/message` provenance; `tool/call` seqs are captured for `tool/result` provenance. - **`packages/session-persistence/session-persistence-sqlite`**: Two new nullable TEXT columns (`source_event_seqs`, `surface_op`) on the `events` table; `SCHEMA_VERSION` bumped (bump-and-reject, no migration). - **`packages/support/invariants`**: Surface-related validation rules. diff --git a/docs/rfc/implemented/testing/2026-06-22-fork-child-replay-seed-boundary.md b/docs/rfc/implemented/testing/2026-06-22-fork-child-replay-seed-boundary.md index a45e62639b..a60d487e91 100644 --- a/docs/rfc/implemented/testing/2026-06-22-fork-child-replay-seed-boundary.md +++ b/docs/rfc/implemented/testing/2026-06-22-fork-child-replay-seed-boundary.md @@ -27,7 +27,7 @@ Record where a session's **inherited** prefix ends, persist it, and have the rep - **JSONL**: a `seedLength` field on the header line (`toHeaderLine`/`fromHeaderLine`). - **SQLite**: a `seed_length` column on the `sessions` table. -The SQLite change is a breaking table-layout change, so `SCHEMA_VERSION` bumps **2 → 3**. Per the repo's pre-release stance (§ "Pre-release stance" in AGENTS.md) the backend **rejects** a non-current `user_version` on open rather than migrating it — there is no persisted user data to preserve, so no migration code is written (the existing reject-not-migrate path at `openDatabase` already enforces this; v1 and now v2 are both rejected). +The SQLite change is a breaking table-layout change, so `SCHEMA_VERSION` bumps. This branch added `seed_length` under version **3**; it later merged with the session-surface branch, which had independently shipped its OWN version-3 layout (the `source_event_seqs`/`surface_op` columns). Because an on-disk `3` is ambiguous between the two sibling layouts, the merged build is version **4** (every column), and an on-disk `3` is rejected like any other non-current version. Per the repo's pre-release stance (§ "Pre-release stance" in AGENTS.md) the backend **rejects** a non-current `user_version` on open rather than migrating it — there is no persisted user data to preserve, so no migration code is written (the existing reject-not-migrate path at `openDatabase` already enforces this; v1, v2, and the collided v3 are all rejected). ### 3. Replay derives a child script after the boundary diff --git a/packages/core/session/README.md b/packages/core/session/README.md index 08323eff0c..2a90d0f792 100644 --- a/packages/core/session/README.md +++ b/packages/core/session/README.md @@ -34,7 +34,7 @@ Creates and holds event-sourced `Session` instances. Persistence is intentionall Plain class (not a Cordis Service). Create via `ctx.sessions.create()`. -- `session.append(type, data, opts?): SessionEvent` — synchronous, never blocks on I/O. **Throws** if `data` is not losslessly JSON-serializable (BigInt, function, symbol, undefined, non-finite number, circular ref, or an exotic object like Map/Set/Date) — the event log is the durable source of truth, so this invariant is enforced at the source (exported as `isJsonValue` for backends to reuse on their replay/fork entry points). A third parameter `opts: SurfaceIntent` carries surface metadata: `surfaceOp` controls how the event enters the surface linked list, and `sourceEventSeqs` records provenance (the seq numbers of events this one derives from). It is **required** for the five `SurfaceEventType` events (every message-producing event must declare how it joins the surface) and rejected by the compiler for non-surface types. +- `session.append(type, data, opts?): SessionEvent` — synchronous, never blocks on I/O. **Throws** if `data` is not losslessly JSON-serializable (BigInt, function, symbol, undefined, non-finite number, circular ref, or an exotic object like Map/Set/Date) — the event log is the durable source of truth, so this invariant is enforced at the source (exported as `isJsonValue` for backends to reuse on their replay/fork entry points). A third parameter `opts: SurfaceIntent` carries surface metadata: `surfaceOp` controls how the event enters the surface linked list, and `sourceEventSeqs` records provenance (the seq numbers of events this one derives from). It is **required** for the five `SurfaceEventType` events (every message-producing event must declare how it joins the surface) and rejected by the compiler for non-surface types. The marker requirement is enforced two ways: the typed overload makes `opts` mandatory when `type` is a specific `SurfaceEventType` literal, AND `append` **throws** at runtime if a surface-eligible event arrives with no `surfaceOp` — covering the case where `type` widens to the `SessionEventType` union (a caller iterating raw events, where the conditional overload collapses to optional) so a marker-less message event can never silently land in the log and vanish from `deriveMessages()`. - `session.deriveMessages(): Message[]` — derive the LLM message history by walking the surface linked list (skipping non-surface events like chunks and boundaries; a `replace` shadows the nodes it covers). The surface is the single source of derived history — there is no raw-log fallback. - `session.surface: SurfaceManager` — the derived surface, lazily rebuilt from `surfaceOp` markers in the log. Processes only new events (delta) on each access — the log is append-only, so prior events never change. - `session.events`, `session.seq`, `session.id` @@ -45,6 +45,7 @@ Plain class (not a Cordis Service). Create via `ctx.sessions.create()`. - `SurfaceOp` — how a surface node entered the linked list: `'append'` (normal tail append) or `{ op: 'replace', start, end }` (replace nodes from `start` through `end` inclusive — both must be valid surface node seqs; `start === end` replaces a single node). Used by compaction to shadow old nodes without deleting them. - `SurfaceIntent` — `{ surfaceOp: SurfaceOp; sourceEventSeqs?: number[] }`, the required third parameter to `session.append()` for surface-eligible types. - `SurfaceNode` — `{ seq: number; prev: number | null; next: number | null }`, one node in the surface linked list. +- `isSurfaceEvent(event)` / `isSurfaceEligibleType(type)` — the first narrows a `SessionEvent` to a fully-formed surface node (type is surface-eligible AND `surfaceOp` present); the second is the type-only check (is this one of the five `SurfaceEventType` values?), used to detect a surface-eligible event MISSING its marker — e.g. when validating a seed/load log. ### Session event vocabulary (`types.ts`) @@ -66,7 +67,7 @@ Every `SessionEvent` carries two optional top-level fields (structural metadata) ### Extension points - Persistence plugins: subscribe to `session/event` (write-behind) and drain on `session/flush` (awaited) and fiber dispose. A durable backend reads the log and reloads it into a live session; the metadata seam (`SessionHeader`, `session.header`) is what such a backend stores beside the log. -- Replay/fork: `ctx.sessions.create(id, { seed })` seeds a new session with an existing event log. The surface rebuilds deterministically from `surfaceOp` markers in the seeded events. +- Replay/fork: `ctx.sessions.create(id, { seed })` seeds a new session with an existing event log. The surface rebuilds deterministically from `surfaceOp` markers in the seeded events. The seed is validated to the SAME invariants `append` enforces — including that every surface-eligible event (`SurfaceEventType`) carries a `surfaceOp` marker — so a marker-less message event is rejected at construction rather than silently vanishing from `deriveMessages()` (the surface is the sole derivation path) on resume. - Compaction: a future plugin appends a new event with `surfaceOp: { op: 'replace', start, end }` to shadow old surface nodes. ### What is NOT here (TODO) diff --git a/packages/core/session/src/index.ts b/packages/core/session/src/index.ts index 8908059dae..2002c93051 100644 --- a/packages/core/session/src/index.ts +++ b/packages/core/session/src/index.ts @@ -12,13 +12,13 @@ import type { ContentBlock, Message, MessageSource } from '@deepseek-ai/dsh-llm' import { SESSION_FORMAT_VERSION, SessionId } from './types.ts' import type { CreateSessionOptions, SessionEvent, SessionEventMap, SessionEventType, SessionHeader, SurfaceIntent, SurfaceEventType } from './types.ts' import { isJsonValue } from './json.ts' -import { SurfaceManager } from './surface.ts' +import { SurfaceManager, isSurfaceEligibleType } from './surface.ts' export * from './types.ts' export { isJsonValue } from './json.ts' export { interruptedTurnClosers } from './repair.ts' export type { SurfaceNode } from './surface.ts' -export { isSurfaceEvent } from './surface.ts' +export { isSurfaceEvent, isSurfaceEligibleType } from './surface.ts' declare module 'cordis' { interface Context { @@ -120,6 +120,16 @@ export class Session { if (!isJsonValue(event.data)) { throw new Error(`seed event "${event.type}" (seq ${event.seq}) carries non-JSON-serializable data`) } + // Surface-eligible events MUST carry a surfaceOp marker — the surface is + // the sole source of derived history, so a marker-less message event + // would load fine yet vanish from deriveMessages(). `append` enforces + // this at compile time via its typed overload; a seed arrives as raw + // SessionEvent[] (replay/fork/load), bypassing that, so re-check at + // runtime here rather than silently resuming with empty history. + if (isSurfaceEligibleType(event.type) + && (event as SessionEvent).surfaceOp === undefined) { + throw new Error(`seed event "${event.type}" (seq ${event.seq}) is surface-eligible but carries no surfaceOp marker`) + } }) // Deep-clone each seed event, NOT just the array: the seed events and // their `data` are still owned by the caller (or the source session of a @@ -172,6 +182,18 @@ export class Session { if (!isJsonValue(data)) { throw new Error(`session event "${type}" carries non-JSON-serializable data`) } + const surfaceOpts: SurfaceIntent | undefined = opts[0] + // Surface-eligible events MUST carry a surfaceOp marker — the surface is the + // sole source of derived history, so a marker-less message event would be + // logged yet vanish from deriveMessages(). The typed `opts` overload makes + // the marker mandatory only when `T` is a SPECIFIC SurfaceEventType literal; + // when `T` widens to the SessionEventType union (a caller iterating raw + // events: `for (const e of log) append(e.type, e.data)`), the conditional + // rest collapses to optional and the compiler stops enforcing it. Re-check + // at runtime so that loophole can't silently drop history. + if (isSurfaceEligibleType(type) && surfaceOpts?.surfaceOp === undefined) { + throw new Error(`session event "${type}" is surface-eligible and requires a surfaceOp marker`) + } // Snapshot `data` into the log, NOT the caller's reference: the validation // above proves it is JSON-serializable AT THIS MOMENT, but the caller still // owns the object and could mutate it afterwards (before a persistence @@ -185,7 +207,6 @@ export class Session { // Surface metadata is snapshot separately: sourceEventSeqs (number[] — // primitives, so array spread is a complete copy) and surfaceOp (a string // primitive, or cloned if it's a replace object). - const surfaceOpts: SurfaceIntent | undefined = opts[0] // Build the event shape with conditional surface fields via spreading. // The result is cast through `unknown` because the conditional spreads // produce an intersection type that the assignability checker can't diff --git a/packages/core/session/src/surface.ts b/packages/core/session/src/surface.ts index 3615668674..57f9a65864 100644 --- a/packages/core/session/src/surface.ts +++ b/packages/core/session/src/surface.ts @@ -22,6 +22,18 @@ const SURFACE_EVENT_TYPES = new Set([ 'steering/message', ]) +/** + * Whether an event's `type` is surface-eligible (one of the five + * message-producing {@link SurfaceEventType} values). This is the TYPE check + * only — it does NOT require `surfaceOp` to be present. Use it to detect a + * surface-eligible event that is MISSING its mandatory marker (e.g. validating + * a seed/load log); use {@link isSurfaceEvent} to narrow to a fully-formed + * {@link SurfaceEvent} with `surfaceOp` present. + */ +export function isSurfaceEligibleType(type: string): boolean { + return SURFACE_EVENT_TYPES.has(type) +} + /** * Narrow a {@link SessionEvent} to {@link SurfaceEvent}: checks that the * event's `type` is surface-eligible AND that `surfaceOp` is present. diff --git a/packages/core/session/tests/properties.spec.ts b/packages/core/session/tests/properties.spec.ts index 42149515f2..4be0cbcf6d 100644 --- a/packages/core/session/tests/properties.spec.ts +++ b/packages/core/session/tests/properties.spec.ts @@ -11,22 +11,29 @@ import { describe, expect, it } from 'vitest' import fc from 'fast-check' import { CallId } from '@deepseek-ai/dsh-llm' import { Session, SessionId } from '@deepseek-ai/dsh-session' -import type { SessionEventMap, SessionEventType } from '@deepseek-ai/dsh-session' +import type { SessionEventMap, SessionEventType, SurfaceIntent } from '@deepseek-ai/dsh-session' -type Appendable = { [T in SessionEventType]: { type: T; data: SessionEventMap[T] } }[SessionEventType] +// An appendable event: its type/data plus, for surface-eligible types, the +// explicit surface intent the generator declares (mirroring how a real caller +// passes it). The intent is part of the generated fixture, NOT synthesized by +// `build`, so each arbitrary states the marker it produces. +type Appendable = { + [T in SessionEventType]: { type: T; data: SessionEventMap[T]; intent?: SurfaceIntent } +}[SessionEventType] const textContentArb = fc.array( fc.record({ type: fc.constant<'text'>('text'), text: fc.string() }), { maxLength: 3 }, ) -// A message-producing event (these DO affect derived history). +// A message-producing event (these DO affect derived history). Each carries an +// explicit `surfaceOp: 'append'` intent — the marker the real loop passes. const messageEventArb: fc.Arbitrary = fc.oneof( - textContentArb.map((content): Appendable => ({ type: 'user/message', data: { content, source: { kind: 'user' } } })), - textContentArb.map((content): Appendable => ({ type: 'assistant/message', data: { turn: 1, step: 1, content } })), - textContentArb.map((content): Appendable => ({ type: 'assistant/message', data: { turn: 1, step: 1, content, usage: { inputTokens: 1, outputTokens: 1 } } })), + textContentArb.map((content): Appendable => ({ type: 'user/message', data: { content, source: { kind: 'user' } }, intent: { surfaceOp: 'append' } })), + textContentArb.map((content): Appendable => ({ type: 'assistant/message', data: { turn: 1, step: 1, content }, intent: { surfaceOp: 'append' } })), + textContentArb.map((content): Appendable => ({ type: 'assistant/message', data: { turn: 1, step: 1, content, usage: { inputTokens: 1, outputTokens: 1 } }, intent: { surfaceOp: 'append' } })), fc.record({ id: fc.string({ minLength: 1 }), content: textContentArb, isError: fc.boolean() }) - .map((r): Appendable => ({ type: 'tool/result', data: { turn: 1, step: 1, callId: CallId(r.id), content: r.content, isError: r.isError } })), + .map((r): Appendable => ({ type: 'tool/result', data: { turn: 1, step: 1, callId: CallId(r.id), content: r.content, isError: r.isError }, intent: { surfaceOp: 'append' } })), ) // A non-message event (trace/replay data — must NOT affect derived history). @@ -44,7 +51,11 @@ const logArb = fc.array(anyEventArb, { maxLength: 25 }) let counter = 0 function build(events: Appendable[]): Session { const session = new Session(SessionId(`prop-${counter++}`)) - for (const e of events) session.append(e.type, e.data) + for (const e of events) { + // Forward the generated intent verbatim; non-surface events carry none. + if (e.intent !== undefined) session.append(e.type, e.data, e.intent) + else session.append(e.type, e.data) + } return session } diff --git a/packages/core/session/tests/session.spec.ts b/packages/core/session/tests/session.spec.ts index 46f96742cc..6138a479f1 100644 --- a/packages/core/session/tests/session.spec.ts +++ b/packages/core/session/tests/session.spec.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest' import { Context } from 'cordis' import { CallId } from '@deepseek-ai/dsh-llm' import SessionStore, { SESSION_FORMAT_VERSION, Session, SessionEvent, SessionId } from '@deepseek-ai/dsh-session' +import type { SessionEventType } from '@deepseek-ai/dsh-session' describe('Session', () => { it('derives message history from the event log', () => { @@ -120,6 +121,21 @@ describe('Session', () => { expect(session.events).toHaveLength(0) }) + it('rejects a surface-eligible append with no surfaceOp marker (runtime guard for the union-widening loophole)', () => { + const session = new Session(SessionId('s5b')) + session.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) + // The typed overload makes surfaceOp mandatory only when the type argument is + // a SPECIFIC SurfaceEventType literal. A caller iterating raw events widens it + // to the SessionEventType union, where the conditional rest collapses to + // optional — the exact shape `for (const e of log) append(e.type, e.data)` + // produces. Reproduce that here and assert the runtime guard rejects it. + const widenedType = 'user/message' as SessionEventType + expect(() => session.append(widenedType, { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } })) + .toThrow(/surface-eligible and requires a surfaceOp marker/) + // The rejected append never entered the log (only turn/start is present). + expect(session.events).toHaveLength(1) + }) + it('accepts dense arrays and nested plain objects', () => { const session = new Session(SessionId('s6')) expect(() => session.append('user/message', { content: [{ type: 'text', text: 'x' }], source: { kind: 'user' }, extra: [1, 2, [3, { a: null, b: true }]] } as never, { surfaceOp: 'append' })).not.toThrow() @@ -143,10 +159,23 @@ describe('Session', () => { expect(() => new Session(SessionId('seed-gap'), gapSeed)).toThrow(/contiguous|seq/) }) + it('validates seed events: rejects a surface-eligible event missing its surfaceOp marker', () => { + // A surface-eligible event (user/message) with no surfaceOp would load fine + // but vanish from deriveMessages() (the surface is the sole derivation path), + // so a resume/fork would silently lose history. append() forbids this at + // compile time; a raw seed must be rejected at runtime to match. + const markerlessSeed = [ + { type: 'turn/start' as const, seq: 0, time: 1, data: { turn: 1, trigger: { kind: 'message' as const, source: { kind: 'user' as const } } } }, + { type: 'user/message' as const, seq: 1, time: 2, data: { content: [{ type: 'text' as const, text: 'hi' }], source: { kind: 'user' as const } } }, + { type: 'turn/end' as const, seq: 2, time: 3, data: { turn: 1, reason: { kind: 'completed' as const } } }, + ] as SessionEvent[] + expect(() => new Session(SessionId('seed-no-marker'), markerlessSeed)).toThrow(/surface-eligible but carries no surfaceOp/) + }) + it('accepts a well-formed contiguous serializable seed', () => { const goodSeed = [ { type: 'turn/start' as const, seq: 0, time: 1, data: { turn: 1, trigger: { kind: 'message' as const, source: { kind: 'user' as const } } } }, - { type: 'user/message' as const, seq: 1, time: 2, data: { content: [{ type: 'text' as const, text: 'hi' }], source: { kind: 'user' as const } } }, + { type: 'user/message' as const, seq: 1, time: 2, data: { content: [{ type: 'text' as const, text: 'hi' }], source: { kind: 'user' as const } }, surfaceOp: 'append' as const }, { type: 'turn/end' as const, seq: 2, time: 3, data: { turn: 1, reason: { kind: 'completed' as const } } }, ] as SessionEvent[] const session = new Session(SessionId('seed-ok'), goodSeed) @@ -156,7 +185,7 @@ describe('Session', () => { it('snapshots the seed: mutating the original after construction does not affect session.events', () => { const seed = [ { type: 'turn/start' as const, seq: 0, time: 1, data: { turn: 1, trigger: { kind: 'message' as const, source: { kind: 'user' as const } } } }, - { type: 'user/message' as const, seq: 1, time: 2, data: { content: [{ type: 'text' as const, text: 'original' }], source: { kind: 'user' as const } } }, + { type: 'user/message' as const, seq: 1, time: 2, data: { content: [{ type: 'text' as const, text: 'original' }], source: { kind: 'user' as const } }, surfaceOp: 'append' as const }, { type: 'turn/end' as const, seq: 2, time: 3, data: { turn: 1, reason: { kind: 'completed' as const } } }, ] as SessionEvent[] const session = new Session(SessionId('seed-snapshot'), seed) diff --git a/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts b/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts index 7df0fd5180..c6b7903357 100644 --- a/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts +++ b/packages/session-persistence/session-persistence-jsonl/tests/jsonl.spec.ts @@ -7,7 +7,7 @@ import SessionStore, { SessionId } from '@deepseek-ai/dsh-session' import type { Session, SessionEvent } from '@deepseek-ai/dsh-session' import SessionPersistenceJsonl from '@deepseek-ai/dsh-session-persistence-jsonl' import { encodeSegment, logPath, scanLog, sessionDir } from '../src/format.ts' -import { runPersistenceContract, meta, oneTurnLog } from '../../session-persistence/tests/contract.ts' +import { runPersistenceContract, meta, oneTurnLog, appendLog } from '../../session-persistence/tests/contract.ts' import { runCoordinatorContract, type CoordinatorFixture } from '../../session-persistence/tests/coordinator-contract.ts' let root: string @@ -121,7 +121,7 @@ describe('SessionPersistenceJsonl: durability and crash semantics', () => { { type: 'step/start', seq: 1, time: 2, data: { turn: 1, step: 1 } }, { type: 'assistant/chunk', seq: 2, time: 3, data: { turn: 1, step: 1, chunk: { type: 'text-delta', index: 0, text: 'he' } } }, { type: 'assistant/chunk', seq: 3, time: 4, data: { turn: 1, step: 1, chunk: { type: 'text-delta', index: 0, text: 'llo' } } }, - { type: 'assistant/message', seq: 4, time: 5, data: { turn: 1, step: 1, content: [{ type: 'text', text: 'hello' }] } }, + { type: 'assistant/message', seq: 4, time: 5, data: { turn: 1, step: 1, content: [{ type: 'text', text: 'hello' }] }, surfaceOp: 'append', sourceEventSeqs: [2, 3] }, { type: 'step/end', seq: 5, time: 6, data: { turn: 1, step: 1 } }, { type: 'turn/end', seq: 6, time: 7, data: { turn: 1, reason: { kind: 'completed' } } }, ] @@ -452,7 +452,7 @@ describe('SessionPersistenceJsonl: edge cases', () => { // Session A materializes a log under id "reuse". const sessFiberA = await ctx.plugin(Object.assign((inner: Context) => { const a = inner.sessions.create(SessionId('reuse'), { meta: { cwd: '/a' } }) - for (const e of oneTurnLog()) a.append(e.type, e.data) + appendLog(a, oneTurnLog()) }, { inject: ['sessions'] })) // Drain A, then dispose ITS fiber (the live session A is gone) while the // backend stays loaded. diff --git a/packages/session-persistence/session-persistence-sqlite/src/schema.ts b/packages/session-persistence/session-persistence-sqlite/src/schema.ts index 37acc6f0f0..d8db0e087b 100644 --- a/packages/session-persistence/session-persistence-sqlite/src/schema.ts +++ b/packages/session-persistence/session-persistence-sqlite/src/schema.ts @@ -15,7 +15,7 @@ import type { SessionEvent, SessionId, SessionHeader, SurfaceOp } from '@deepsee * layout; orthogonal to a session's own `version` (which versions the EVENT * vocabulary, stored per session in the `sessions` row). */ -export const SCHEMA_VERSION = 3 +export const SCHEMA_VERSION = 4 /** * A row of the `sessions` table — the out-of-log metadata ({@link SessionHeader}). @@ -56,9 +56,15 @@ export interface EventRow { * current {@link SCHEMA_VERSION}; an existing database whose version is NOT the * current one (written by a different, incompatible build — older or newer) is * REJECTED rather than opened against a layout this build does not understand. - * There are no migrations: an earlier layout (v1's different `sessions` shape, - * v2 without the `seed_length`/`source_event_seqs`/`surface_op` columns) is not - * upgraded in place — it is rejected. + * There are no migrations: an earlier layout is not upgraded in place — it is + * rejected. v1 had a different `sessions` shape; v2 lacked all of + * `seed_length`/`source_event_seqs`/`surface_op`. v3 is SKIPPED: two unmerged + * branches each shipped a DISTINCT v3 (one adding only `seed_length`, the other + * adding only the surface columns), so an on-disk v3 is ambiguous — it could be + * either sibling layout, neither of which has all of this build's columns. v4 + * is the merged layout carrying every column; bumping past the collided v3 + * makes the version check reject both sibling v3 databases instead of opening + * one against columns it does not have. */ export function openDatabase(path: string): DatabaseSync { const db = new DatabaseSync(path) diff --git a/packages/session-persistence/session-persistence-sqlite/tests/sqlite.spec.ts b/packages/session-persistence/session-persistence-sqlite/tests/sqlite.spec.ts index 0b8b83cd4e..7138718aa5 100644 --- a/packages/session-persistence/session-persistence-sqlite/tests/sqlite.spec.ts +++ b/packages/session-persistence/session-persistence-sqlite/tests/sqlite.spec.ts @@ -7,7 +7,7 @@ import SessionStore, { SessionId } from '@deepseek-ai/dsh-session' import type { Session, SessionEvent, SurfaceEvent, SurfaceEventType } from '@deepseek-ai/dsh-session' import SessionPersistenceSqlite, { SCHEMA_VERSION } from '@deepseek-ai/dsh-session-persistence-sqlite' import { openDatabase, rowToEvent, scanRows, type EventRow } from '../src/schema.ts' -import { runPersistenceContract, meta, oneTurnLog } from '../../session-persistence/tests/contract.ts' +import { runPersistenceContract, meta, oneTurnLog, appendLog } from '../../session-persistence/tests/contract.ts' import { runCoordinatorContract, type CoordinatorFixture } from '../../session-persistence/tests/coordinator-contract.ts' const dirs: string[] = [] @@ -66,9 +66,17 @@ runCoordinatorContract('sqlite', async (): Promise => { describe('scanRows', () => { // scanRows works off EventRows (data is a JSON string column); build them from - // SessionEvents so the unit tests read in terms of the event vocabulary. + // SessionEvents so the unit tests read in terms of the event vocabulary. Surface + // fields are serialized to their nullable columns so a round trip is faithful. const rows = (events: SessionEvent[]): EventRow[] => - events.map(e => ({ seq: e.seq, type: e.type, time: e.time, data: JSON.stringify(e.data), source_event_seqs: null, surface_op: null })) + events.map((e) => { + const se = e as SessionEvent + return { + seq: e.seq, type: e.type, time: e.time, data: JSON.stringify(e.data), + source_event_seqs: se.sourceEventSeqs !== undefined ? JSON.stringify(se.sourceEventSeqs) : null, + surface_op: se.surfaceOp !== undefined ? JSON.stringify(se.surfaceOp) : null, + } + }) it('preserves the full log when it ends exactly on a turn/end (no torn tail)', () => { const { preserved, tornFrom } = scanRows(rows(oneTurnLog())) @@ -246,6 +254,20 @@ describe('SessionPersistenceSqlite: durability and crash semantics', () => { expect(() => openDatabase(olderPath)).toThrow(/incompatible with this build/) }) + it('rejects a sibling v3 database (the merge-collided version) rather than opening it against missing columns', async () => { + // Two unmerged branches each shipped a DISTINCT layout under user_version 3 + // (one added only `seed_length`, the other only the surface columns). The + // merged build is v4; an on-disk v3 is ambiguous and is missing at least one + // of this build's columns, so it MUST be rejected, not opened. Stamp a v3 + // database and confirm the version check refuses it. + const path = await freshDbPath() + openDatabase(path).close() // creates + stamps user_version = SCHEMA_VERSION (4) + const db = openDatabase(path) + db.exec('PRAGMA user_version = 3') + db.close() + expect(() => openDatabase(path)).toThrow(/schema version 3, incompatible with this build/) + }) + it('a corrupt-JSON row in the uncommitted tail is discarded on load, not unloadable', async () => { const path = await freshDbPath() const m = meta('corrupt-tail') @@ -315,7 +337,7 @@ describe('SessionPersistenceSqlite: durability and crash semantics', () => { }) it('exposes the schema version constant', () => { - expect(SCHEMA_VERSION).toBe(3) + expect(SCHEMA_VERSION).toBe(4) }) }) @@ -353,7 +375,7 @@ describe('SessionPersistenceSqlite: edge cases', () => { // Instance 1 materializes a session and disposes. const b1 = await backend(path) const s1 = b1.ctx.sessions.create(SessionId('hmr-collide')) - for (const e of oneTurnLog()) s1.append(e.type, e.data) + appendLog(s1, oneTurnLog()) await b1.ctx.parallel('session/flush', s1) await b1.dispose() diff --git a/packages/session-persistence/session-persistence/tests/contract.ts b/packages/session-persistence/session-persistence/tests/contract.ts index a0f0e7bfa0..9f2facb827 100644 --- a/packages/session-persistence/session-persistence/tests/contract.ts +++ b/packages/session-persistence/session-persistence/tests/contract.ts @@ -10,7 +10,7 @@ import { describe, expect, it } from 'vitest' import { SESSION_FORMAT_VERSION, SessionId } from '@deepseek-ai/dsh-session' -import type { SessionEvent, SessionHeader } from '@deepseek-ai/dsh-session' +import type { Session, SessionEvent, SessionHeader, SurfaceEventType, SurfaceIntent } from '@deepseek-ai/dsh-session' import { CallId } from '@deepseek-ai/dsh-llm' import type { SessionPersistence } from '../src/index.ts' @@ -34,14 +34,40 @@ export function meta(id: string, cwd?: string): SessionHeader { export function oneTurnLog(): SessionEvent[] { return [ { type: 'turn/start', seq: 0, time: 1, data: { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } } }, - { type: 'user/message', seq: 1, time: 2, data: { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } } }, + { type: 'user/message', seq: 1, time: 2, data: { content: [{ type: 'text', text: 'hi' }], source: { kind: 'user' } }, surfaceOp: 'append' }, { type: 'step/start', seq: 2, time: 3, data: { turn: 1, step: 1 } }, - { type: 'assistant/message', seq: 3, time: 4, data: { turn: 1, step: 1, content: [{ type: 'text', text: 'hello' }] } }, + { type: 'assistant/message', seq: 3, time: 4, data: { turn: 1, step: 1, content: [{ type: 'text', text: 'hello' }] }, surfaceOp: 'append' }, { type: 'step/end', seq: 4, time: 5, data: { turn: 1, step: 1 } }, { type: 'turn/end', seq: 5, time: 6, data: { turn: 1, reason: { kind: 'completed' } } }, ] } +/** + * Append a whole event log to a LIVE session, event by event, forwarding the + * surface metadata each event already carries. A bare `append(e.type, e.data)` + * over a `SessionEvent[]` widens the type argument to the union, where the + * typed overload's mandatory-marker rule collapses to optional — and `append`'s + * runtime guard then rejects a surface-eligible event with no marker. This + * helper forwards the `surfaceOp`/`sourceEventSeqs` VERBATIM from the source + * event (it does not synthesize a default), so a well-formed recorded log + * round-trips through a live session intact and a fixture that forgot a marker + * still trips the guard. + */ +export function appendLog(session: Session, events: readonly SessionEvent[]): void { + for (const e of events) { + const se = e as SessionEvent + if (se.surfaceOp !== undefined) { + const intent: SurfaceIntent = { + surfaceOp: se.surfaceOp, + ...se.sourceEventSeqs !== undefined ? { sourceEventSeqs: se.sourceEventSeqs } : {}, + } + session.append(e.type, e.data, intent) + } else { + session.append(e.type, e.data) + } + } +} + /** * Run the backend-agnostic contract suite. `make()` MUST return a fresh, empty * backend each call. diff --git a/packages/session-persistence/session-persistence/tests/coordinator-contract.ts b/packages/session-persistence/session-persistence/tests/coordinator-contract.ts index a21e5a7183..42583c4fe3 100644 --- a/packages/session-persistence/session-persistence/tests/coordinator-contract.ts +++ b/packages/session-persistence/session-persistence/tests/coordinator-contract.ts @@ -31,7 +31,7 @@ import { Context, type Fiber } from 'cordis' import SessionStore, { SESSION_FORMAT_VERSION, SessionId } from '@deepseek-ai/dsh-session' import type { Session, SessionEvent } from '@deepseek-ai/dsh-session' import type { SessionPersistence } from '../src/index.ts' -import { meta, oneTurnLog } from './contract.ts' +import { meta, oneTurnLog, appendLog } from './contract.ts' /** * The backend-specific capabilities the orchestration suite needs beyond the @@ -76,7 +76,7 @@ function inits(persistence: SessionPersistence): Map> { /** Append a whole event log to a live session, event by event (drives session/event). */ function send(session: Session, events: readonly SessionEvent[]): void { - for (const e of events) session.append(e.type, e.data) + appendLog(session, events) } /** A live session created inside its OWN fiber, so it survives a backend reload. */ diff --git a/packages/support/invariants/tests/invariants.spec.ts b/packages/support/invariants/tests/invariants.spec.ts index 21f6356aee..6ae2b0edf1 100644 --- a/packages/support/invariants/tests/invariants.spec.ts +++ b/packages/support/invariants/tests/invariants.spec.ts @@ -306,7 +306,7 @@ describe('dev-freeze', () => { const { ctx } = await setup() const seed = [ { type: 'turn/start' as const, seq: 0, time: 0, data: { turn: 1, trigger: { kind: 'message' as const, source: { kind: 'user' as const } } } }, - { type: 'user/message' as const, seq: 1, time: 0, data: { content: [{ type: 'text' as const, text: 'seeded' }], source: { kind: 'user' as const } } }, + { type: 'user/message' as const, seq: 1, time: 0, data: { content: [{ type: 'text' as const, text: 'seeded' }], source: { kind: 'user' as const } }, surfaceOp: 'append' as const }, ] const session = ctx.sessions.create(undefined, { seed }) expect(Object.isFrozen(session.events[0])).toBe(true)