From 2745879132f682b59e61ff59c02b6f184c5a275f Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Sat, 4 Jul 2026 17:21:13 +0800 Subject: [PATCH] refactor(llm): drop the image content block until a path can honor it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ImageBlock had no production producer and every consumer dropped it: the deepseek serializer skipped it, the pi-ai converter skipped it as unrepresentable, the ACP bridge neither advertises image prompt capability nor forwards image blocks, and compact-basic charged a flat 85-token estimate and rendered an [image] placeholder. A block constructed today would silently vanish from the wire — the vocabulary advertised a capability no path honors, the silent-data-loss shape the defensive patterns warn against. The only constructors were tests pinning the skip/estimate branches. Remove ImageBlock and its ContentBlockMap entry (its cache?: CacheHint field leaves with it; CacheHint itself and the other two cache? fields are out of scope). compact-basic loses its explicit image estimate and placeholder arms (the merge-extensible default arms absorb the case); the deepseek serializer, pi-ai converter, and ACP codec already handled image in their default arms, so only their image-naming comments change. The codec's inbound rejection of ACP-protocol image prompt content stays — that guards wire content a client can send regardless of our vocabulary. Tests that constructed harness image blocks to pin the removed branches are dropped (the 85-token estimate pin) or retargeted onto plugin-added block types / other non-text blocks, which the surviving default arms own. Docs, the type-equiv pastes, and the content-block vocabulary RFC's block list and multimodal-home consequence are updated in the same change; the RFC moves to implemented/ and the index is regenerated. A real multimodal feature reintroduces image via declaration merging together with the adapter mapping, ACP advertisement, and compaction pricing that honor it. --- docs/architecture.md | 2 +- docs/core-data-structures/core.md | 3 +- docs/core-data-structures/llm-streaming.md | 1 - docs/rfc/README.md | 2 +- .../2026-06-11-content-block-vocabulary.md | 4 +- .../2026-07-04-drop-image-content-block.md | 27 ++++++++++++ .../2026-07-04-drop-image-content-block.md | 27 ------------ ...-prune-producerless-vocabulary-variants.md | 2 +- packages/bash/tool-bash/tests/tools.spec.ts | 2 +- packages/compact/compact-basic/README.md | 4 +- packages/compact/compact-basic/src/index.ts | 17 ++------ .../compact-basic/tests/compact-basic.spec.ts | 42 +++++++++---------- packages/llm/llm-deepseek/README.md | 1 - packages/llm/llm-deepseek/src/serialize.ts | 1 - .../llm/llm-deepseek/tests/serialize.spec.ts | 14 +++++-- packages/llm/llm-pi-ai/README.md | 2 +- packages/llm/llm-pi-ai/src/convert.ts | 2 +- packages/llm/llm-pi-ai/tests/convert.spec.ts | 6 +-- packages/llm/llm/README.md | 2 +- packages/llm/llm/src/types.ts | 16 ++++--- packages/llm/llm/tests/assembler.spec.ts | 12 +++--- packages/llm/llm/tests/properties.spec.ts | 2 +- packages/ui/acp/src/codec.ts | 6 +-- packages/ui/acp/tests/codec.spec.ts | 5 ++- packages/ui/acp/tests/stream-update.spec.ts | 2 +- 25 files changed, 97 insertions(+), 107 deletions(-) create mode 100644 docs/rfc/implemented/simplification/2026-07-04-drop-image-content-block.md delete mode 100644 docs/rfc/proposed/simplification/2026-07-04-drop-image-content-block.md diff --git a/docs/architecture.md b/docs/architecture.md index 0266f1a2e9..84c113c13b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -93,7 +93,7 @@ The web capability uses the same three-package split but folds two capabilities ## The vocabulary (dsh-llm) -Messages are arrays of typed **content blocks** (`text`, `reasoning`, `tool-call`, `tool-result`, `image`); the union is derived from the merge-extensible `ContentBlockMap`, so plugins can add block types via declaration merging. The same merge-extensible-map pattern is used for `MessageSource`, `FinishReason`, `TurnTrigger`, and `TurnEndReason` — typed sum types instead of strings. +Messages are arrays of typed **content blocks** (`text`, `reasoning`, `tool-call`, `tool-result`); the union is derived from the merge-extensible `ContentBlockMap`, so plugins can add block types via declaration merging. The core set is limited to blocks every shipping path honors — multimodal content (images, audio, …) has no core block type; a feature that needs one adds it via the map in the same coordinated change that maps it in the adapters, surfaces it in the UI bridges, and prices it in compaction ([the drop-image RFC](rfc/implemented/simplification/2026-07-04-drop-image-content-block.md)). The same merge-extensible-map pattern is used for `MessageSource`, `FinishReason`, `TurnTrigger`, and `TurnEndReason` — typed sum types instead of strings. Streaming is a raw chunk protocol (`block-start`, `text-delta`, `reasoning-delta`, `tool-call-delta`, `block-end`, `usage`, `finish`). `BlockAssembler` is the single shared implementation that assembles chunks into blocks/messages; the loop logs raw chunks (replay fidelity) while feeding the same chunks through an assembler. diff --git a/docs/core-data-structures/core.md b/docs/core-data-structures/core.md index 9a282e305c..e2f3cd35f5 100644 --- a/docs/core-data-structures/core.md +++ b/docs/core-data-structures/core.md @@ -87,11 +87,10 @@ interface ContentBlockMap { 'reasoning': ReasoningBlock 'tool-call': ToolCallBlock 'tool-result': ToolResultBlock - 'image': ImageBlock } ``` -The block interfaces (full fields in source): `TextBlock` (`text`), `ReasoningBlock` (thinking, distinct from visible text), `ToolCallBlock` (`id: CallId`, `name`, raw-JSON `arguments`), `ToolResultBlock` (`toolCallId`, nested `content: ContentBlock[]`, `isError?`), `ImageBlock` (`url`, `mimeType?`). `ContentBlock = ContentBlockMap[ContentBlockType]`. +The block interfaces (full fields in source): `TextBlock` (`text`), `ReasoningBlock` (thinking, distinct from visible text), `ToolCallBlock` (`id: CallId`, `name`, raw-JSON `arguments`), `ToolResultBlock` (`toolCallId`, nested `content: ContentBlock[]`, `isError?`). `ContentBlock = ContentBlockMap[ContentBlockType]`. The core set is limited to blocks every shipping path honors — multimodal content (images, audio, …) has no core block type; a feature that needs one adds it via the merge-extensible map together with the adapter/UI/compaction support that honors it. A `Message` is a role plus blocks: diff --git a/docs/core-data-structures/llm-streaming.md b/docs/core-data-structures/llm-streaming.md index 7439eb14a6..997928a3a0 100644 --- a/docs/core-data-structures/llm-streaming.md +++ b/docs/core-data-structures/llm-streaming.md @@ -59,7 +59,6 @@ interface ContentBlockMap { 'reasoning': ReasoningBlock 'tool-call': ToolCallBlock 'tool-result': ToolResultBlock - 'image': ImageBlock } ``` diff --git a/docs/rfc/README.md b/docs/rfc/README.md index d53684bf6b..f0987d2e65 100644 --- a/docs/rfc/README.md +++ b/docs/rfc/README.md @@ -52,7 +52,6 @@ Do NOT write one for a mechanical or local choice (a variable name, a one-file r | Title | First proposed | |---|---| | [Unify the agent id and the session id](proposed/simplification/2026-06-20-unify-agent-and-session-id.md) | 2026-06-20 | -| [Drop the `image` content block until a path can honor it](proposed/simplification/2026-07-04-drop-image-content-block.md) | 2026-07-04 | | [Drop `GenerateOptions.prefill` and `ToolSchema.strict` — request knobs with no working end-to-end path](proposed/simplification/2026-07-04-drop-inert-request-knobs.md) | 2026-07-04 | | [Drop the unconsumed web observation surface — the `providers-change` event and the status methods](proposed/simplification/2026-07-04-drop-unconsumed-web-observation-surface.md) | 2026-07-04 | | [Fold the stdio UI helper into the stdio app](proposed/simplification/2026-07-04-fold-stdio-ui-helper.md) | 2026-07-04 | @@ -120,6 +119,7 @@ Do NOT write one for a mechanical or local choice (a variable name, a one-file r | [Stop mirroring durable boundaries as agent events](implemented/simplification/2026-06-20-remove-agent-boundary-mirror-events.md) | 2026-06-20 | | [Split the filesystem seam — provider text mutations plus the `dsh-fs-policy` plugin](implemented/simplification/2026-06-26-fsspec-style-fs-seam.md) | 2026-06-26 | | [Stop mirroring the token stream as an agent event](implemented/simplification/2026-07-02-remove-stream-chunk-mirror.md) | 2026-07-02 | +| [Drop the `image` content block until a path can honor it](implemented/simplification/2026-07-04-drop-image-content-block.md) | 2026-07-04 | ### Architecture diff --git a/docs/rfc/implemented/architecture/2026-06-11-content-block-vocabulary.md b/docs/rfc/implemented/architecture/2026-06-11-content-block-vocabulary.md index 49d55badac..d2d2162588 100644 --- a/docs/rfc/implemented/architecture/2026-06-11-content-block-vocabulary.md +++ b/docs/rfc/implemented/architecture/2026-06-11-content-block-vocabulary.md @@ -10,12 +10,12 @@ The harness needs one internal language for messages that the loop, session log, ## Decision -Own it: messages are arrays of typed content blocks (`text`, `reasoning`, `tool-call`, `tool-result`, `image`), with the union derived from the merge-extensible `ContentBlockMap` so plugins add block types via declaration merging. The same merge-extensible-map pattern types every "stringly" field (`MessageSource`, `FinishReason`, `TurnTrigger`, `TurnEndReason`). Streaming is a raw chunk protocol; `BlockAssembler` is the single shared assembly implementation. Adapters translate to provider wire formats — mapping cost lives in adapters, where it belongs. +Own it: messages are arrays of typed content blocks (`text`, `reasoning`, `tool-call`, `tool-result`), with the union derived from the merge-extensible `ContentBlockMap` so plugins add block types via declaration merging. The same merge-extensible-map pattern types every "stringly" field (`MessageSource`, `FinishReason`, `TurnTrigger`, `TurnEndReason`). Streaming is a raw chunk protocol; `BlockAssembler` is the single shared assembly implementation. Adapters translate to provider wire formats — mapping cost lives in adapters, where it belongs. In-session context injection (`context/message`, `steering/message`) renders as tagged user-role envelopes (the system-reminder pattern) rather than a new role, so adapters carry zero burden. Live-adapter review has since validated the tagged-envelope rendering against current DeepSeek behavior; a future provider-specific mismatch should be handled in that adapter rather than by adding a new role to the canonical content vocabulary. ## Consequences -- Reasoning, prefill, cache hints, and multimodal content all have a home without provider contortions. +- Reasoning, prefill, and cache hints have a home without provider contortions. Multimodal content deliberately has NO core block type: the core set is limited to blocks every shipping path honors, and a multimodal feature adds its block type through the merge-extensible map in the same coordinated change that maps it in the adapters, surfaces it in the UI bridges, and prices it in compaction — see [the drop-image RFC](../simplification/2026-07-04-drop-image-content-block.md). - Every adapter pays a translation cost; the first real adapters have since validated the streaming protocol, and new adapters should continue proving their provider-specific mapping in adapter-local tests. - IDs that cross package boundaries are branded (`CallId`, `SessionId`, `AgentId`) — nominal typing at zero runtime cost. diff --git a/docs/rfc/implemented/simplification/2026-07-04-drop-image-content-block.md b/docs/rfc/implemented/simplification/2026-07-04-drop-image-content-block.md new file mode 100644 index 0000000000..ccbf5d755d --- /dev/null +++ b/docs/rfc/implemented/simplification/2026-07-04-drop-image-content-block.md @@ -0,0 +1,27 @@ +# RFC: Drop the `image` content block until a path can honor it + +Status: implemented (proposed and accepted 2026-07-04) + +## Problem + +`ImageBlock` (`packages/llm/llm/src/types.ts`) had no production producer, and every consumer on every path DROPPED it: the deepseek adapter's serializer skipped image blocks (a documented MVP limitation), the pi-ai converter skipped them as unrepresentable, the ACP codec neither advertises image prompt capability nor forwarded image blocks outbound and REJECTS image prompt content inbound, and the compaction estimator charged a flat token constant and rendered `[image]`. An `ImageBlock` constructed then would silently vanish from the wire — the vocabulary advertised a capability no path honored, which is the silent-data-loss shape AGENTS.md's defensive patterns warn against. The only constructors anywhere were tests pinning the skip/drop/estimate branches. + +## Decision + +Remove `ImageBlock`, its `ContentBlockMap` entry (and its `cache?: CacheHint` field with it), the explicit `image` estimate/placeholder arms in compact-basic, and the image-naming comments in the deepseek serializer's, pi-ai converter's, and ACP codec's default arms — those default arms absorb the case the way they absorb any unknown block type. Updated in the same change: the vocabulary line in [architecture.md](../../../architecture.md), the block list in `packages/llm/llm/README.md`, the deepseek README's image-skip row, the pi-ai README's images-not-representable row, the compact-basic README's image-estimation and `[image]`-placeholder rows, the pastes in [core.md](../../../core-data-structures/core.md) and [llm-streaming.md](../../../core-data-structures/llm-streaming.md), and the [content-block vocabulary RFC](../architecture/2026-06-11-content-block-vocabulary.md)'s block list and multimodal-home consequence per [implemented/AGENTS.md](../AGENTS.md); the tests that constructed image blocks to exercise the removed branches were dropped (the estimate pin) or retargeted onto the merge-extensible default arms (plugin-added block types). The ACP codec's inbound rejection of image PROMPT content is unaffected — that guard is about protocol content a client can send regardless of our vocabulary, and it stays. + +## Why not keep it? + +This was the most contested cut in the batch. Multimodal input (screenshots) is a plausible near-term coding-agent feature, and the [content-block vocabulary RFC](../architecture/2026-06-11-content-block-vocabulary.md) reserved the slot deliberately. Two responses. First, `ContentBlockMap` is merge-extensible by design: a real multimodal feature reintroduces `image` in core in the same coordinated change that maps it in the adapters, advertises and renders it in ACP, and prices it in compaction — the producer and its consumers arrive together, which is how the map is meant to grow. Second, the middle option — keep the type but make adapters throw UNSUPPORTED instead of silently dropping — converts this into exactly the shape the sibling request-knobs proposal (`2026-07-04-drop-inert-request-knobs`) argues against: surface whose only implementation is rejection. Absence (a compile error at the would-be producer) is strictly clearer than either silent loss or universal throw. + +The recorded fallback, had review landed on keeping the slot: keep `ImageBlock` but replace every silent skip with a loud rejection, and document that policy in the vocabulary — the silent drop was the one state with no defender. Review landed on removal; the fallback stands as the documented alternative should the slot ever return ahead of a full feature. + +## Acceptance criteria + +- No `ImageBlock` / harness `type: 'image'` block construction outside this RFC; the codec's inbound ACP-image rejection still passes its tests. +- Adapter/codec/compaction switches handle the case through their unknown-block default arms (pinned by the plugin-added-block tests). +- Doc pastes, the manifest, and the architecture vocabulary list updated; `pnpm run doc-sync` green. + +## Risks + +Re-adding a core vocabulary type later touches several packages at once — but that coordinated change is the shape a real multimodal feature needs anyway (adapter mapping, ACP advertisement, compaction pricing), and none of it existed to preserve. diff --git a/docs/rfc/proposed/simplification/2026-07-04-drop-image-content-block.md b/docs/rfc/proposed/simplification/2026-07-04-drop-image-content-block.md deleted file mode 100644 index e9144cc3aa..0000000000 --- a/docs/rfc/proposed/simplification/2026-07-04-drop-image-content-block.md +++ /dev/null @@ -1,27 +0,0 @@ -# RFC: Drop the `image` content block until a path can honor it - -Status: proposed - -## Problem - -`ImageBlock` (`packages/llm/llm/src/types.ts`) has no production producer, and every consumer on every path DROPS it: the deepseek adapter's serializer skips image blocks (a documented MVP limitation), the pi-ai converter skips them as unrepresentable, the ACP codec neither advertises image prompt capability nor forwards image blocks outbound and REJECTS image prompt content inbound, and the compaction estimator charges a flat token constant and renders `[image]`. An `ImageBlock` constructed today would silently vanish from the wire — the vocabulary advertises a capability no path honors, which is the silent-data-loss shape AGENTS.md's defensive patterns warn against. The only constructors anywhere are tests pinning the skip/drop/estimate branches. - -## Proposal - -Remove `ImageBlock`, its `ContentBlockMap` entry, the explicit `image` estimate/placeholder arms in compact-basic, and the image-naming comments in the deepseek serializer's, pi-ai converter's, and ACP codec's default arms — those default arms already absorb the case the way they absorb any unknown block type. Update the vocabulary line in [architecture.md](../../../architecture.md), the block list in `packages/llm/llm/README.md`, the deepseek README's image-skip row, the pi-ai README's images-not-representable row, the compact-basic README's image-estimation and `[image]`-placeholder rows, the pastes in [core.md](../../../core-data-structures/core.md) and [llm-streaming.md](../../../core-data-structures/llm-streaming.md), and the type-equiv manifest; amend the [content-block vocabulary RFC](../../implemented/architecture/2026-06-11-content-block-vocabulary.md)'s block list and multimodal-home consequence per [implemented/AGENTS.md](../../implemented/AGENTS.md); drop or retarget the tests that construct image blocks to exercise the removed branches. The ACP codec's inbound rejection of image PROMPT content is unaffected — that guard is about protocol content a client can send regardless of our vocabulary, and it stays. - -## Why not keep it? - -This is the most contested cut in the batch. Multimodal input (screenshots) is a plausible near-term coding-agent feature, and the [content-block vocabulary RFC](../../implemented/architecture/2026-06-11-content-block-vocabulary.md) reserved the slot deliberately. Two responses. First, `ContentBlockMap` is merge-extensible by design: a real multimodal feature reintroduces `image` in core in the same coordinated change that maps it in the adapters, advertises and renders it in ACP, and prices it in compaction — the producer and its consumers arrive together, which is how the map is meant to grow. Second, the middle option — keep the type but make adapters throw UNSUPPORTED instead of silently dropping — converts this into exactly the shape the [request-knobs RFC](2026-07-04-drop-inert-request-knobs.md) argues against: surface whose only implementation is rejection. Absence (a compile error at the would-be producer) is strictly clearer than either silent loss or universal throw. - -If review lands on keeping the slot, the fallback this RFC records is: keep `ImageBlock` but replace every silent skip with a loud rejection, and document that policy in the vocabulary — the current silent drop is the one state with no defender. - -## Acceptance criteria - -- No `ImageBlock` / harness `type: 'image'` block construction outside this RFC; the codec's inbound ACP-image rejection still passes its tests. -- Adapter/codec/compaction switches handle the case through their unknown-block default arms (pinned by the existing plugin-added-block tests where present). -- Doc pastes, the manifest, and the architecture vocabulary list updated; `pnpm run doc-sync` green. - -## Risks - -Re-adding a core vocabulary type later touches several packages at once — but that coordinated change is the shape a real multimodal feature needs anyway (adapter mapping, ACP advertisement, compaction pricing), and none of it exists today to preserve. diff --git a/docs/rfc/proposed/simplification/2026-07-04-prune-producerless-vocabulary-variants.md b/docs/rfc/proposed/simplification/2026-07-04-prune-producerless-vocabulary-variants.md index d967e1d3b2..38565c9214 100644 --- a/docs/rfc/proposed/simplification/2026-07-04-prune-producerless-vocabulary-variants.md +++ b/docs/rfc/proposed/simplification/2026-07-04-prune-producerless-vocabulary-variants.md @@ -28,4 +28,4 @@ The [content-block vocabulary RFC](../../implemented/architecture/2026-06-11-con ## Risks -None operational — nothing can construct these values today. The mirror-event removals (recorded in [the boundary-mirror RFC](../../implemented/simplification/2026-06-20-remove-agent-boundary-mirror-events.md) and [the stream-chunk RFC](../../implemented/simplification/2026-07-02-remove-stream-chunk-mirror.md)) touch only transient `agent/*` events, never the durable vocabulary, so there is no collision. Elsewhere in the vocabulary the admission policy already holds: `rejected`, `prompt/blocked`, and `hook/invoked`/`hook/result` each have live producers — this RFC extends the same bar to the three variants that lack one. If the [image-block RFC](2026-07-04-drop-image-content-block.md) ships first, one of the three `cache?` fields leaves with it; the two proposals are independent and compose in either order. +None operational — nothing can construct these values today. The mirror-event removals (recorded in [the boundary-mirror RFC](../../implemented/simplification/2026-06-20-remove-agent-boundary-mirror-events.md) and [the stream-chunk RFC](../../implemented/simplification/2026-07-02-remove-stream-chunk-mirror.md)) touch only transient `agent/*` events, never the durable vocabulary, so there is no collision. Elsewhere in the vocabulary the admission policy already holds: `rejected`, `prompt/blocked`, and `hook/invoked`/`hook/result` each have live producers — this RFC extends the same bar to the three variants that lack one. If the [image-block RFC](../../implemented/simplification/2026-07-04-drop-image-content-block.md) ships first, one of the three `cache?` fields leaves with it; the two proposals are independent and compose in either order. diff --git a/packages/bash/tool-bash/tests/tools.spec.ts b/packages/bash/tool-bash/tests/tools.spec.ts index 82b69fb133..6d5f78b8d0 100644 --- a/packages/bash/tool-bash/tests/tools.spec.ts +++ b/packages/bash/tool-bash/tests/tools.spec.ts @@ -824,7 +824,7 @@ describe('tool-owned UI presentation (presentCall / presentResult)', () => { const ctx = await setup() const present = ctx.tools.get('bash')!.presentResult!( { command: 'x', description: 'x' }, - { content: [{ type: 'image', url: 'https://x/y.png' }], isError: false }, + { content: [{ type: 'reasoning', text: 'unexpected' }], isError: false }, ) expect(present).toBeUndefined() }) diff --git a/packages/compact/compact-basic/README.md b/packages/compact/compact-basic/README.md index c195ae42fb..00cf72bd16 100644 --- a/packages/compact/compact-basic/README.md +++ b/packages/compact/compact-basic/README.md @@ -8,10 +8,10 @@ This is the implementation tier of the compaction capability — see the [interf The abstract contract states only WHAT compaction does; this backend owns every HOW decision: -- **Token estimation** — `estimateContentTokens()`: char/4 with per-block structural overhead (`text`/`reasoning` = `ceil(len/4) + 4`, `tool-call` from name + arguments, `tool-result` recursive, `image` = 85, unknown blocks via JSON length). +- **Token estimation** — `estimateContentTokens()`: char/4 with per-block structural overhead (`text`/`reasoning` = `ceil(len/4) + 4`, `tool-call` from name + arguments, `tool-result` recursive, unknown blocks via JSON length). - **Retention policy** — `compactIfNeeded()` walks the surface nodes tail→head summing per-node token estimates, and retains the smallest tail-run of WHOLE units (a closed step, or a single no-step node such as a pre-step `user/message` or inter-step `steering/message`) whose total reaches `retainTokens`; everything older is compacted. Retention is **turn-agnostic** — turn boundaries play no role, so a single runaway turn that alone exceeds the window compacts its OWN early closed steps rather than being retained verbatim (the failure mode that motivated dropping turn-protection: a tool-heavy turn must stay compactable or the harness dies exactly when compaction is needed). The only structural guard is **tool-pairing balance**: the compacted region's edges are balanced cuts on the surface (no unanswered tool-call crosses either edge), so it never splits a step's `assistant/message` tool-calls from their `tool/result`s. When the only compactable content left is an un-splittable open tail step, it declines (returns `null`) and retries once an older step closes. **Single-unit overflow is out of scope, by design**: if one retained unit (a single closed step, or a large pasted `user/message`) ALONE exceeds the budget, compaction cannot help and the call may go out over-budget — bounding an individual unit's size is a separate concern. `compactRegion()` enforces tool-pairing balance strictly, throwing on a boundary that would split a step. `dsh-session` exports `isToolPairingBalanced` for the check. - **Dynamic convergence** — no static summary-length config pretends to bound what the model will write. If framing/estimator/system overhead leaves the compacted surface above threshold, `compactIfNeeded()` re-compacts the head checkpoint up to `compactionRetries` extra times; if it still cannot get below threshold, it throws. A summary whose estimated stored size is not smaller than the shadowed content fails closed before it mutates the surface. -- **Summarization** — `summarize()`: a `GenerateOptions` request assembled via `BlockAssembler` with a fixed system prompt that asks for a structured checkpoint (Primary Request and Intent · Key Technical Concepts · Files and Code · Errors and Fixes · Pending Tasks · Current Work · Next Step · Critical Context), every section mandatory, exact paths/commands/identifiers preserved. The request runs through the `agent/request` waterfall before `ctx.llm.stream()`, so router agents that choose the concrete model there also route compaction summaries. `maxTokens` is the provider-side generation cap; only text blocks from the model's reply are kept before the checkpoint is stored (reasoning is dropped so private chain-of-thought never leaks into the durable summary, and a stray `tool-call` is dropped so the synthesized `user/message` summary cannot land an orphaned call with no matching `tool-result`). The compacted region is flattened to a plain-text transcript first: text and reasoning contribute their text, and every non-text block (image, tool-call, tool-result, plugin-added types) contributes a type-tagged placeholder (`[image]`, `[tool-call: name(args)]`, …) so the summarizer is told what existed rather than silently dropping it. +- **Summarization** — `summarize()`: a `GenerateOptions` request assembled via `BlockAssembler` with a fixed system prompt that asks for a structured checkpoint (Primary Request and Intent · Key Technical Concepts · Files and Code · Errors and Fixes · Pending Tasks · Current Work · Next Step · Critical Context), every section mandatory, exact paths/commands/identifiers preserved. The request runs through the `agent/request` waterfall before `ctx.llm.stream()`, so router agents that choose the concrete model there also route compaction summaries. `maxTokens` is the provider-side generation cap; only text blocks from the model's reply are kept before the checkpoint is stored (reasoning is dropped so private chain-of-thought never leaks into the durable summary, and a stray `tool-call` is dropped so the synthesized `user/message` summary cannot land an orphaned call with no matching `tool-result`). The compacted region is flattened to a plain-text transcript first: text and reasoning contribute their text, and every non-text block (tool-call, tool-result, plugin-added types) contributes a type-tagged placeholder (`[tool-call: name(args)]`, `[tool-result: …]`, …) so the summarizer is told what existed rather than silently dropping it. - **Checkpoint framing** — the raw summary is not landed directly. `compactRegion()` wraps it in a checkpoint preamble (so a resuming model reads it as a checkpoint, not a fresh user request, and builds on the captured context rather than restating it) plus `` tags. Because region compaction can be invoked manually, a surface may hold several checkpoints, so the framing does not claim everything after it is recent or verbatim. The tags make a prior checkpoint detectable in the transcript on the next compaction cycle: the summarization prompt then instructs the model to merge it in place (preserve still-true facts, drop stale ones) rather than re-summarize it verbatim — a cheap incremental merge that needs no extra log/event machinery. The unframed summary stays on the `compact/summary` provenance event. - **Surface mutation** — `compactRegion()` appends the `compact/start` → `compact/summary` → `compact/end` log records and the single `user/message` replace node carrying the framed summary (see the interface README). - **Auto-compaction** — an `agent/pre-step` listener delegates to `compactIfNeeded()` before every step (not just a turn's first — a tool-heavy turn grows the surface mid-turn, so a runaway turn still compacts, and per-step firing is the only moment to rescue it before overflow). `agent/pre-step` is a serial (awaited, in-order) surface-mutation checkpoint that fires after `turn/start` and BEFORE the step opens (`step/start`) and its request history is derived, so compaction mutates the surface — with its log-only `compact/*` records landing cleanly outside any step — and the loop derives once from the result: no double-derive, and the listener cannot see (or need to rewrite) an already-assembled `messages` array. The listener owns no threshold logic of its own (the single token-pressure check lives in `compactIfNeeded()`); because Cordis `serial` bails early on non-void return values, the listener returns `void` and does not use the dispatcher's bail channel as a veto surface. diff --git a/packages/compact/compact-basic/src/index.ts b/packages/compact/compact-basic/src/index.ts index f53dace461..2986c3d15d 100644 --- a/packages/compact/compact-basic/src/index.ts +++ b/packages/compact/compact-basic/src/index.ts @@ -44,9 +44,6 @@ export { resolveConfig } from './types.ts' /** Per-block structural overhead for JSON framing / type tag. */ const BLOCK_OVERHEAD = 4 -/** Heuristic token count for an image block (~85 tokens for low-res URL). */ -const IMAGE_TOKEN_COST = 85 - /** Role-field framing overhead added per message in {@link BasicCompactService.estimateTokens}. */ const ROLE_OVERHEAD = 4 @@ -230,9 +227,6 @@ export class BasicCompactService extends CompactService { case 'tool-result': tokens += this.estimateContentTokens(block.content) + BLOCK_OVERHEAD break - case 'image': - tokens += IMAGE_TOKEN_COST - break default: // Unknown block types (merge-extensible ContentBlockMap): // estimate conservatively via JSON stringify. @@ -706,10 +700,10 @@ export class BasicCompactService extends CompactService { /** * Render content blocks to a single plain-text string for the summarization * prompt. Text and reasoning contribute their text; every other block type - * contributes a type-tagged placeholder (`[image]`, `[tool-call: name(args)]`, - * …) so the summarizer is told what non-text content existed in the region - * rather than silently losing it. Blocks join with newlines; empty-text - * blocks contribute nothing. + * contributes a type-tagged placeholder (`[tool-call: name(args)]`, + * `[tool-result: …]`, …) so the summarizer is told what non-text content + * existed in the region rather than silently losing it. Blocks join with + * newlines; empty-text blocks contribute nothing. */ private _blocksToText(blocks: readonly ContentBlock[]): string { const parts: string[] = [] @@ -729,9 +723,6 @@ export class BasicCompactService extends CompactService { parts.push(inner ? `[tool-result: ${inner}]` : '[tool-result]') break } - case 'image': - parts.push('[image]') - break // ContentBlockMap is merge-extensible — render an unknown block as a // bare type-tagged placeholder so a plugin-added block type is still // signalled to the summarizer rather than dropped. diff --git a/packages/compact/compact-basic/tests/compact-basic.spec.ts b/packages/compact/compact-basic/tests/compact-basic.spec.ts index 1929e656be..0e73e1cc1a 100644 --- a/packages/compact/compact-basic/tests/compact-basic.spec.ts +++ b/packages/compact/compact-basic/tests/compact-basic.spec.ts @@ -811,11 +811,6 @@ describe('BasicCompactService token estimation (char/4 heuristic)', () => { ])).toBe(10) }) - it('estimates image blocks at fixed 85 tokens', () => { - const svc = new BasicCompactService(new Context(), cfg({ auto: false })) - expect(svc.estimateContentTokens([{ type: 'image', url: 'https://example.com/img.png' }])).toBe(85) - }) - it('returns 0 for empty content blocks', () => { const svc = new BasicCompactService(new Context(), cfg({ auto: false })) expect(svc.estimateContentTokens([])).toBe(0) @@ -1324,7 +1319,7 @@ describe('BasicCompactService edge cases', () => { s.append('assistant/message', { turn: 1, step: 1, content: [ - { type: 'tool-result', toolCallId: CallId('n1'), content: [{ type: 'image', url: 'https://x/n.png' }] }, + { type: 'tool-result', toolCallId: CallId('n1'), content: [{ type: 'chart', data: 'x' } as unknown as ContentBlock] }, { type: 'custom-widget', payload: 'x' } as unknown as ContentBlock, { type: 'tool-call', id: CallId('b1'), name: 'bash', arguments: '{}' }, ], @@ -1343,7 +1338,7 @@ describe('BasicCompactService edge cases', () => { const nodes = s.surface.nodes await compactRegion(svc, s, nodes[0]!.seq, nodes[nodes.length - 1]!.seq, 'm') const { text } = svc.summarizeCalls[0]! - expect(text).toContain('[tool-result: [image]]') // nested tool-result with content + expect(text).toContain('[tool-result: [chart]]') // nested tool-result with content expect(text).toContain('[custom-widget]') // unknown block placeholder expect(text).toContain('Tool result (call b1): [tool-result]') // empty nested → bare placeholder }) @@ -1510,25 +1505,28 @@ describe('BasicCompactService edge cases', () => { it('renders non-text blocks as type-tagged placeholders across all message kinds', async () => { const svc = createTestService() const s = new Session(SessionId('placeholders')) + // A plugin-added block type (merge-extensible ContentBlockMap) — the + // placeholder path must cover every message kind, not just assistant. + const chart = (id: string): ContentBlock => ({ type: 'chart', data: id } as unknown as ContentBlock) s.append('turn/start', { turn: 1, trigger: { kind: 'message', source: { kind: 'user' } } }) s.append('step/start', { turn: 1, step: 1 }) - // user/message with only an image block → '[image]' placeholder. - s.append('user/message', { content: [{ type: 'image', url: 'https://x/y.png' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) - // assistant/message with an image block AND the tool-call its tool/result - // answers (so the surface is tool-pairing balanced) → '[image]' placeholder. + // user/message with only a plugin-added block → '[chart]' placeholder. + s.append('user/message', { content: [chart('y')], source: { kind: 'user' } }, { surfaceOp: 'append' }) + // assistant/message with a plugin-added block AND the tool-call its + // tool/result answers (so the surface is tool-pairing balanced). s.append('assistant/message', { turn: 1, step: 1, content: [ - { type: 'image', url: 'https://x/z.png' }, + chart('z'), { type: 'tool-call', id: CallId('e1'), name: 'bash', arguments: '{}' }, ], }, { surfaceOp: 'append' }) - // tool/result with an image block → '[image]' placeholder. + // tool/result with a plugin-added block → '[chart]' placeholder. s.append('tool/call', { turn: 1, step: 1, callId: CallId('e1'), name: 'bash', arguments: '{}' }) - s.append('tool/result', { turn: 1, step: 1, callId: CallId('e1'), content: [{ type: 'image', url: 'https://x/r.png' }], isError: false }, { surfaceOp: 'append' }) - // context/message and steering/message with image content. - s.append('context/message', { content: [{ type: 'image', url: 'https://x/c.png' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) - s.append('steering/message', { turn: 1, content: [{ type: 'image', url: 'https://x/s.png' }], source: { kind: 'user' } }, { surfaceOp: 'append' }) + s.append('tool/result', { turn: 1, step: 1, callId: CallId('e1'), content: [chart('r')], isError: false }, { surfaceOp: 'append' }) + // context/message and steering/message with plugin-added content. + s.append('context/message', { content: [chart('c')], source: { kind: 'user' } }, { surfaceOp: 'append' }) + s.append('steering/message', { turn: 1, content: [chart('s')], source: { kind: 'user' } }, { surfaceOp: 'append' }) s.append('step/end', { turn: 1, step: 1 }) s.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) s.append('turn/start', { turn: 2, trigger: { kind: 'message', source: { kind: 'user' } } }) @@ -1537,11 +1535,11 @@ describe('BasicCompactService edge cases', () => { await compactRegion(svc, s, nodes[0]!.seq, nodes[nodes.length - 1]!.seq, 'm') const { text } = svc.summarizeCalls[0]! // Every non-text block surfaces as a placeholder rather than being dropped. - expect(text).toContain('User: [image]') - expect(text).toContain('Assistant: [image]') - expect(text).toContain('Tool result (call e1): [image]') - expect(text).toContain('[Context: [image]]') - expect(text).toContain('[Steering: [image]]') + expect(text).toContain('User: [chart]') + expect(text).toContain('Assistant: [chart]') + expect(text).toContain('Tool result (call e1): [chart]') + expect(text).toContain('[Context: [chart]]') + expect(text).toContain('[Steering: [chart]]') }) }) diff --git a/packages/llm/llm-deepseek/README.md b/packages/llm/llm-deepseek/README.md index 76af182580..66d6ae3e9b 100644 --- a/packages/llm/llm-deepseek/README.md +++ b/packages/llm/llm-deepseek/README.md @@ -34,7 +34,6 @@ A second, independent implementation of the same seam exists in `@deepseek-ai/ds ## Limitations (MVP, documented deliberately) - `prefill` throws `LlmError('UNSUPPORTED')` — DeepSeek's chat-prefix completion is a Beta feature on the `/beta` base URL; future work. -- `image` blocks are skipped (no vision support on these models). - `tool_choice` is not mapped (not part of the core vocabulary). ## Errors diff --git a/packages/llm/llm-deepseek/src/serialize.ts b/packages/llm/llm-deepseek/src/serialize.ts index 4e967d6667..c4e9dcb0a0 100644 --- a/packages/llm/llm-deepseek/src/serialize.ts +++ b/packages/llm/llm-deepseek/src/serialize.ts @@ -11,7 +11,6 @@ * rule for thinking mode — required there, ignored elsewhere, so we save * the tokens elsewhere); `tool-call` → `tool_calls[]` * - `tool-result` → its own `{role: 'tool'}` message (text flattened) - * - `image` → skipped (MVP limitation, documented in the README) * * @module dsh-llm-deepseek/serialize */ diff --git a/packages/llm/llm-deepseek/tests/serialize.spec.ts b/packages/llm/llm-deepseek/tests/serialize.spec.ts index 04387d7aa8..51d4788fe4 100644 --- a/packages/llm/llm-deepseek/tests/serialize.spec.ts +++ b/packages/llm/llm-deepseek/tests/serialize.spec.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' import { CallId, LlmError } from '@deepseek-ai/dsh-llm' -import type { GenerateOptions, Message } from '@deepseek-ai/dsh-llm' +import type { ContentBlock, GenerateOptions, Message } from '@deepseek-ai/dsh-llm' import { serializeMessages, serializeRequest } from '@deepseek-ai/dsh-llm-deepseek' function request(overrides: Partial = {}): GenerateOptions { @@ -110,11 +110,17 @@ describe('serializeMessages', () => { ]) }) - it('skips image blocks (documented MVP limitation)', () => { + it('skips plugin-added block types (merge-extensible ContentBlockMap)', () => { const wire = serializeMessages([ - { role: 'user', content: [{ type: 'image', url: 'data:image/png;base64,x' }, { type: 'text', text: 'see image' }] }, + { + role: 'user', + content: [ + { type: 'chart', data: 'x' } as unknown as ContentBlock, + { type: 'text', text: 'see chart' }, + ], + }, ]) - expect(wire).toEqual([{ role: 'user', content: 'see image' }]) + expect(wire).toEqual([{ role: 'user', content: 'see chart' }]) }) it('emits an empty user message rather than dropping block-less messages', () => { diff --git a/packages/llm/llm-pi-ai/README.md b/packages/llm/llm-pi-ai/README.md index 70f61fdb62..ca34a8f586 100644 --- a/packages/llm/llm-pi-ai/README.md +++ b/packages/llm/llm-pi-ai/README.md @@ -31,7 +31,7 @@ pi-ai declares the openai/anthropic/google/mistral/AWS SDKs as install-time depe ## Limitations -Same MVP contract as llm-deepseek: `prefill` throws `UNSUPPORTED`, images are not representable, `tool_choice` is not mapped. +Same MVP contract as llm-deepseek: `prefill` throws `UNSUPPORTED`, `tool_choice` is not mapped. ## Testing diff --git a/packages/llm/llm-pi-ai/src/convert.ts b/packages/llm/llm-pi-ai/src/convert.ts index 0ddc41386a..6fa96a0597 100644 --- a/packages/llm/llm-pi-ai/src/convert.ts +++ b/packages/llm/llm-pi-ai/src/convert.ts @@ -93,7 +93,7 @@ export function toPiContext(options: GenerateOptions): PiContext { }) break default: - // image / plugin-added block types: not representable here. + // plugin-added block types: not representable here. break } } diff --git a/packages/llm/llm-pi-ai/tests/convert.spec.ts b/packages/llm/llm-pi-ai/tests/convert.spec.ts index be42c9e9b4..078d2a4d3b 100644 --- a/packages/llm/llm-pi-ai/tests/convert.spec.ts +++ b/packages/llm/llm-pi-ai/tests/convert.spec.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' import { CallId } from '@deepseek-ai/dsh-llm' -import type { StreamChunk } from '@deepseek-ai/dsh-llm' +import type { ContentBlock, StreamChunk } from '@deepseek-ai/dsh-llm' import type { AssistantMessage, AssistantMessageEvent, Usage } from '@earendil-works/pi-ai' import { mapStopReason, mapUsage, toPiContext, toStreamChunks } from '@deepseek-ai/dsh-llm-pi-ai' @@ -171,13 +171,13 @@ describe('toPiContext', () => { expect(context.messages.map(message => message.role)).toEqual(['user', 'user', 'toolResult']) }) - it('skips image and unknown blocks in assistant content', () => { + it('skips plugin-added (unknown) blocks in assistant content', () => { const context = toPiContext({ model: 'm', messages: [{ role: 'assistant', content: [ - { type: 'image', url: 'data:,x' }, + { type: 'chart', data: 'x' } as unknown as ContentBlock, { type: 'text', text: 'visible' }, ], }], diff --git a/packages/llm/llm/README.md b/packages/llm/llm/README.md index 4227f5fdef..49a188e523 100644 --- a/packages/llm/llm/README.md +++ b/packages/llm/llm/README.md @@ -25,7 +25,7 @@ An adapter registry plus a single streaming call surface, interceptable via a wa ### Content-block vocabulary (`types.ts`) -Messages are arrays of typed content blocks: `text`, `reasoning`, `tool-call`, `tool-result`, `image`. The union is derived from the merge-extensible `ContentBlockMap`, so plugins can add block types via declaration merging. +Messages are arrays of typed content blocks: `text`, `reasoning`, `tool-call`, `tool-result`. The union is derived from the merge-extensible `ContentBlockMap`, so plugins can add block types via declaration merging. The core set is limited to blocks every shipping path honors — multimodal content (images, audio, …) has no core block type; a feature that needs one adds it via the map together with the adapter/UI/compaction support that honors it. Streaming is a raw chunk protocol (`block-start`, `text-delta`, `reasoning-delta`, `tool-call-delta`, `block-end`, `usage`, `finish`). `BlockAssembler` is the single shared implementation that assembles chunks into blocks/messages. diff --git a/packages/llm/llm/src/types.ts b/packages/llm/llm/src/types.ts index 7b1b9bdc47..fbfc7eb523 100644 --- a/packages/llm/llm/src/types.ts +++ b/packages/llm/llm/src/types.ts @@ -57,24 +57,22 @@ export interface ToolResultBlock { cache?: CacheHint } -/** An image, by URL or data URL. */ -export interface ImageBlock { - type: 'image' - url: string - mimeType?: string - cache?: CacheHint -} - /** * All known content block shapes, keyed by their `type` tag. * Merge-extensible: plugins add new block types via declaration merging. + * + * The core set is deliberately limited to blocks every shipping path honors. + * Multimodal content (images, audio, …) has no core block type: a feature + * that needs one adds it via declaration merging in the same coordinated + * change that maps it in the adapters, surfaces it in the UI bridges, and + * prices it in compaction — a producer never lands without its consumers + * (see docs/rfc/implemented/simplification/2026-07-04-drop-image-content-block.md). */ export interface ContentBlockMap { 'text': TextBlock 'reasoning': ReasoningBlock 'tool-call': ToolCallBlock 'tool-result': ToolResultBlock - 'image': ImageBlock } export type ContentBlockType = keyof ContentBlockMap diff --git a/packages/llm/llm/tests/assembler.spec.ts b/packages/llm/llm/tests/assembler.spec.ts index e8ad04e3b5..d9a4fe33f3 100644 --- a/packages/llm/llm/tests/assembler.spec.ts +++ b/packages/llm/llm/tests/assembler.spec.ts @@ -63,12 +63,12 @@ describe('BlockAssembler', () => { it('throws from assemble() when a partial has an unhandled blockType', () => { const assembler = new BlockAssembler() - // Directly push a block-end for an image block whose block-start never - // called ensure — but the image block-type flows through normally. - // What we really need is a partial whose blockType is not text/reasoning/tool-call. - // We can achieve this via a block-start for 'image' followed by blocks(). - assembler.push({ type: 'block-start', index: 0, blockType: 'image' } as unknown as StreamChunk) - expect(() => assembler.blocks()).toThrow('cannot assemble incomplete block of type "image"') + // A partial whose blockType is not text/reasoning/tool-call cannot be + // assembled without its block-end. A plugin-added block type (here + // 'video', via the merge-extensible ContentBlockMap) opened by a + // block-start with no closing block-end exercises that throw. + assembler.push({ type: 'block-start', index: 0, blockType: 'video' } as unknown as StreamChunk) + expect(() => assembler.blocks()).toThrow('cannot assemble incomplete block of type "video"') }) it('mustGet throws when an index is missing from the partials map (invariant violation)', () => { diff --git a/packages/llm/llm/tests/properties.spec.ts b/packages/llm/llm/tests/properties.spec.ts index c63d56abbb..3bb2a76c38 100644 --- a/packages/llm/llm/tests/properties.spec.ts +++ b/packages/llm/llm/tests/properties.spec.ts @@ -81,7 +81,7 @@ describe('BlockAssembler properties', () => { fc.assert(fc.property(streamArb, (chunks) => { const blocks = feed(chunks).blocks() for (const block of blocks) { - expect(['text', 'reasoning', 'tool-call', 'tool-result', 'image']).toContain(block.type) + expect(['text', 'reasoning', 'tool-call', 'tool-result']).toContain(block.type) } })) }) diff --git a/packages/ui/acp/src/codec.ts b/packages/ui/acp/src/codec.ts index 3b03a81c83..444e71545c 100644 --- a/packages/ui/acp/src/codec.ts +++ b/packages/ui/acp/src/codec.ts @@ -69,8 +69,8 @@ export function turnEndToStopReason(reason: TurnEndReason): StopReason { * client as message content. Today only `text` maps; `resource_link` is an * ACP prompt-only input rendered into text by {@link acpPromptToText}; * `reasoning` is surfaced via `agent_thought_chunk` - * streaming rather than as a message block, and `tool-call`/`tool-result`/ - * `image` are handled by the tool-call update path or not advertised. + * streaming rather than as a message block, and `tool-call`/`tool-result` + * are handled by the tool-call update path. */ export function harnessBlockToAcpContent(block: ContentBlock): AcpContentBlock | undefined { switch (block.type) { @@ -78,7 +78,7 @@ export function harnessBlockToAcpContent(block: ContentBlock): AcpContentBlock | return { type: 'text', text: block.text } // reasoning → streamed as agent_thought_chunk, not a message block // tool-call / tool-result → the tool_call / tool_call_update path - // image → not advertised + // plugin-added block types → not surfaced default: return undefined } diff --git a/packages/ui/acp/tests/codec.spec.ts b/packages/ui/acp/tests/codec.spec.ts index 859e8d40cd..b4f0c10792 100644 --- a/packages/ui/acp/tests/codec.spec.ts +++ b/packages/ui/acp/tests/codec.spec.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from 'vitest' +import type { ContentBlock } from '@deepseek-ai/dsh-llm' import type { TurnEndReason } from '@deepseek-ai/dsh-session' import type { ContentBlock as AcpContentBlock } from '@agentclientprotocol/sdk' import { @@ -33,9 +34,9 @@ describe('harnessBlockToAcpContent', () => { expect(harnessBlockToAcpContent({ type: 'text', text: 'hi' })).toEqual({ type: 'text', text: 'hi' }) }) - it('returns undefined for non-text blocks (reasoning/tool/image)', () => { + it('returns undefined for non-text blocks (reasoning / plugin-added)', () => { expect(harnessBlockToAcpContent({ type: 'reasoning', text: 'think' })).toBeUndefined() - expect(harnessBlockToAcpContent({ type: 'image', url: 'https://x/y.png', mimeType: 'image/png' })).toBeUndefined() + expect(harnessBlockToAcpContent({ type: 'chart', data: 'x' } as unknown as ContentBlock)).toBeUndefined() }) }) diff --git a/packages/ui/acp/tests/stream-update.spec.ts b/packages/ui/acp/tests/stream-update.spec.ts index b580a5fc3d..036bb4f7bb 100644 --- a/packages/ui/acp/tests/stream-update.spec.ts +++ b/packages/ui/acp/tests/stream-update.spec.ts @@ -91,7 +91,7 @@ describe('streamSessionEventUpdate', () => { it('drops non-text tool-result content (text-only)', () => { const update = updatesFor(evt('tool/result', { turn: 1, step: 1, callId: CallId('c1'), - content: [{ type: 'image', url: 'https://x/y.png' }], + content: [{ type: 'reasoning', text: 'private' }], isError: false, }))[0] expect((update as { content: unknown[] }).content).toEqual([])