fix(gui): harden multimodal image attachments

This commit is contained in:
Yichen Jiang
2026-07-23 19:38:37 +08:00
parent eea595fcb4
commit 580e05b794
61 changed files with 1700 additions and 214 deletions

View File

@@ -13,7 +13,7 @@ This backend owns the compaction policy:
- **Model-free pruning** — after pressure or canonical overflow qualifies, the optional [`ctx.toolResultPrune`](../compact-tool-result-prune/README.md) service rewrites oversized tool results before range selection. Compact-basic remeasures through `ctx.tokenMeter`, skips summarization when pressure becomes safe, and otherwise summarizes the pruned surface. Below-pressure post-step checks never prune.
- **Retention** — compact the oldest whole surface units while preserving a recent tail and balanced tool-call/result cuts through the [`dsh-compact` boundary helpers](../compact/README.md#tool-pairing-boundaries). Turn boundaries do not protect old steps inside a runaway turn. An open indivisible tail declines until it closes. The optional pruner can repair an oversized closed tool unit when its text-bearing result is the removable bulk; indivisible non-tool units and non-prunable tool remainders remain out of scope.
- **Convergence** — retry head-checkpoint compaction up to `compactionRetries`; reject a summary that does not shrink its source, and throw if retries cannot return below threshold.
- **Summarization** — a direct `llm/stream` call uses the configured provider/model pair and cap, falling back to the latest logged request target and then the agent target, without running the loop-only `agent/request` seam. The call replays the conversation's own system prompt, tools, and shadowed-region messages verbatim and appends the compaction instruction as the final user message, so it reuses the provider's warm prefix cache instead of invalidating it. It sets `GenerateOptions.purpose` to `compaction`, which adapters may forward as request attribution (the DeepSeek adapter sends `x-deepseek-harness-compact: 1`) without touching the model-visible body. Only returned text enters the checkpoint, excluding reasoning and tool calls that would leak private reasoning or create an orphaned call.
- **Summarization** — a direct `llm/stream` call uses the configured provider/model pair and cap, falling back to the latest logged request target and then the agent target, without running the loop-only `agent/request` seam. The call replays the conversation's own system prompt, tools, and shadowed-region messages verbatim, including image references, and appends the compaction instruction as the final user message, so it reuses the provider's warm prefix cache instead of invalidating it. The selected adapter must resolve or explicitly reject those images. It sets `GenerateOptions.purpose` to `compaction`, which adapters may forward as request attribution (the DeepSeek adapter sends `x-deepseek-harness-compact: 1`) without touching the model-visible body. Only returned text enters the checkpoint, excluding reasoning and tool calls that would leak private reasoning or create an orphaned call; image output fails with `UNSUPPORTED_CONTENT` rather than disappearing.
- **Framing** — the replacement user message marks established checkpoint context with `<compacted-summary>` tags. The raw summary remains on the provenance event, and later automatic cycles merge the prior checkpoint.
- **Lifecycle** — `compactRegion()` mutates `agent.session` and records its start, summary, replacement, and end. After asynchronous summarization it rejects a changed surface-node snapshot, while unrelated log-only events may append without invalidating the selected span. The serial `agent/post-step` listener checks pressure after successful output and tool work are durable but before `step/end`. Canonical provider overflow is handled through `agent/request-error` after the failed step closes.
- **Overflow recovery** — provider-confirmed overflow needs no capacity metadata: it bypasses normal pressure and retention, prunes, then attempts one maximal balanced head reduction while leaving the newest indivisible unit. Retry is authorized whenever `surface.replaceGeneration` advances, including when pruning lands before later summary work throws. No replacement, an exhausted target-specific cap, cancellation, or an unknown/noncanonical error preserves the original provider failure.
@@ -100,7 +100,7 @@ Replacing rather than append-only. Each checkpoint invalidates reuse from the fi
#### What the model sees
The summarization model receives the conversation replayed verbatim — the same system prompt, tool schemas, and messages the last routed request sent for the shadowed region — followed by one final user message: the compaction instruction below. The conversation model never sees this private request or its reasoning; only returned text is stored.
The summarization model receives the conversation replayed verbatim — the same system prompt, tool schemas, and messages, including image references, that the last routed request sent for the shadowed region — followed by one final user message: the compaction instruction below. The conversation model never sees this private request or its reasoning; only returned text is stored.
##### Compaction instruction (final user message)

View File

@@ -5,7 +5,7 @@
*/
import type { Context } from 'cordis'
import { BlockAssembler } from '@deepseek-ai/dsh-llm'
import { BlockAssembler, LlmError } from '@deepseek-ai/dsh-llm'
import type { ContentBlock, FinishReason, GenerateOptions, Message, ToolSchema } from '@deepseek-ai/dsh-llm'
import type { Agent } from '@deepseek-ai/dsh-agent'
@@ -145,7 +145,7 @@ export async function summarizeWithLlm(
const error = finishError(assembler.finish)
if (error !== undefined) throw error
const summary = textOnly(assembler.message().content)
const summary = summaryText(assembler.message().content)
if (!summary.some(block => block.text.trim().length > 0)) {
throw new Error('summarization produced no text summary content')
}
@@ -189,9 +189,18 @@ function finishError(finish: FinishReason): Error | undefined {
}
}
/** Keep only text blocks before synthesizing a user message. */
function textOnly(
/** Reject visual output and keep only text before synthesizing a user message. */
function summaryText(
blocks: readonly ContentBlock[],
): Array<Extract<ContentBlock, { type: 'text' }>> {
if (containsImage(blocks)) {
throw new LlmError('compaction summary cannot contain image output', 'UNSUPPORTED_CONTENT')
}
return blocks.filter((block): block is Extract<ContentBlock, { type: 'text' }> => block.type === 'text')
}
/** Detect images recursively so no structured result can hide a silent visual drop. */
function containsImage(blocks: readonly ContentBlock[]): boolean {
return blocks.some(block => block.type === 'image'
|| (block.type === 'tool-result' && containsImage(block.content)))
}

View File

@@ -1,5 +1,6 @@
import { describe, expect, it, vi } from 'vitest'
import { Context } from 'cordis'
import { AttachmentId } from '@deepseek-ai/dsh-attachment'
import BasicCompactService from '@deepseek-ai/dsh-compact-basic'
import type { BasicCompactConfig } from '@deepseek-ai/dsh-compact-basic'
import { selectCompactableRange } from '@deepseek-ai/dsh-compact-basic/src/region.ts'
@@ -1103,7 +1104,22 @@ describe('default one-shot summarizer', () => {
it('replays the conversation prefix and appends the instruction as the final message', async () => {
const { adapter, compact } = await summarizerHarness([{ type: 'text', text: 'summary' }])
const tools = [{ name: 'do_thing', description: 'd', parameters: { type: 'object' } }]
const prefix: Message = { role: 'user', content: [{ type: 'text', text: 'earlier turn' }] }
const prefix: Message = {
role: 'user',
content: [
{ type: 'text', text: 'earlier turn' },
{
type: 'image',
attachment: {
attachmentId: AttachmentId(`sha256:${'a'.repeat(64)}`),
mediaType: 'image/png',
bytes: 1,
width: 1,
height: 1,
},
},
],
}
await compact.runSummarize({
system: 'REPLAYED SYSTEM',
tools,
@@ -1256,6 +1272,43 @@ describe('default one-shot summarizer', () => {
await expect(compact.runSummarize(promptInput('history'), agent(conversation(1), MODEL)))
.rejects.toThrow(/no text summary content/)
})
it('rejects image summary output instead of silently dropping it', async () => {
const { compact } = await summarizerHarness([
{
type: 'image',
attachment: {
attachmentId: AttachmentId(`sha256:${'b'.repeat(64)}`),
mediaType: 'image/png',
bytes: 1,
width: 1,
height: 1,
},
},
{ type: 'text', text: 'partial summary' },
])
await expect(compact.runSummarize(promptInput('history'), agent(conversation(1), MODEL)))
.rejects.toMatchObject({ code: 'UNSUPPORTED_CONTENT' })
})
it('rejects image summary output nested in a tool result', async () => {
const { compact } = await summarizerHarness([{
type: 'tool-result',
toolCallId: CallId('summary-tool'),
content: [{
type: 'image',
attachment: {
attachmentId: AttachmentId(`sha256:${'c'.repeat(64)}`),
mediaType: 'image/png',
bytes: 1,
width: 1,
height: 1,
},
}],
}])
await expect(compact.runSummarize(promptInput('history'), agent(conversation(1), MODEL)))
.rejects.toMatchObject({ code: 'UNSUPPORTED_CONTENT' })
})
})
describe('automatic listener and loader composition', () => {