Files
deepseek-harness/packages/guard/repeat-tool-guard/tests/repeat-tool-guard.spec.ts
creatixchu b7034e4a26 feat(web): declare the remaining context forms on every shipped producer
Four values complete the vocabulary, so the opaque body is reached only by
producers that genuinely promise no shape.

`snapshot` — current state a later snapshot supersedes. system-prompt now
exposes `renderContextSections()`, the named contributions
`renderContextSnapshot()` already joins for the model, so the body attributes
each part to the subsystem that produced it instead of re-splitting joined
prose. The runtime snapshot, time-context, and tmux-context declare it.

`notice` — a one-off account of what just happened, declared by tool-tasks,
goal state changes, tool-goal wrap-up, plan-mode switches, and
repeat-tool-guard. Its `summary` rides the COLLAPSED row: these five are the
majority of shipped producers and none of them needs expanding to be read.
The task summary bounds itself because its inputs are unbounded caller text.

`relay` — a message another agent addressed to this one; both subagent
sources declare it and the body names the sender above what it said.

`recall` — material lifted from another session's log. session-reference
needed no new field: its references already record retained and omitted
counts and the truncation flag, which the body shows first, because recalled
context is bounded on the way in.

`ContextFormed` is now discriminated by `form`, so a producer cannot declare
a shape without the facts that shape is presented from — a notice without its
summary, or a snapshot without its sections, fails to compile.

Only the two hook bridges stay opaque, by design: their content is whatever
an external program printed, so no shape can be promised for it. Unknown
kinds and unreadable records land there too.
2026-08-05 17:11:48 +08:00

404 lines
19 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { describe, expect, it } from 'vitest'
import { Context } from 'cordis'
import { createUserMessage, CallId } from '@deepseek-ai/dsh-llm'
import { SessionId, type SessionEvent } from '@deepseek-ai/dsh-session'
import { defineContentToolFixture } from '@deepseek-ai/dsh-tools'
import type { Agent } from '@deepseek-ai/dsh-agent'
import AgentLoop from '@deepseek-ai/dsh-agent-loop'
import { mountAgentLoopTestDependencies } from '@deepseek-ai/dsh-agent-loop-testkit'
import * as RepeatToolGuard from '@deepseek-ai/dsh-repeat-tool-guard'
import type { Config } from '@deepseek-ai/dsh-repeat-tool-guard'
import { MockAdapter, textResponse, toolCallResponse } from '../../../core/agent-loop/tests/mock-adapter.ts'
const testToolSignal = new AbortController().signal
/**
* Behavior suite for the repeat-tool-call guard: chain semantics (identical /
* different-tracked / untracked-transparent / per-agent / resets), threshold
* escalation incl. the `thresholds[0]` gentle-text rule, canonicalization,
* fold-onto-downstream-decision, and fail-loud config validation — all driven
* through a real agent loop against a scripted mock adapter (no network).
*/
/** Boot the core spine + the guard; the caller registers adapters and extra listeners. */
async function harness(config: Config = {}): Promise<Context> {
const ctx = new Context()
await mountAgentLoopTestDependencies(ctx)
await ctx.plugin(AgentLoop, { agents: [] })
await ctx.plugin(RepeatToolGuard, config)
ctx.tools.register(defineContentToolFixture({ name: 'probe', description: 'p', parameters: {}, async execute() { return [{ type: 'text', text: 'ok' }] } }))
ctx.tools.register(defineContentToolFixture({ name: 'other', description: 'o', parameters: {}, async execute() { return [{ type: 'text', text: 'ok' }] } }))
return ctx
}
function waitForIdle(ctx: Context, agent: Agent): Promise<void> {
return new Promise((resolve) => { const d = ctx.on('agent/status', (s, st) => { if (s === agent && st === 'idle') { d(); resolve() } }) })
}
/** Every injected-context user message in the agent's log, flattened to joined text + source for terse assertions. */
function reminders(agent: Agent): { text: string; source: unknown }[] {
return [...agent.session.events]
.filter((e): e is SessionEvent<'user/message'> => e.type === 'user/message' && e.data.source.kind !== 'user')
.map(e => ({
text: e.data.content.map(block => block.type === 'text' ? block.text : '').join('|'),
source: e.data.source,
}))
}
// The reminder is a `notice`-form context; its summary names the repeated
// call so a reader sees it without expanding the row.
const guardSource = (tool: string, count: number) => ({
kind: 'plugin',
plugin: 'repeat-tool-guard',
form: 'notice',
summary: `${tool} × ${count}`,
})
describe('threshold escalation', () => {
it('reminds gently at the first default threshold (3) and in detail at the second (5)', async () => {
const ctx = await harness()
const adapter = new MockAdapter([
...Array.from({ length: 5 }, (_, i) => toolCallResponse(`c${i}`, 'probe', { q: 'same' })),
textResponse('done'),
])
ctx.llm.registerAdapter(['mock'], adapter)
const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
await waitForIdle(ctx, agent)
const found = reminders(agent)
expect(found).toHaveLength(2)
expect(found[0]!.text).toContain('repeating the exact same tool call')
expect(found[0]!.source).toEqual(guardSource('probe', 3))
expect(found[1]!.text).toContain('consecutive_calls: 5')
expect(found[1]!.text).toContain('- tool: probe')
expect(found[1]!.text).toContain('{"q":"same"}')
expect(found[1]!.source).toEqual(guardSource('probe', 5))
})
it('keys the gentle text to thresholds[0], not the literal 3', async () => {
const ctx = await harness({ thresholds: [4, 2] }) // unsorted on purpose: normalized ascending
const adapter = new MockAdapter([
...Array.from({ length: 4 }, (_, i) => toolCallResponse(`c${i}`, 'probe', {})),
textResponse('done'),
])
ctx.llm.registerAdapter(['mock'], adapter)
const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
await waitForIdle(ctx, agent)
const found = reminders(agent)
expect(found).toHaveLength(2)
expect(found[0]!.text).toContain('repeating the exact same tool call') // gentle at 2
expect(found[1]!.text).toContain('consecutive_calls: 4') // detailed at 4
})
})
describe('chain semantics', () => {
it('caps the detailed reminder arguments at argumentsPreviewChars (detection still keys on the full string)', async () => {
const ctx = await harness({ thresholds: [2, 3], argumentsPreviewChars: 24 })
const bigPayload = 'x'.repeat(400)
const adapter = new MockAdapter([
toolCallResponse('c1', 'probe', { body: bigPayload }),
toolCallResponse('c2', 'probe', { body: bigPayload }),
toolCallResponse('c3', 'probe', { body: bigPayload }),
textResponse('done'),
])
ctx.llm.registerAdapter(['mock'], adapter)
const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
await waitForIdle(ctx, agent)
const found = reminders(agent)
expect(found).toHaveLength(2) // gentle at 2, detailed at 3 — full-key matching survived the cap
const detailed = found[1]!.text
expect(detailed).toContain('- arguments: {"body":"xxxxxxxxxxxxxx') // 24-char head
expect(detailed).toContain('… (+387 more chars)')
expect(detailed).not.toContain(bigPayload)
})
it('a different tracked call resets the chain', async () => {
const ctx = await harness()
const adapter = new MockAdapter([
toolCallResponse('c1', 'probe', { q: 1 }),
toolCallResponse('c2', 'probe', { q: 1 }),
toolCallResponse('c3', 'other', {}), // tracked, different → reset
toolCallResponse('c4', 'probe', { q: 1 }),
toolCallResponse('c5', 'probe', { q: 1 }),
toolCallResponse('c6', 'probe', { q: 1 }), // 3rd consecutive AFTER the reset
textResponse('done'),
])
ctx.llm.registerAdapter(['mock'], adapter)
const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
await waitForIdle(ctx, agent)
expect(reminders(agent)).toHaveLength(1)
})
it('excluded calls are transparent: they neither count nor reset', async () => {
const ctx = await harness({ exclude: ['other'] })
const adapter = new MockAdapter([
toolCallResponse('c1', 'probe', { q: 1 }),
toolCallResponse('c2', 'other', {}), // excluded → invisible to the chain
toolCallResponse('c3', 'probe', { q: 1 }),
toolCallResponse('c4', 'other', {}),
toolCallResponse('c5', 'probe', { q: 1 }), // 3rd consecutive probe
textResponse('done'),
])
ctx.llm.registerAdapter(['mock'], adapter)
const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
await waitForIdle(ctx, agent)
const found = reminders(agent)
expect(found).toHaveLength(1)
expect(found[0]!.text).toContain('repeating the exact same tool call')
})
it('include patterns track only matching tools (wildcard star)', async () => {
const ctx = await harness({ include: ['pro*'] })
const adapter = new MockAdapter([
toolCallResponse('c1', 'other', {}),
toolCallResponse('c2', 'other', {}),
toolCallResponse('c3', 'other', {}), // 3 identical, but untracked
toolCallResponse('c4', 'probe', {}),
toolCallResponse('c5', 'probe', {}),
toolCallResponse('c6', 'probe', {}), // 3 identical, tracked
textResponse('done'),
])
ctx.llm.registerAdapter(['mock'], adapter)
const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
await waitForIdle(ctx, agent)
const found = reminders(agent)
expect(found).toHaveLength(1)
expect(found[0]!.text).toContain('repeating the exact same tool call')
})
it('escapes regex metacharacters in patterns (a dot matches only a literal dot)', async () => {
const ctx = await harness({ exclude: ['pr.be'] }) // would match 'probe' as a regex; must not as a wildcard
const adapter = new MockAdapter([
...Array.from({ length: 3 }, (_, i) => toolCallResponse(`c${i}`, 'probe', {})),
textResponse('done'),
])
ctx.llm.registerAdapter(['mock'], adapter)
const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
await waitForIdle(ctx, agent)
expect(reminders(agent)).toHaveLength(1) // probe was NOT excluded
})
it('canonicalization ignores property order, deeply', async () => {
const ctx = await harness()
const adapter = new MockAdapter([
toolCallResponse('c1', 'probe', { a: 1, nested: { x: [1, 2], y: null } }),
toolCallResponse('c2', 'probe', { nested: { y: null, x: [1, 2] }, a: 1 }),
toolCallResponse('c3', 'probe', { a: 1, nested: { x: [1, 2], y: null } }),
textResponse('done'),
])
ctx.llm.registerAdapter(['mock'], adapter)
const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
await waitForIdle(ctx, agent)
expect(reminders(agent)).toHaveLength(1) // all three canonicalize identically
})
it('keys chains per agent: one agent repeating never trips another', async () => {
const ctx = await harness()
ctx.llm.registerAdapter(['mock-a'], new MockAdapter([
toolCallResponse('a1', 'probe', { q: 1 }),
toolCallResponse('a2', 'probe', { q: 1 }),
textResponse('done'),
]))
ctx.llm.registerAdapter(['mock-b'], new MockAdapter([
toolCallResponse('b1', 'probe', { q: 1 }),
toolCallResponse('b2', 'probe', { q: 1 }),
toolCallResponse('b3', 'probe', { q: 1 }),
textResponse('done'),
]))
const agentA = ctx.agentLoop.create(SessionId('a'), { provider: 'mock-a', model: 'model-a' })
const agentB = ctx.agentLoop.create(SessionId('b'), { provider: 'mock-b', model: 'model-b' })
agentA.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
agentB.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
await Promise.all([waitForIdle(ctx, agentA), waitForIdle(ctx, agentB)])
expect(reminders(agentA)).toHaveLength(0) // 2 repeats < 3, despite B's 3 in the same registry
expect(reminders(agentB)).toHaveLength(1)
})
it('a new user prompt resets the chain', async () => {
const ctx = await harness()
const adapter = new MockAdapter([
toolCallResponse('c1', 'probe', { q: 1 }),
toolCallResponse('c2', 'probe', { q: 1 }),
textResponse('turn one done'),
toolCallResponse('c3', 'probe', { q: 1 }), // without the reset this would be the 3rd
textResponse('turn two done'),
])
ctx.llm.registerAdapter(['mock'], adapter)
const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
await waitForIdle(ctx, agent)
agent.followup(createUserMessage({ content: [{ type: 'text', text: 'again' }], source: { kind: 'user' } }))
await waitForIdle(ctx, agent)
expect(reminders(agent)).toHaveLength(0)
})
it('drops an agent chain on disposal', async () => {
const ctx = await harness({ thresholds: [2] })
ctx.llm.registerAdapter(['mock'], new MockAdapter([
toolCallResponse('c1', 'probe', { q: 1 }),
textResponse('done'),
toolCallResponse('c2', 'probe', { q: 1 }), // same id, fresh agent: count 1, not 2
textResponse('done'),
]))
// Loop agents are torn down by disposing the scope that created them
// (the loop.spec pattern): a child plugin fiber owns `first`.
let first!: Agent
const fiber = await ctx.plugin(Object.assign((inner: Context) => {
first = inner.agentLoop.create(SessionId('reused'), { provider: 'mock', model: 'mock' })
}, { inject: ['agentLoop'] }))
first.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
await waitForIdle(ctx, first)
await fiber.dispose()
await first.whenIdle()
const second = ctx.agentLoop.create(SessionId('reused'), { provider: 'mock', model: 'mock' })
second.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
await waitForIdle(ctx, second)
expect(reminders(second)).toHaveLength(0)
})
it('counts denied calls: hammering a denied tool still draws the reminder', async () => {
const ctx = await harness({ thresholds: [2] })
ctx.on('tools/pre-execute', async () => ({ kind: 'deny' as const, reason: 'sealed' }))
const adapter = new MockAdapter([
toolCallResponse('c1', 'probe', { q: 1 }),
toolCallResponse('c2', 'probe', { q: 1 }),
textResponse('done'),
])
ctx.llm.registerAdapter(['mock'], adapter)
const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
await waitForIdle(ctx, agent)
expect(reminders(agent)).toHaveLength(1)
})
it('ignores direct executes with no agent (they neither crash nor advance any chain)', async () => {
const ctx = await harness({ thresholds: [2] })
const direct = await ctx.tools.execute({ signal: testToolSignal, callId: CallId('d1'), name: 'probe', arguments: { q: 1 } })
expect(direct.isError).toBe(false)
ctx.llm.registerAdapter(['mock'], new MockAdapter([
toolCallResponse('c1', 'probe', { q: 1 }), // if the direct call had counted, this would be #2
textResponse('done'),
]))
const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
await waitForIdle(ctx, agent)
expect(reminders(agent)).toHaveLength(0)
})
})
describe('fold onto the downstream decision', () => {
it('folds the reminder onto a downstream block and keeps its feedback', async () => {
const ctx = await harness({ thresholds: [2] })
ctx.on('tools/post-execute', async () => ({
kind: 'block' as const,
feedback: [{ type: 'text' as const, text: 'nope' }],
additionalContexts: [createUserMessage({
content: [{ type: 'text' as const, text: 'downstream-ctx' }], source: { kind: 'plugin' as const, plugin: 'test' },
})],
}))
const adapter = new MockAdapter([
toolCallResponse('c1', 'probe', { q: 1 }),
toolCallResponse('c2', 'probe', { q: 1 }),
textResponse('done'),
])
ctx.llm.registerAdapter(['mock'], adapter)
const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
await waitForIdle(ctx, agent)
const found = reminders(agent)
expect(found).toHaveLength(3)
// Only the repeated call adds guard context; downstream provenance survives.
expect(found[0]!.text).toBe('downstream-ctx')
expect(found[0]!.source).toEqual({ kind: 'plugin', plugin: 'test' })
expect(found[1]!.text).toContain('repeating the exact same tool call')
expect(found[1]!.source).toEqual(guardSource('probe', 2))
expect(found[2]).toEqual({ text: 'downstream-ctx', source: { kind: 'plugin', plugin: 'test' } })
// The block's feedback reached the tool result unchanged.
const results = [...agent.session.events].filter((e): e is SessionEvent<'tool/result'> => e.type === 'tool/result')
expect(results.every(r => r.data.message.content[0].isError)).toBe(true)
expect(results[1]!.data.message.content[0].content).toEqual([{ type: 'text', text: 'nope' }])
})
it('preserves a downstream canonical value replacement while folding', async () => {
const ctx = await harness({ thresholds: [2] })
ctx.on('tools/post-execute', async () => ({
kind: 'accept' as const,
value: [{ type: 'text' as const, text: 'replaced' }],
}))
const adapter = new MockAdapter([
toolCallResponse('c1', 'probe', { q: 1 }),
toolCallResponse('c2', 'probe', { q: 1 }),
textResponse('done'),
])
ctx.llm.registerAdapter(['mock'], adapter)
const agent = ctx.agentLoop.create(SessionId('a1'), { provider: 'mock', model: 'mock' })
agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } }))
await waitForIdle(ctx, agent)
const found = reminders(agent)
expect(found).toHaveLength(1)
expect(found[0]!.text).toContain('repeating the exact same tool call')
const results = [...agent.session.events].filter((e): e is SessionEvent<'tool/result'> => e.type === 'tool/result')
expect(results[1]!.data.message.content[0].content).toEqual([{ type: 'text', text: 'replaced' }])
})
})
describe('config validation fails loud', () => {
async function spine(): Promise<Context> {
const ctx = new Context()
await mountAgentLoopTestDependencies(ctx)
await ctx.plugin(AgentLoop, { agents: [] })
return ctx
}
it('rejects an empty thresholds list', async () => {
const ctx = await spine()
await expect(ctx.plugin(RepeatToolGuard, { thresholds: [] })).rejects.toThrow(/must not be empty/)
})
it('rejects a threshold below 2', async () => {
const ctx = await spine()
await expect(ctx.plugin(RepeatToolGuard, { thresholds: [1, 3] })).rejects.toThrow(/integer >= 2/)
})
it('rejects a non-integer threshold', async () => {
const ctx = await spine()
await expect(ctx.plugin(RepeatToolGuard, { thresholds: [2.5] })).rejects.toThrow(/integer >= 2/)
})
it('rejects duplicate thresholds', async () => {
const ctx = await spine()
await expect(ctx.plugin(RepeatToolGuard, { thresholds: [3, 3] })).rejects.toThrow(/duplicates/)
})
it('rejects a non-positive or fractional argumentsPreviewChars', async () => {
const ctx = await spine()
await expect(ctx.plugin(RepeatToolGuard, { argumentsPreviewChars: 0 })).rejects.toThrow(/argumentsPreviewChars/)
const ctx2 = await spine()
await expect(ctx2.plugin(RepeatToolGuard, { argumentsPreviewChars: 12.5 })).rejects.toThrow(/argumentsPreviewChars/)
})
})