test(todo): add the todo-plan ACP snapshot scenario and a with-key e2e

Record the `todo-plan` snapshot scenario: a real prompt drives the model to call
todo_write, and the golden captures the resulting `plan` sessionUpdate (three
entries, priority synthesized as medium, status 1:1) plus the persisted
todo/write event. Registered in SCENARIOS; replays deterministically keyless.

Add a with-key coding-agent e2e that verifies the WORLD — a real model call to
todo_write lands a todo/write event whose snapshot is a valid, one-in-progress
list — not the agent's self-report. Wire tool-todo into the e2e harness.
This commit is contained in:
Tianyi Cui
2026-06-29 10:56:55 +08:00
parent e17c4b748d
commit 93a6dc6716
7 changed files with 249 additions and 3 deletions

View File

@@ -8,19 +8,26 @@ import AgentRegistry from '@deepseek-ai/dsh-agent'
import AgentLoop, { ReactLoopAgent } from '@deepseek-ai/dsh-agent-loop'
import { LocalBashExecutor } from '@deepseek-ai/dsh-bash-local'
import * as ToolBash from '@deepseek-ai/dsh-tool-bash'
import * as ToolTodo from '@deepseek-ai/dsh-tool-todo'
import * as LlmDeepSeek from '@deepseek-ai/dsh-llm-deepseek'
import SessionPersistenceJsonl from '@deepseek-ai/dsh-session-persistence-jsonl'
/**
* Shared harness for the coding-agent e2e suites: the full plugin stack
* with the real DeepSeek adapter and the real bash tool. Lives outside the
* *.e2e.ts pattern so importing it never re-registers another file's tests.
* with the real DeepSeek adapter and the real bash + todo_write tools. Lives
* outside the *.e2e.ts pattern so importing it never re-registers another
* file's tests.
*/
export const SYSTEM_PROMPT = 'You are a coding agent. Your only tool is bash; '
+ 'do file operations with cat/grep/heredocs, check [exit code: N] markers, '
+ 'and report results briefly.'
/** System prompt for the todo_write e2e: nudges the model to plan with the tool. */
export const TODO_SYSTEM_PROMPT = 'You are a coding agent. For multi-step work, '
+ 'use the todo_write tool to track a task list: send the WHOLE list each call, '
+ 'keep exactly one task in_progress, and mark a task completed as soon as it is done.'
export async function codingHarness(workdir: string, persistenceRoot?: string): Promise<Context> {
const ctx = new Context()
await ctx.plugin(LlmService)
@@ -32,6 +39,7 @@ export async function codingHarness(workdir: string, persistenceRoot?: string):
await ctx.plugin(LlmDeepSeek, { models: ['deepseek-v4-flash'] })
await ctx.plugin(LocalBashExecutor, { cwd: workdir, timeoutMs: 30_000 })
await ctx.plugin(ToolBash)
await ctx.plugin(ToolTodo)
// Durable JSONL persistence is opt-in: only the resume e2e needs it, and the
// other suites stay file-free. Loaded last so a resume's deferred
// `ctx.inject(['sessionPersistence'])` resolves once this is present.

View File

@@ -0,0 +1,55 @@
import { afterEach, describe, expect, it } from 'vitest'
import type { Context } from 'cordis'
import { AgentId } from '@deepseek-ai/dsh-agent'
import type { TodoItem } from '@deepseek-ai/dsh-session'
import { codingHarness, TODO_SYSTEM_PROMPT, waitForIdle } from './harness.ts'
/**
* A REAL model drives the REAL todo_write tool: verify the WORLD (the session
* log gains a todo/write event whose snapshot the model actually produced), not
* the agent's self-report. Key-gated (see vitest.e2e.config.ts).
*/
let ctx: Context | undefined
afterEach(async () => {
await ctx?.fiber.dispose()
ctx = undefined
})
describe.skipIf(!process.env.DEEPSEEK_API_KEY)('todo_write: real model records a plan', () => {
it('appends a todo/write event with the model-produced task list', async () => {
ctx = await codingHarness(process.cwd())
const agent = ctx.agentLoop.create(AgentId('e2e-todo'), {
model: 'deepseek-v4-flash',
systemPrompt: TODO_SYSTEM_PROMPT,
})
agent.send([{ type: 'text', text:
'Use the todo_write tool to record a plan of exactly two steps: first '
+ '"inspect the failing test" (in_progress), then "apply the fix" (pending). '
+ 'Send both in one todo_write call, then reply with the single word DONE.' }])
await waitForIdle(ctx, agent)
const events = [...agent.session.events]
// The model actually called the tool.
const calls = events.filter(event => event.type === 'tool/call')
expect(calls.some(event => event.data.name === 'todo_write')).toBe(true)
// And the tool wrote a todo/write event to the log — verify the WORLD.
const todoEvents = events.filter(event => event.type === 'todo/write')
expect(todoEvents.length).toBeGreaterThan(0)
const todos = (todoEvents.at(-1)!).data.todos
expect(todos.length).toBeGreaterThanOrEqual(2)
// Every entry has a non-empty content and a valid status…
const valid: TodoItem['status'][] = ['pending', 'in_progress', 'completed']
for (const todo of todos) {
expect(todo.content.trim().length).toBeGreaterThan(0)
expect(valid).toContain(todo.status)
}
// …and the one-in-progress invariant the tool enforces held.
expect(todos.filter(t => t.status === 'in_progress').length).toBeLessThanOrEqual(1)
}, 120_000)
})