mirror of
https://github.com/deepseek-ai/deepseek-harness
synced 2026-08-15 21:04:50 +00:00
test(todo): align real-model e2e prompt and scenario with parallel todos
The key-gated todo_write e2e still pinned the previous contract: its TODO_SYSTEM_PROMPT instructed at most one in_progress task and the scenario never exercised parallel active tasks. Update the prompt to the new guidance and make the scenario record two simultaneously in_progress tasks; verified against the real API.
This commit is contained in:
@@ -28,8 +28,9 @@ export const SYSTEM_PROMPT = 'You are a coding agent. Use bash for file operatio
|
||||
/** System prompt for the todo_write e2e: nudges the model to plan with the tool. */
|
||||
export const TODO_SYSTEM_PROMPT = 'You are a coding agent. For multi-step work, '
|
||||
+ 'use the todo_write tool to track a task list: send the WHOLE list each call, '
|
||||
+ 'keep at most one task in_progress (exactly one while work remains), and mark '
|
||||
+ 'a task completed as soon as it is done.'
|
||||
+ 'mark every task being actively worked on in_progress (several at once when '
|
||||
+ 'work runs in parallel, at least one while work remains), and mark a task '
|
||||
+ 'completed as soon as it is done.'
|
||||
|
||||
/** Options for {@link codingHarness}. */
|
||||
export interface CodingHarnessOptions {
|
||||
|
||||
@@ -29,9 +29,10 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY)('todo_write: real model records a
|
||||
const agent = ctx.agentLoop.create(SessionId('e2e-todo'), { provider: 'deepseek', model: 'deepseek-v4-flash' })
|
||||
|
||||
agent.followup([{ type: 'text', text:
|
||||
'Use the todo_write tool to record a plan of exactly two steps: first '
|
||||
+ '"inspect the failing test" (in_progress), then "apply the fix" (pending). '
|
||||
+ 'Send both in one todo_write call, then reply with the single word DONE.' }])
|
||||
'Use the todo_write tool to record a plan of exactly three steps for work '
|
||||
+ 'running in parallel: "inspect the failing test" (in_progress), '
|
||||
+ '"watch the background build" (in_progress), then "apply the fix" (pending). '
|
||||
+ 'Send all three in one todo_write call, then reply with the single word DONE.' }])
|
||||
await waitForIdle(ctx, agent)
|
||||
|
||||
const events = [...agent.session.events]
|
||||
@@ -40,13 +41,15 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY)('todo_write: real model records a
|
||||
const calls = events.filter(event => event.type === 'tool/call')
|
||||
expect(calls.some(event => event.data.name === 'todo_write')).toBe(true)
|
||||
|
||||
// And the tool wrote a todo/write event to the log — verify the WORLD.
|
||||
// And the tool wrote a todo/write event to the log — verify the WORLD,
|
||||
// including two simultaneously in_progress tasks (the parallel contract).
|
||||
const todoEvents = events.filter(event => event.type === 'todo/write')
|
||||
expect(todoEvents.length).toBeGreaterThan(0)
|
||||
|
||||
const todos = (todoEvents.at(-1)!).data.todos
|
||||
expect(todos).toEqual([
|
||||
{ content: 'inspect the failing test', status: 'in_progress' },
|
||||
{ content: 'watch the background build', status: 'in_progress' },
|
||||
{ content: 'apply the fix', status: 'pending' },
|
||||
])
|
||||
}, 120_000)
|
||||
|
||||
Reference in New Issue
Block a user