From 15640ca697b5bfcee514faad467a1584a6b0cf01 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Mon, 20 Jul 2026 16:53:10 +0800 Subject: [PATCH] fix(goal): require explained model blockers --- ...26-07-19-model-facing-goal-tools.i18n.yaml | 4 +- .../2026-07-19-model-facing-goal-tools.md | 8 +- .../2026-07-19-model-facing-goal-tools.zh.md | 8 +- docs/tool-catalog.md | 8 +- .../tests/fixtures/goal/tool-goal/cordis.yml | 26 ---- .../fixtures/goal/tool-goal/scripted-llm.ts | 88 ------------ .../headless-agent/goal.cordis.snapshot.yml | 13 ++ examples/headless-agent/goal.cordis.yml | 12 ++ .../headless-agent/tests/headless.snapshot.ts | 109 ++++++++++++--- .../tests/snapshots/goal-tools/input.json | 9 ++ .../snapshots/goal-tools/replay.override.json | 33 +++++ .../goal-tools/stream-json.expected.jsonl | 34 +++++ knip.json | 1 - packages/goal/tool-goal/README.md | 8 +- packages/goal/tool-goal/src/index.ts | 34 ++++- .../goal/tool-goal/tests/tool-goal.e2e.ts | 127 ------------------ .../goal/tool-goal/tests/tool-goal.spec.ts | 54 +++++++- 17 files changed, 289 insertions(+), 287 deletions(-) delete mode 100644 examples/echo-agent/tests/fixtures/goal/tool-goal/cordis.yml delete mode 100644 examples/echo-agent/tests/fixtures/goal/tool-goal/scripted-llm.ts create mode 100644 examples/headless-agent/goal.cordis.snapshot.yml create mode 100644 examples/headless-agent/goal.cordis.yml create mode 100644 examples/headless-agent/tests/snapshots/goal-tools/input.json create mode 100644 examples/headless-agent/tests/snapshots/goal-tools/replay.override.json create mode 100644 examples/headless-agent/tests/snapshots/goal-tools/stream-json.expected.jsonl delete mode 100644 packages/goal/tool-goal/tests/tool-goal.e2e.ts diff --git a/.agents/notes/implemented/feature/2026-07-19-model-facing-goal-tools.i18n.yaml b/.agents/notes/implemented/feature/2026-07-19-model-facing-goal-tools.i18n.yaml index f857c5360e..fd97351a0e 100644 --- a/.agents/notes/implemented/feature/2026-07-19-model-facing-goal-tools.i18n.yaml +++ b/.agents/notes/implemented/feature/2026-07-19-model-facing-goal-tools.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -2026-07-19-model-facing-goal-tools.md: e37eaabecfd1984e1198c26a460e78a92375dac1 -2026-07-19-model-facing-goal-tools.zh.md: c0d289271d2a8053193299c16a2bc9477f2f1038 +2026-07-19-model-facing-goal-tools.md: 2ef77b53cd8b95c9cdffd12e20c723fb1cff3a5d +2026-07-19-model-facing-goal-tools.zh.md: 08619600355c71ccd30d608e3c4e5a7fba753d4e diff --git a/.agents/notes/implemented/feature/2026-07-19-model-facing-goal-tools.md b/.agents/notes/implemented/feature/2026-07-19-model-facing-goal-tools.md index e37eaabecf..2ef77b53cd 100644 --- a/.agents/notes/implemented/feature/2026-07-19-model-facing-goal-tools.md +++ b/.agents/notes/implemented/feature/2026-07-19-model-facing-goal-tools.md @@ -16,9 +16,9 @@ The surface also needs to preserve the separation between durable state and live ### Tools and model contract -`get_goal()` returns the current goal or `null`. A non-null result contains the compare-and-set id and revision, objective, durable phase, admitted and maximum goal rounds, plus the process-local activation observation. `create_goal(objective, max_goal_rounds?)` creates one long-running same-session objective. `update_goal(goal_id, revision, action, objective?, max_goal_rounds?)` supports `edit`, `pause`, `resume`, `complete`, and `blocked`; replacement fields are valid only for `edit`. +`get_goal()` returns the current goal or `null`. A non-null result contains the compare-and-set id and revision, objective, durable phase, admitted and maximum goal rounds, any blocker reason, plus the process-local activation observation. `create_goal(objective, max_goal_rounds?)` creates one long-running same-session objective. `update_goal(goal_id, revision, action, objective?, max_goal_rounds?, blocked_reason?)` supports `edit`, `pause`, `resume`, `complete`, and `blocked`; replacement fields are valid only for `edit`, while a non-empty `blocked_reason` is required only for `blocked` and persists under the stable `model-reported` code. -The prompt tells the model that it may infer goal intent from a direct human request in any wording or language, but should not convert routine single-turn work into a goal. It must read the current goal before updating and copy the exact id and revision. On a restored or forked active-but-disarmed goal, a semantic human request to continue is grounds for `resume`. Completion is reserved for an achieved objective, and difficulty or uncertainty alone is not a blocker. +The prompt tells the model that it may infer goal intent from a direct human request in any wording or language, but should not convert routine single-turn work into a goal. It must read the current goal before updating and copy the exact id and revision. On a restored or forked active-but-disarmed goal, a semantic human request to continue is grounds for `resume`. Completion is reserved for an achieved objective, and difficulty or uncertainty alone is not a blocker; a block report must name the concrete condition. All three tools use exclusive execution so a model-ordered batch observes prior mutations and their new revisions. Results are compact JSON. ACP presentation is a pure function of arguments and uses generic read or mutation cards; activation is reported only as live observation and is never written into replay state. @@ -34,11 +34,11 @@ Complete and blocked accept either direct-human authority or the exact current g ### Blocking threshold -`blockedAfterConsecutiveRounds` is a validated positive safe-integer configuration with default `3`. When an autonomous goal round calls `blocked`, the plugin mechanically requires at least that many admitted rounds; the configured value also appears in model guidance. The runtime cannot determine whether those rounds encountered the same blocking condition, so semantic equivalence remains a model judgment. This count is deliberately separate from the goal's generous continuation cap. +`blockedAfterConsecutiveRounds` is a validated positive safe-integer configuration with default `3`. When an autonomous goal round calls `blocked`, the plugin mechanically requires at least that many admitted rounds and a non-empty explanation; the configured value also appears in model guidance. The runtime cannot determine whether those rounds encountered the same blocking condition, so semantic equivalence remains a model judgment. This count is deliberately separate from the goal's generous continuation cap. ## Testing -Unit coverage pins registration and disposal, exclusive scheduling, generated prompt policy, generic presentation, direct-human creation in a non-English turn, exact/stale/non-running agent and driver checks, live-child rejection, resumed-fork root authority, steering, mismatched initiators, read/create/edit/pause/resume behavior, rearming after a session-start edge, authority-before-conditional-argument failures, exact goal-round completion, autonomous-only terminal stopping, the configured blocking threshold, and immediate human blocking. A keyless Loader/stdio process test mounts the real goal, tool, loop, and persistence plugins through `cordis.yml`, drives scripted model tool calls through a human pause and assistant acknowledgment, and reads the JSONL externally to verify the model-visible create/pause snapshots, structured tool results, and configured prompt text. +Unit coverage pins registration and disposal, exclusive scheduling, generated prompt policy, generic presentation, direct-human creation in a non-English turn, exact/stale/non-running agent and driver checks, live-child rejection, resumed-fork root authority, steering, mismatched initiators, read/create/edit/pause/resume behavior, conditional blocker explanations, rearming after a session-start edge, authority-before-conditional-argument failures, exact goal-round completion, autonomous-only terminal stopping, the configured blocking threshold, and immediate human blocking. A keyless replay snapshot mounts the goal domain and tools into the real headless one-shot application, drives `create_goal` and `get_goal` through the shipped loop and persistence stack, pins its stream-json transcript, and inspects the externally persisted goal change. The echo-agent fixture is intentionally not used as an application-UX surrogate. ## Alternatives considered diff --git a/.agents/notes/implemented/feature/2026-07-19-model-facing-goal-tools.zh.md b/.agents/notes/implemented/feature/2026-07-19-model-facing-goal-tools.zh.md index c0d289271d..0861960035 100644 --- a/.agents/notes/implemented/feature/2026-07-19-model-facing-goal-tools.zh.md +++ b/.agents/notes/implemented/feature/2026-07-19-model-facing-goal-tools.zh.md @@ -16,9 +16,9 @@ Status: implemented ### 工具与模型契约 -`get_goal()` 返回当前目标或 `null`。非空结果包含用于比较并交换的 id 与修订号、目标描述、持久阶段、已接纳和最大目标回合数,以及进程本地激活态观察。`create_goal(objective, max_goal_rounds?)` 创建一个长时间运行的同会话目标。`update_goal(goal_id, revision, action, objective?, max_goal_rounds?)` 支持 `edit`、`pause`、`resume`、`complete` 和 `blocked`;替换字段仅对 `edit` 有效。 +`get_goal()` 返回当前目标或 `null`。非空结果包含用于比较并交换的 id 与修订号、目标描述、持久阶段、已接纳和最大目标回合数、可能存在的阻塞原因,以及进程本地激活态观察。`create_goal(objective, max_goal_rounds?)` 创建一个长时间运行的同会话目标。`update_goal(goal_id, revision, action, objective?, max_goal_rounds?, blocked_reason?)` 支持 `edit`、`pause`、`resume`、`complete` 和 `blocked`;替换字段仅对 `edit` 有效,非空的 `blocked_reason` 仅在 `blocked` 时必填,并以稳定代码 `model-reported` 持久化。 -提示词告诉模型:它可以从任何措辞或语言的直接人类请求中推断目标意图,但不应把常规单轮工作转换为目标。更新前必须读取当前目标,并复制准确的 id 和修订号。对于恢复或派生后处于活跃但未激活状态的目标,人类在语义上要求继续即可成为执行 `resume` 的依据。只有目标已经实现时才能标记完成,困难或不确定性本身不构成阻塞。 +提示词告诉模型:它可以从任何措辞或语言的直接人类请求中推断目标意图,但不应把常规单轮工作转换为目标。更新前必须读取当前目标,并复制准确的 id 和修订号。对于恢复或派生后处于活跃但未激活状态的目标,人类在语义上要求继续即可成为执行 `resume` 的依据。只有目标已经实现时才能标记完成,困难或不确定性本身不构成阻塞;阻塞报告必须说明具体条件。 三个工具都采用独占执行,使模型排序的批次可以观察此前变更及其新修订号。结果为紧凑 JSON。ACP 展示是参数的纯函数,使用通用读取或变更卡片;激活态仅作为实时观察返回,绝不会写入回放状态。 @@ -34,11 +34,11 @@ Status: implemented ### 阻塞阈值 -`blockedAfterConsecutiveRounds` 是经过校验的正安全整数配置,默认值为 `3`。自主目标回合调用 `blocked` 时,插件会机械地要求至少已经接纳该数量的回合;配置值也会出现在模型指导中。运行时无法判断这些回合是否遇到了语义上相同的阻塞条件,因此语义等价性仍由模型判断。该计数特意与目标的宽裕继续执行上限分离。 +`blockedAfterConsecutiveRounds` 是经过校验的正安全整数配置,默认值为 `3`。自主目标回合调用 `blocked` 时,插件会机械地要求至少已经接纳该数量的回合并提供非空说明;配置值也会出现在模型指导中。运行时无法判断这些回合是否遇到了语义上相同的阻塞条件,因此语义等价性仍由模型判断。该计数特意与目标的宽裕继续执行上限分离。 ## 测试 -单元测试固定注册与释放、独占调度、生成的提示词策略、通用展示、非英语轮次中的直接人类创建、精确/陈旧/非运行中智能体与驱动检查、实时子智能体拒绝、恢复后派生根的权限、steering、发起者不匹配、读取/创建/编辑/暂停/恢复行为、会话启动边沿后的重新激活、权限先于条件参数失败、准确目标回合的完成、仅自主回合触发终止、可配置阻塞阈值,以及人类立即阻塞。无密钥 Loader/stdio 进程测试通过 `cordis.yml` 挂载真实的目标、工具、循环和持久化插件,驱动脚本化模型工具调用经过人类暂停与智能体确认,并从外部读取 JSONL,以验证模型可见的创建/暂停快照、结构化工具结果和配置后的提示词文本。 +单元测试固定注册与释放、独占调度、生成的提示词策略、通用展示、非英语轮次中的直接人类创建、精确/陈旧/非运行中智能体与驱动检查、实时子智能体拒绝、恢复后派生根的权限、steering、发起者不匹配、读取/创建/编辑/暂停/恢复行为、条件式阻塞说明、会话启动边沿后的重新激活、权限先于条件参数失败、准确目标回合的完成、仅自主回合触发终止、可配置阻塞阈值,以及人类立即阻塞。无密钥回放快照把目标领域和工具挂载到真实的 headless 单次运行应用中,通过随附循环与持久化栈驱动 `create_goal` 和 `get_goal`,固定 stream-json 转录,并检查外部持久化的目标变更。这里有意不把 echo-agent 测试夹具当作应用 UX 的替代品。 ## 考虑过的替代方案 diff --git a/docs/tool-catalog.md b/docs/tool-catalog.md index febcf6a4bb..0244e39629 100644 --- a/docs/tool-catalog.md +++ b/docs/tool-catalog.md @@ -423,7 +423,7 @@ Source: [`packages/goal/tool-goal/src/index.ts`](../packages/goal/tool-goal/src/ ### `get_goal` -Read the current same-session goal, including its exact id/revision, objective, phase, completed continuation rounds, round limit, and whether another continuation is armed. Call this before updating a goal. +Read the current same-session goal, including its exact id/revision, objective, phase, completed continuation rounds, round limit, blocker reason when present, and whether another continuation is armed. Call this before updating a goal. ```json { @@ -436,7 +436,7 @@ Source: [`packages/goal/tool-goal/src/index.ts`](../packages/goal/tool-goal/src/ ### `update_goal` -Update the exact current goal revision. edit, pause, and resume require a direct top-level human request. During an automatic continuation of the current goal, complete and blocked are also allowed. blocked is rejected before the configured minimum round count; the model remains responsible for judging that the same condition persisted across those rounds. +Update the exact current goal revision. edit, pause, and resume require a direct top-level human request. During an automatic continuation of the current goal, complete and blocked are also allowed. blocked is rejected before the configured minimum round count; the model remains responsible for judging that the same condition persisted across those rounds and must explain it in blocked_reason. ```json { @@ -468,6 +468,10 @@ Update the exact current goal revision. edit, pause, and resume require a direct "max_goal_rounds": { "type": "number", "description": "Replacement cap; valid only with action edit." + }, + "blocked_reason": { + "type": "string", + "description": "Concrete blocking condition; required only with action blocked." } }, "required": [ diff --git a/examples/echo-agent/tests/fixtures/goal/tool-goal/cordis.yml b/examples/echo-agent/tests/fixtures/goal/tool-goal/cordis.yml deleted file mode 100644 index fa63160399..0000000000 --- a/examples/echo-agent/tests/fixtures/goal/tool-goal/cordis.yml +++ /dev/null @@ -1,26 +0,0 @@ -# Test-only composition: drive all three goal tools through a real root agent. -- id: scripted-llm - name: './scripted-llm.ts' - -- id: bash - name: '@deepseek-ai/dsh-bash-local' - -- id: goal - name: '@deepseek-ai/dsh-goal' - config: - defaultMaxGoalRounds: 11 - -- id: tool-goal - name: '@deepseek-ai/dsh-tool-goal' - config: - blockedAfterConsecutiveRounds: 3 - -- id: stdio-agent - name: '@deepseek-ai/dsh-stdio-demo' - config: - provider: goal-script - model: goal-script - persona: 'Execute the deterministic goal-tool composition test.' - welcome: 'goal-tools e2e ready.' - persistenceRoot: './.sessions' - workspaceContext: false diff --git a/examples/echo-agent/tests/fixtures/goal/tool-goal/scripted-llm.ts b/examples/echo-agent/tests/fixtures/goal/tool-goal/scripted-llm.ts deleted file mode 100644 index 310737050d..0000000000 --- a/examples/echo-agent/tests/fixtures/goal/tool-goal/scripted-llm.ts +++ /dev/null @@ -1,88 +0,0 @@ -/** Deterministic adapter that creates, reads, pauses, then acknowledges one goal. */ - -import type { Context } from 'cordis' -import { CallId, LlmAdapter } from '@deepseek-ai/dsh-llm' -import type { GenerateOptions, Message, StreamChunk } from '@deepseek-ai/dsh-llm' - -interface GoalState { - readonly id: string - readonly revision: number -} - -/** Text from the latest ordinary user message, excluding raw goal-state context. */ -function latestPrompt(messages: readonly Message[]): { index: number; text: string } { - for (let index = messages.length - 1; index >= 0; index -= 1) { - const message = messages[index] - if (message?.role !== 'user') continue - const text = message.content - .filter(block => block.type === 'text' && !block.text.startsWith('')) - .map(block => block.type === 'text' ? block.text : '') - .join('\n') - if (text.length > 0) return { index, text } - } - return { index: -1, text: '' } -} - -/** Parse the latest domain snapshot rendered into history. */ -function latestGoal(messages: readonly Message[]): GoalState | undefined { - for (const message of [...messages].reverse()) { - for (const block of [...message.content].reverse()) { - if (block.type !== 'text' || !block.text.startsWith('')) continue - const json = block.text.slice(''.length, -''.length) - const value = JSON.parse(json) as { goal?: GoalState } - if (value.goal !== undefined) return value.goal - } - } - return undefined -} - -/** Names of tool calls recorded after the latest ordinary prompt. */ -function callsAfter(messages: readonly Message[], index: number): string[] { - return messages.slice(index + 1).flatMap(message => message.content) - .filter(block => block.type === 'tool-call') - .map(block => block.type === 'tool-call' ? block.name : '') -} - -/** Emit one tool-call response. */ -async function* toolCall(name: string, args: object): AsyncIterable { - const id = CallId(`call-${name}`) - const raw = JSON.stringify(args) - yield { type: 'block-start', index: 0, blockType: 'tool-call' } - yield { type: 'tool-call-delta', index: 0, id, name, argumentsDelta: raw } - yield { type: 'block-end', index: 0, block: { type: 'tool-call', id, name, arguments: raw } } - yield { type: 'finish', reason: { kind: 'tool-calls' } } -} - -/** Emit one terminal text response. */ -async function* textReply(text: string): AsyncIterable { - yield { type: 'block-start', index: 0, blockType: 'text' } - yield { type: 'text-delta', index: 0, text } - yield { type: 'block-end', index: 0, block: { type: 'text', text } } - yield { type: 'finish', reason: { kind: 'stop' } } -} - -class GoalScriptAdapter extends LlmAdapter { - override stream(options: GenerateOptions): AsyncIterable { - const prompt = latestPrompt(options.messages) - const calls = callsAfter(options.messages, prompt.index) - if (prompt.text === 'start' && !calls.includes('create_goal')) { - return toolCall('create_goal', { objective: 'Finish the composed goal-tool proof', max_goal_rounds: 7 }) - } - if (prompt.text === 'start' && !calls.includes('get_goal')) return toolCall('get_goal', {}) - if (prompt.text === 'start') return textReply('GOAL CREATED') - if (prompt.text === 'pause' && !calls.includes('update_goal')) { - const goal = latestGoal(options.messages) - if (goal === undefined) throw new Error('scripted goal state missing') - return toolCall('update_goal', { goal_id: goal.id, revision: goal.revision, action: 'pause' }) - } - if (prompt.text === 'pause') return textReply('GOAL PAUSED') - return textReply('UNEXPECTED PROMPT') - } -} - -export const name = 'goal-tool-scripted-llm' -export const inject = ['llm'] - -export function apply(ctx: Context): void { - ctx.llm.registerAdapter(['goal-script'], new GoalScriptAdapter()) -} diff --git a/examples/headless-agent/goal.cordis.snapshot.yml b/examples/headless-agent/goal.cordis.snapshot.yml new file mode 100644 index 0000000000..185d3dd11e --- /dev/null +++ b/examples/headless-agent/goal.cordis.snapshot.yml @@ -0,0 +1,13 @@ +# Replay counterpart to goal.cordis.yml; only the live model is replaced. +- id: base + name: '@cordisjs/plugin-include' + config: + path: ./goal.cordis.yml + patches: + - id: llm-deepseek + name: '@deepseek-ai/dsh-llm-deepseek' + disabled: true + - insert: + - id: llm-replay + name: '@deepseek-ai/dsh-llm-replay' + diff --git a/examples/headless-agent/goal.cordis.yml b/examples/headless-agent/goal.cordis.yml new file mode 100644 index 0000000000..01f1726100 --- /dev/null +++ b/examples/headless-agent/goal.cordis.yml @@ -0,0 +1,12 @@ +# Add the persisted goal domain and its model-facing tools to the real one-shot app. +- id: base + name: '@cordisjs/plugin-include' + config: + path: ./cordis.yml + patches: + - insert: + - id: goal + name: '@deepseek-ai/dsh-goal' + - id: tool-goal + name: '@deepseek-ai/dsh-tool-goal' + diff --git a/examples/headless-agent/tests/headless.snapshot.ts b/examples/headless-agent/tests/headless.snapshot.ts index dc448b9b23..d799c6250d 100644 --- a/examples/headless-agent/tests/headless.snapshot.ts +++ b/examples/headless-agent/tests/headless.snapshot.ts @@ -11,10 +11,12 @@ import { LOADER_SMOKE_TEST_TIMEOUT_MS, runLoaderSmoke } from '@deepseek-ai/dsh-l import { describe, expect, it } from 'vitest' const snapshotsDir = join(dirname(fileURLToPath(import.meta.url)), 'snapshots') -const scenarioDir = join(snapshotsDir, 'advanced-toolchain') -const sessionFixture = join(scenarioDir, 'session.jsonl') -const streamExpected = join(scenarioDir, 'stream-json.expected.jsonl') -const configPath = fileURLToPath(new URL('../advanced.cordis.snapshot.yml', import.meta.url)) +const advancedScenarioDir = join(snapshotsDir, 'advanced-toolchain') +const advancedSessionFixture = join(advancedScenarioDir, 'session.jsonl') +const advancedStreamExpected = join(advancedScenarioDir, 'stream-json.expected.jsonl') +const advancedConfigPath = fileURLToPath(new URL('../advanced.cordis.snapshot.yml', import.meta.url)) +const goalScenarioDir = join(snapshotsDir, 'goal-tools') +const goalConfigPath = fileURLToPath(new URL('../goal.cordis.snapshot.yml', import.meta.url)) const binScript = fileURLToPath(new URL('../../../packages/examples/cli-demo/src/bin.ts', import.meta.url)) const tsconfigPath = fileURLToPath(new URL('../../../tsconfig.json', import.meta.url)) const refreshing = process.env.DSH_SNAPSHOT === 'refresh' @@ -70,12 +72,36 @@ function normalizeHeadlessStream(rawStdout: string, cwd: string): string { return normalizeStdout(`${normalizedRecords.map(record => JSON.stringify(record)).join('\n')}\n`, context) } -async function advancedPrompt(): Promise { - const input = JSON.parse(await readFile(join(scenarioDir, 'input.json'), 'utf8')) as { +/** Zero durable goal timestamps inside both metadata records and rendered XML JSON. */ +function normalizeGoalTimestamps(value: unknown): unknown { + if (typeof value === 'string') { + return value.replace(/("(?:createdAt|updatedAt|clearedAt)":)\d+/g, '$10') + } + if (Array.isArray(value)) return value.map(normalizeGoalTimestamps) + if (value !== null && typeof value === 'object') { + return Object.fromEntries(Object.entries(value).map(([key, item]) => [ + key, + ['createdAt', 'updatedAt', 'clearedAt'].includes(key) && typeof item === 'number' + ? 0 + : normalizeGoalTimestamps(item), + ])) + } + return value +} + +/** Normalize the stream's durable goal timestamps after the shared scrubbers. */ +function normalizeGoalStream(rawStdout: string, cwd: string): string { + return parseJsonl(normalizeHeadlessStream(rawStdout, cwd)) + .map(record => JSON.stringify(normalizeGoalTimestamps(record))) + .join('\n') + '\n' +} + +async function scenarioPrompt(dir: string, label: string): Promise { + const input = JSON.parse(await readFile(join(dir, 'input.json'), 'utf8')) as { steps?: { op?: unknown; text?: unknown }[] } const prompt = input.steps?.find(step => step.op === 'prompt')?.text - if (typeof prompt !== 'string') throw new Error('advanced-toolchain input has no prompt step') + if (typeof prompt !== 'string') throw new Error(`${label} input has no prompt step`) return prompt } @@ -90,24 +116,27 @@ async function persistedLogs(cwd: string): Promise { describe('headless stream-json snapshots', () => { it('replays the advanced toolchain through the one-shot app', async () => { - const prompt = await advancedPrompt() + const prompt = await scenarioPrompt(advancedScenarioDir, 'advanced-toolchain') const expectedSessions = await Promise.all([ - sessionFixture, - join(scenarioDir, 'session.1.jsonl'), - join(scenarioDir, 'session.2.jsonl'), + advancedSessionFixture, + join(advancedScenarioDir, 'session.1.jsonl'), + join(advancedScenarioDir, 'session.2.jsonl'), ].map(file => readFile(file, 'utf8'))) let runCwd = '' const result = await runLoaderSmoke({ label: 'advanced headless stream-json snapshot', tempDirPrefix: 'headless-snapshot-advanced-', binScript, - configPath, - binArgs: ['--config', configPath, '--output-format', 'stream-json', prompt], + configPath: advancedConfigPath, + binArgs: ['--config', advancedConfigPath, '--output-format', 'stream-json', prompt], tsconfigPath, env: { DSH_SNAPSHOT: 'replay', - DSH_SNAPSHOT_FILE: sessionFixture, - DSH_SNAPSHOT_CHILD_FILES: [join(scenarioDir, 'session.1.jsonl'), join(scenarioDir, 'session.2.jsonl')].join(delimiter), + DSH_SNAPSHOT_FILE: advancedSessionFixture, + DSH_SNAPSHOT_CHILD_FILES: [ + join(advancedScenarioDir, 'session.1.jsonl'), + join(advancedScenarioDir, 'session.2.jsonl'), + ].join(delimiter), NODE_OPTIONS: [process.env.NODE_OPTIONS, '--disable-warning=ExperimentalWarning'].filter(Boolean).join(' '), }, prepare: (cwd) => { runCwd = cwd }, @@ -134,6 +163,56 @@ describe('headless stream-json snapshots', () => { expect(result.stderr).toBe('') const normalized = normalizeHeadlessStream(result.stdout, runCwd) + if (refreshing) await writeFile(advancedStreamExpected, normalized) + expect(normalized).toBe(await readFile(advancedStreamExpected, 'utf8')) + }, LOADER_SMOKE_TEST_TIMEOUT_MS) + + it('replays persisted goal tools through the one-shot app', async () => { + const prompt = await scenarioPrompt(goalScenarioDir, 'goal-tools') + const streamExpected = join(goalScenarioDir, 'stream-json.expected.jsonl') + let runCwd = '' + const result = await runLoaderSmoke({ + label: 'goal tools headless stream-json snapshot', + tempDirPrefix: 'headless-snapshot-goal-tools-', + binScript, + configPath: goalConfigPath, + binArgs: ['--config', goalConfigPath, '--output-format', 'stream-json', prompt], + tsconfigPath, + env: { + DSH_SNAPSHOT: 'replay', + DSH_SNAPSHOT_FILE: join(goalScenarioDir, 'session.jsonl'), + DSH_SNAPSHOT_OVERRIDE: join(goalScenarioDir, 'replay.override.json'), + NODE_OPTIONS: [process.env.NODE_OPTIONS, '--disable-warning=ExperimentalWarning'].filter(Boolean).join(' '), + }, + prepare: (cwd) => { runCwd = cwd }, + inspect: async (cwd) => { + const logs = await persistedLogs(cwd) + expect(logs).toHaveLength(1) + const records = parseJsonl(logs[0]?.content ?? '') + const calls = records.filter(record => record.type === 'tool/call') + .map(record => (record.data as JsonObject | undefined)?.name) + expect(calls).toEqual(['create_goal', 'get_goal']) + const goalChanges = records.filter((record) => { + if (record.type !== 'context/message') return false + const data = record.data as JsonObject | undefined + const meta = data?.meta as JsonObject | undefined + return meta?.kind === 'goal/change' + }) + expect(goalChanges).toHaveLength(1) + const data = goalChanges[0]?.data as JsonObject | undefined + const meta = data?.meta as JsonObject | undefined + const goal = meta?.goal as JsonObject | undefined + expect(meta?.operation).toBe('create') + expect(goal).toMatchObject({ + objective: 'Finish the headless goal-tool snapshot proof', + phase: 'active', + maxGoalRounds: 7, + }) + }, + }) + + expect(result.stderr).toBe('') + const normalized = normalizeGoalStream(result.stdout, runCwd) if (refreshing) await writeFile(streamExpected, normalized) expect(normalized).toBe(await readFile(streamExpected, 'utf8')) }, LOADER_SMOKE_TEST_TIMEOUT_MS) diff --git a/examples/headless-agent/tests/snapshots/goal-tools/input.json b/examples/headless-agent/tests/snapshots/goal-tools/input.json new file mode 100644 index 0000000000..cb0b3bbd82 --- /dev/null +++ b/examples/headless-agent/tests/snapshots/goal-tools/input.json @@ -0,0 +1,9 @@ +{ + "steps": [ + { + "op": "prompt", + "text": "Create a durable goal to finish the snapshot proof, then inspect it." + } + ] +} + diff --git a/examples/headless-agent/tests/snapshots/goal-tools/replay.override.json b/examples/headless-agent/tests/snapshots/goal-tools/replay.override.json new file mode 100644 index 0000000000..7e4fd90c6c --- /dev/null +++ b/examples/headless-agent/tests/snapshots/goal-tools/replay.override.json @@ -0,0 +1,33 @@ +[ + { + "kind": "chunks", + "chunks": [ + { "type": "block-start", "index": 0, "blockType": "tool-call" }, + { "type": "tool-call-delta", "index": 0, "id": "call_goal_create", "name": "create_goal", "argumentsDelta": "{\"objective\":\"Finish the headless goal-tool snapshot proof\",\"max_goal_rounds\":7}" }, + { "type": "block-end", "index": 0, "block": { "type": "tool-call", "id": "call_goal_create", "name": "create_goal", "arguments": "{\"objective\":\"Finish the headless goal-tool snapshot proof\",\"max_goal_rounds\":7}" } }, + { "type": "usage", "usage": { "inputTokens": 20, "outputTokens": 8 } }, + { "type": "finish", "reason": { "kind": "tool-calls" } } + ] + }, + { + "kind": "chunks", + "chunks": [ + { "type": "block-start", "index": 0, "blockType": "tool-call" }, + { "type": "tool-call-delta", "index": 0, "id": "call_goal_get", "name": "get_goal", "argumentsDelta": "{}" }, + { "type": "block-end", "index": 0, "block": { "type": "tool-call", "id": "call_goal_get", "name": "get_goal", "arguments": "{}" } }, + { "type": "usage", "usage": { "inputTokens": 30, "outputTokens": 4 } }, + { "type": "finish", "reason": { "kind": "tool-calls" } } + ] + }, + { + "kind": "chunks", + "chunks": [ + { "type": "block-start", "index": 0, "blockType": "text" }, + { "type": "text-delta", "index": 0, "text": "GOAL READY" }, + { "type": "block-end", "index": 0, "block": { "type": "text", "text": "GOAL READY" } }, + { "type": "usage", "usage": { "inputTokens": 35, "outputTokens": 2 } }, + { "type": "finish", "reason": { "kind": "stop" } } + ] + } +] + diff --git a/examples/headless-agent/tests/snapshots/goal-tools/stream-json.expected.jsonl b/examples/headless-agent/tests/snapshots/goal-tools/stream-json.expected.jsonl new file mode 100644 index 0000000000..b6e9ad7e70 --- /dev/null +++ b/examples/headless-agent/tests/snapshots/goal-tools/stream-json.expected.jsonl @@ -0,0 +1,34 @@ +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"turn/start","seq":0,"time":0,"data":{"turn":1,"trigger":{"kind":"message","source":{"kind":"user"}}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"user/message","seq":1,"time":0,"data":{"content":[{"type":"text","text":"Create a durable goal to finish the snapshot proof, then inspect it."}],"source":{"kind":"user"}},"surfaceOp":"append"}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"step/start","seq":2,"time":0,"data":{"turn":1,"step":1}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"request/header","seq":3,"time":0,"data":{"header":{"config":{"provider":"deepseek","model":"deepseek-v4-flash"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":4,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":5,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":0,"id":"call_goal_create","name":"create_goal","argumentsDelta":"{\"objective\":\"Finish the headless goal-tool snapshot proof\",\"max_goal_rounds\":7}"}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":6,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"call_goal_create","name":"create_goal","arguments":"{\"objective\":\"Finish the headless goal-tool snapshot proof\",\"max_goal_rounds\":7}"}}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":7,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"usage","usage":{"inputTokens":20,"outputTokens":8}}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":8,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/message","seq":9,"time":0,"data":{"turn":1,"step":1,"content":[{"type":"tool-call","id":"call_goal_create","name":"create_goal","arguments":"{\"objective\":\"Finish the headless goal-tool snapshot proof\",\"max_goal_rounds\":7}"}],"provenance":{"provider":"deepseek","model":"deepseek-v4-flash"},"usage":{"inputTokens":20,"outputTokens":8}},"sourceEventSeqs":[4,5,6,7,8],"surfaceOp":"append"}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"tool/call","seq":10,"time":0,"data":{"turn":1,"step":1,"callId":"call_goal_create","name":"create_goal","arguments":"{\"objective\":\"Finish the headless goal-tool snapshot proof\",\"max_goal_rounds\":7}"}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"tool/result","seq":11,"time":0,"data":{"turn":1,"step":1,"callId":"call_goal_create","content":[{"type":"text","text":"{\"goal\":{\"id\":\"goal-{{sessionId}}\",\"revision\":1,\"objective\":\"Finish the headless goal-tool snapshot proof\",\"phase\":\"active\",\"roundsStarted\":0,\"maxGoalRounds\":7},\"activation\":\"armed\"}"}],"isError":false},"sourceEventSeqs":[10],"surfaceOp":"append"}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"context/message","seq":12,"time":0,"data":{"content":[{"type":"text","text":"{\"goal\":{\"id\":\"goal-{{sessionId}}\",\"revision\":1,\"objective\":\"Finish the headless goal-tool snapshot proof\",\"phase\":\"active\",\"maxGoalRounds\":7},\"roundsStarted\":0,\"createdAt\":0,\"updatedAt\":0}"}],"source":{"kind":"goal","goalId":"goal-{{sessionId}}","revision":1,"round":0},"envelope":"raw","meta":{"kind":"goal/change","version":1,"operation":"create","goal":{"id":"goal-{{sessionId}}","revision":1,"objective":"Finish the headless goal-tool snapshot proof","phase":"active","maxGoalRounds":7},"roundsStarted":0,"createdAt":0,"updatedAt":0}},"surfaceOp":"append"}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"step/end","seq":13,"time":0,"data":{"turn":1,"step":1}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"step/start","seq":14,"time":0,"data":{"turn":1,"step":2}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":15,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":16,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"tool-call-delta","index":0,"id":"call_goal_get","name":"get_goal","argumentsDelta":"{}"}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":17,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"call_goal_get","name":"get_goal","arguments":"{}"}}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":18,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"usage","usage":{"inputTokens":30,"outputTokens":4}}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":19,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/message","seq":20,"time":0,"data":{"turn":1,"step":2,"content":[{"type":"tool-call","id":"call_goal_get","name":"get_goal","arguments":"{}"}],"provenance":{"provider":"deepseek","model":"deepseek-v4-flash"},"usage":{"inputTokens":30,"outputTokens":4}},"sourceEventSeqs":[15,16,17,18,19],"surfaceOp":"append"}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"tool/call","seq":21,"time":0,"data":{"turn":1,"step":2,"callId":"call_goal_get","name":"get_goal","arguments":"{}"}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"tool/result","seq":22,"time":0,"data":{"turn":1,"step":2,"callId":"call_goal_get","content":[{"type":"text","text":"{\"goal\":{\"id\":\"goal-{{sessionId}}\",\"revision\":1,\"objective\":\"Finish the headless goal-tool snapshot proof\",\"phase\":\"active\",\"roundsStarted\":0,\"maxGoalRounds\":7},\"activation\":\"armed\"}"}],"isError":false},"sourceEventSeqs":[21],"surfaceOp":"append"}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"step/end","seq":23,"time":0,"data":{"turn":1,"step":2}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"step/start","seq":24,"time":0,"data":{"turn":1,"step":3}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":25,"time":0,"data":{"turn":1,"step":3,"chunk":{"type":"block-start","index":0,"blockType":"text"}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":26,"time":0,"data":{"turn":1,"step":3,"chunk":{"type":"text-delta","index":0,"text":"GOAL READY"}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":27,"time":0,"data":{"turn":1,"step":3,"chunk":{"type":"block-end","index":0,"block":{"type":"text","text":"GOAL READY"}}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":28,"time":0,"data":{"turn":1,"step":3,"chunk":{"type":"usage","usage":{"inputTokens":35,"outputTokens":2}}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/chunk","seq":29,"time":0,"data":{"turn":1,"step":3,"chunk":{"type":"finish","reason":{"kind":"stop"}}}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"assistant/message","seq":30,"time":0,"data":{"turn":1,"step":3,"content":[{"type":"text","text":"GOAL READY"}],"provenance":{"provider":"deepseek","model":"deepseek-v4-flash"},"usage":{"inputTokens":35,"outputTokens":2}},"sourceEventSeqs":[25,26,27,28,29],"surfaceOp":"append"}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"step/end","seq":31,"time":0,"data":{"turn":1,"step":3}}} +{"type":"session_event","sessionId":"{{sessionId}}","event":{"type":"turn/end","seq":32,"time":0,"data":{"turn":1,"reason":{"kind":"completed"}}}} +{"type":"result","success":true,"sessionId":"{{sessionId}}","turn":1,"result":"GOAL READY","reason":{"kind":"completed"},"usage":{"inputTokens":85,"outputTokens":14}} diff --git a/knip.json b/knip.json index eb4eff3a9f..9b7b511c44 100644 --- a/knip.json +++ b/knip.json @@ -11,7 +11,6 @@ "entry": [ "echo-agent/src/*.ts", "echo-agent/tests/fixtures/goal/goal/seed-goal.ts", - "echo-agent/tests/fixtures/goal/tool-goal/scripted-llm.ts", "headless-agent/tests/fixtures/cli-mock-llm.ts", "tui-agent/tests/fixtures/tui-scripted-llm.ts", "*/tests/**/*.e2e.ts", diff --git a/packages/goal/tool-goal/README.md b/packages/goal/tool-goal/README.md index 1ac03ac04a..3b97d7aa82 100644 --- a/packages/goal/tool-goal/README.md +++ b/packages/goal/tool-goal/README.md @@ -4,9 +4,9 @@ The model-facing control surface for [`ctx.goals`](../goal/README.md): `get_goal ## Tools -- `get_goal()` returns the current goal or `null`, including the compare-and-set id/revision, durable phase, admitted/capped goal rounds, and current process-local activation. +- `get_goal()` returns the current goal or `null`, including the compare-and-set id/revision, durable phase, admitted/capped goal rounds, any blocker reason, and current process-local activation. - `create_goal(objective, max_goal_rounds?)` creates one goal from a direct top-level human turn. The model may infer long-running goal intent without an exact command phrase; non-human turns and subagents are rejected at execution. -- `update_goal(goal_id, revision, action, objective?, max_goal_rounds?)` supports `edit`, `pause`, `resume`, `complete`, and `blocked`. Replacements belong only to `edit`. +- `update_goal(goal_id, revision, action, objective?, max_goal_rounds?, blocked_reason?)` supports `edit`, `pause`, `resume`, `complete`, and `blocked`. Replacements belong only to `edit`; `blocked_reason` is required only for `blocked` and is persisted with the stable code `model-reported`. All calls are exclusive, so a model-ordered batch observes earlier mutations and their new revisions. ACP and other clients receive pure generic cards: read for `get_goal`, other for mutations. @@ -18,7 +18,7 @@ Execution requires the exact live `exec.agent`, its inherited `AgentRegistry` in `{ kind: 'user' }` is a host attestation. `Agent.send()` and `steer()` assign it when their caller omits a source, so plugins, schedulers, and other non-human producers must pass their own source rather than inheriting human authority. -Complete and blocked also accept the exact current goal round: a goal-sourced `user/message` whose id, revision, and round equal the folded current goal. A goal-round blocked call is mechanically rejected until `blockedAfterConsecutiveRounds`; the model judges whether the same condition actually persisted. Direct human authority may stop a goal immediately. +Complete and blocked also accept the exact current goal round: a goal-sourced `user/message` whose id, revision, and round equal the folded current goal. A goal-round blocked call is mechanically rejected until `blockedAfterConsecutiveRounds`; the model judges whether the same condition actually persisted and must describe it in `blocked_reason`. Direct human authority may stop a goal immediately. ## Config @@ -42,7 +42,7 @@ A fixed goal policy says when semantic human intent warrants creation, requires ##### Goal policy ```markdown -Use goal tools for one long-running completion objective in the current session. create_goal may infer goal intent from a direct human request in any language; do not create a goal for routine single-turn work. Call get_goal before update_goal and copy its exact goal_id and revision. After session resume or fork, an active goal is disarmed: when a human asks to continue or resume in any wording or language, use update_goal action resume to rearm it. Mark complete only when the objective is actually achieved. Mark blocked only after the same blocking condition persists for at least 3 consecutive goal rounds; difficulty, uncertainty, or useful remaining work is not blocked. +Use goal tools for one long-running completion objective in the current session. create_goal may infer goal intent from a direct human request in any language; do not create a goal for routine single-turn work. Call get_goal before update_goal and copy its exact goal_id and revision. After session resume or fork, an active goal is disarmed: when a human asks to continue or resume in any wording or language, use update_goal action resume to rearm it. Mark complete only when the objective is actually achieved. Mark blocked only after the same blocking condition persists for at least 3 consecutive goal rounds, and report that concrete condition in blocked_reason; difficulty, uncertainty, or useful remaining work is not blocked. ``` #### Token effect diff --git a/packages/goal/tool-goal/src/index.ts b/packages/goal/tool-goal/src/index.ts index cd105b3705..075264f93e 100644 --- a/packages/goal/tool-goal/src/index.ts +++ b/packages/goal/tool-goal/src/index.ts @@ -51,7 +51,8 @@ const CREATE_DESCRIPTION = const GET_DESCRIPTION = 'Read the current same-session goal, including its exact id/revision, objective, phase, completed ' - + 'continuation rounds, round limit, and whether another continuation is armed. Call this before updating a goal.' + + 'continuation rounds, round limit, blocker reason when present, and whether another continuation is armed. ' + + 'Call this before updating a goal.' /** Render policy guidance with its deployment-selected blocked threshold. */ function guidance(blockedAfter: number): string { @@ -62,7 +63,8 @@ function guidance(blockedAfter: number): string { + 'a human asks to continue or resume in any wording or language, use update_goal action ' + 'resume to rearm it. Mark complete only when the objective is actually achieved. Mark ' + `blocked only after the same blocking condition persists for at least ${blockedAfter} ` - + 'consecutive goal rounds; difficulty, uncertainty, or useful remaining work is not blocked.' + + 'consecutive goal rounds, and report that concrete condition in blocked_reason; difficulty, uncertainty, ' + + 'or useful remaining work is not blocked.' } /** Validate config even when apply is called directly outside Loader normalization. */ @@ -97,6 +99,7 @@ function renderGoal(goal: GoalView | undefined): string { phase: goal.phase, roundsStarted: goal.roundsStarted, maxGoalRounds: goal.maxGoalRounds, + ...goal.blockedReason === undefined ? {} : { blockedReason: goal.blockedReason }, }, activation: goal.activation, }) @@ -183,7 +186,7 @@ export function apply(ctx: Context, config: Config): void { description: 'Update the exact current goal revision. edit, pause, and resume require a direct ' + 'top-level human request. During an automatic continuation of the current goal, complete ' + 'and blocked are also allowed. blocked is rejected before the configured minimum round count; the model remains ' - + 'responsible for judging that the same condition persisted across those rounds.', + + 'responsible for judging that the same condition persisted across those rounds and must explain it in blocked_reason.', parameters: { goal_id: { type: 'string', required: true, description: 'Exact id returned by get_goal.' }, revision: { type: 'number', required: true, description: 'Exact positive revision returned by get_goal.' }, @@ -195,6 +198,10 @@ export function apply(ctx: Context, config: Config): void { }, objective: { type: 'string', description: 'Replacement objective; valid only with action edit.' }, max_goal_rounds: { type: 'number', description: 'Replacement cap; valid only with action edit.' }, + blocked_reason: { + type: 'string', + description: 'Concrete blocking condition; required only with action blocked.', + }, }, execute(args, exec) { const execution = goalToolExecution(ctx, exec) @@ -205,6 +212,9 @@ export function apply(ctx: Context, config: Config): void { } if (args.action === 'edit') { requireDirectHuman(ctx, execution) + if (args.blocked_reason !== undefined) { + throw new HarnessError('blocked_reason is valid only with action blocked', 'GOAL_TOOL_INVALID_UPDATE') + } const goal = ctx.goals.edit(execution.agent, ref, replacements) observeMutation(terminalTurns, execution, false) return Promise.resolve([{ @@ -214,9 +224,9 @@ export function apply(ctx: Context, config: Config): void { } if (args.action === 'pause' || args.action === 'resume') { requireDirectHuman(ctx, execution) - if (args.objective !== undefined || args.max_goal_rounds !== undefined) { + if (args.objective !== undefined || args.max_goal_rounds !== undefined || args.blocked_reason !== undefined) { throw new HarnessError( - 'objective and max_goal_rounds are valid only with action edit', + 'objective and max_goal_rounds are valid only with action edit; blocked_reason is valid only with action blocked', 'GOAL_TOOL_INVALID_UPDATE', ) } @@ -233,6 +243,13 @@ export function apply(ctx: Context, config: Config): void { 'GOAL_TOOL_INVALID_UPDATE', ) } + if (args.action === 'complete' && args.blocked_reason !== undefined) { + throw new HarnessError('blocked_reason is valid only with action blocked', 'GOAL_TOOL_INVALID_UPDATE') + } + if (args.action === 'blocked' + && (args.blocked_reason === undefined || args.blocked_reason.trim().length === 0)) { + throw new HarnessError('blocked_reason is required with action blocked', 'GOAL_TOOL_INVALID_UPDATE') + } if (args.action === 'blocked' && authority.kind === 'goal-round' && authority.goal.roundsStarted < resolved.blockedAfterConsecutiveRounds) { throw new HarnessError( @@ -243,14 +260,17 @@ export function apply(ctx: Context, config: Config): void { } const goal = args.action === 'complete' ? ctx.goals.complete(execution.agent, ref) - : ctx.goals.block(execution.agent, ref) + : ctx.goals.block(execution.agent, ref, { + code: 'model-reported', + message: args.blocked_reason as string, + }) observeMutation(terminalTurns, execution, authority.kind === 'goal-round') return Promise.resolve([{ type: 'text', text: renderGoal(goal) }]) }, presentCall: args => present( `${args.action === 'blocked' ? 'Mark' : args.action.charAt(0).toUpperCase() + args.action.slice(1)} goal`, 'other', - args.objective ?? args.goal_id, + args.blocked_reason ?? args.objective ?? args.goal_id, ), })) } diff --git a/packages/goal/tool-goal/tests/tool-goal.e2e.ts b/packages/goal/tool-goal/tests/tool-goal.e2e.ts deleted file mode 100644 index b19a4ee851..0000000000 --- a/packages/goal/tool-goal/tests/tool-goal.e2e.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process' -import { mkdtemp, readFile, readdir, rm } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { fileURLToPath } from 'node:url' -import { afterEach, describe, expect, it } from 'vitest' -import { decodeGoalChange } from '@deepseek-ai/dsh-goal' -import type { SessionEvent } from '@deepseek-ai/dsh-session' -import { resolveExampleLaunch } from '@deepseek-ai/dsh-loader-smoke' - -const binScript = fileURLToPath(new URL('../../../examples/stdio-demo/src/bin.ts', import.meta.url)) -const configPath = fileURLToPath(new URL( - '../../../../examples/echo-agent/tests/fixtures/goal/tool-goal/cordis.yml', - import.meta.url, -)) -const repoTsconfig = fileURLToPath(new URL('../../../../tsconfig.json', import.meta.url)) -const PROCESS_TIMEOUT_MS = 30_000 -const TEST_TIMEOUT_MS = PROCESS_TIMEOUT_MS + 15_000 -const PAUSED_RESULT = '"phase":"paused"' - -let child: ChildProcessWithoutNullStreams | undefined -let workdir: string | undefined - -afterEach(async () => { - if (child !== undefined && child.exitCode === null) child.kill('SIGKILL') - child = undefined - if (workdir !== undefined) await rm(workdir, { recursive: true, force: true }) - workdir = undefined -}) - -async function jsonlFiles(dir: string): Promise { - const entries = await readdir(dir, { withFileTypes: true }) - const paths = await Promise.all(entries.map(async (entry) => { - const path = join(dir, entry.name) - if (entry.isDirectory()) return jsonlFiles(path) - return entry.isFile() && entry.name.endsWith('.jsonl') ? [path] : [] - })) - return paths.flat() -} - -async function runComposition(): Promise<{ stdout: string; stderr: string }> { - workdir = await mkdtemp(join(tmpdir(), 'goal-tools-e2e-')) - const cwd = workdir - return new Promise((resolve, reject) => { - const launch = resolveExampleLaunch({ - srcBin: binScript, - configArgs: [configPath], - tsconfigPath: repoTsconfig, - exposeInternals: true, - env: { - DSH_HOME: join(cwd, '.dsh'), - DSH_AGENTS_HOME: join(cwd, '.agents'), - }, - }) - const proc = spawn(launch.command, launch.args, { - cwd, - env: { ...process.env, ...launch.env }, - stdio: ['pipe', 'pipe', 'pipe'], - }) - child = proc - let stdout = '' - let stderr = '' - let pauseSent = false - let inputClosed = false - proc.stdout.setEncoding('utf8') - proc.stdout.on('data', (chunk: string) => { - stdout += chunk - if (!pauseSent && stdout.includes('GOAL CREATED') && stdout.includes('\n> ')) { - pauseSent = true - proc.stdin.write('pause\n') - } - const pausedAt = stdout.indexOf(PAUSED_RESULT) - if (!inputClosed && pausedAt >= 0 && stdout.indexOf('\n> ', pausedAt) >= 0) { - inputClosed = true - proc.stdin.end() - } - }) - proc.stderr.setEncoding('utf8') - proc.stderr.on('data', (chunk: string) => { stderr += chunk }) - - const timer = setTimeout(() => { - proc.kill('SIGKILL') - reject(new Error( - `goal-tools e2e did not exit within ${PROCESS_TIMEOUT_MS / 1_000}s. stdout:\n${stdout}\nstderr:\n${stderr}`, - )) - }, PROCESS_TIMEOUT_MS) - proc.on('exit', (code) => { - clearTimeout(timer) - if (code === 0) resolve({ stdout, stderr }) - else reject(new Error(`goal-tools e2e exited ${String(code)}. stdout:\n${stdout}\nstderr:\n${stderr}`)) - }) - proc.on('error', (error) => { clearTimeout(timer); reject(error) }) - proc.stdin.write('start\n') - }) -} - -describe('goal tools through a real Loader, app, and stdio process', () => { - it('creates, reads, and pauses one root goal with durable tool and state records', async () => { - const { stdout, stderr } = await runComposition() - expect(stderr).not.toContain('UNHANDLED') - expect(stdout).toContain('goal-tools e2e ready.') - expect(stdout).toContain('GOAL CREATED') - expect(stdout).toContain(PAUSED_RESULT) - expect(stdout).toContain('GOAL PAUSED') - - const logs = await jsonlFiles(join(workdir as string, '.sessions')) - expect(logs).toHaveLength(1) - const lines = (await readFile(logs[0] as string, 'utf8')).trimEnd().split('\n') - const events = lines.slice(1).map(line => JSON.parse(line) as SessionEvent) - const calls = events.filter(event => event.type === 'tool/call') - expect(calls.map(event => event.data.name)).toEqual(['create_goal', 'get_goal', 'update_goal']) - const results = events.filter(event => event.type === 'tool/result') - expect(results).toHaveLength(3) - expect(results.every(event => !event.data.isError)).toBe(true) - - const changes = events - .filter(event => event.type === 'context/message' && event.data.source.kind === 'goal') - .map(event => event.type === 'context/message' ? decodeGoalChange(event.data.meta) : undefined) - expect(changes.map(change => change?.operation)).toEqual(['create', 'pause']) - expect(changes[1]).toMatchObject({ goal: { phase: 'paused', revision: 2, maxGoalRounds: 7 } }) - expect(JSON.stringify(changes)).not.toContain('activation') - - const headers = events.filter(event => event.type === 'request/header') - expect(JSON.stringify(headers)).toContain('infer goal intent') - expect(JSON.stringify(headers)).toContain('at least 3 consecutive goal rounds') - }, TEST_TIMEOUT_MS) -}) diff --git a/packages/goal/tool-goal/tests/tool-goal.spec.ts b/packages/goal/tool-goal/tests/tool-goal.spec.ts index 044fe2369a..cd45c145c7 100644 --- a/packages/goal/tool-goal/tests/tool-goal.spec.ts +++ b/packages/goal/tool-goal/tests/tool-goal.spec.ts @@ -135,8 +135,8 @@ describe('goal tool registration and presentation', () => { card: 'generic', title: 'Create goal', kind: 'other', rawInput: 'ship', }) expect(ctx.tools.get('update_goal')?.presentCall?.({ - goal_id: 'goal-1', revision: 2, action: 'blocked', - })).toEqual({ card: 'generic', title: 'Mark goal', kind: 'other', rawInput: 'goal-1' }) + goal_id: 'goal-1', revision: 2, action: 'blocked', blocked_reason: 'Waiting for a human choice.', + })).toEqual({ card: 'generic', title: 'Mark goal', kind: 'other', rawInput: 'Waiting for a human choice.' }) expect(ctx.tools.get('update_goal')?.presentCall?.({ goal_id: 'goal-1', revision: 2, action: 'resume', })).toEqual({ card: 'generic', title: 'Resume goal', kind: 'other', rawInput: 'goal-1' }) @@ -391,6 +391,26 @@ describe('goal tool state transitions', () => { max_goal_rounds: 2, }, root.agent) expect(terminalUpdate.error?.code).toBe('GOAL_TOOL_INVALID_UPDATE') + const blockedWithoutReason = await execute(ctx, 'update_goal', { + goal_id: created.id, revision: created.revision, action: 'blocked', + }, root.agent) + expect(blockedWithoutReason.error?.code).toBe('GOAL_TOOL_INVALID_UPDATE') + const blockedWithEmptyReason = await execute(ctx, 'update_goal', { + goal_id: created.id, revision: created.revision, action: 'blocked', blocked_reason: ' ', + }, root.agent) + expect(blockedWithEmptyReason.error?.code).toBe('GOAL_TOOL_INVALID_UPDATE') + const completeWithReason = await execute(ctx, 'update_goal', { + goal_id: created.id, revision: created.revision, action: 'complete', blocked_reason: 'Not a blocker.', + }, root.agent) + expect(completeWithReason.error?.code).toBe('GOAL_TOOL_INVALID_UPDATE') + const editWithReason = await execute(ctx, 'update_goal', { + goal_id: created.id, + revision: created.revision, + action: 'edit', + objective: 'still valid', + blocked_reason: 'Not valid for edit.', + }, root.agent) + expect(editWithReason.error?.code).toBe('GOAL_TOOL_INVALID_UPDATE') const malformedRef = await execute(ctx, 'update_goal', { goal_id: '', revision: 0, action: 'edit', objective: 'x', }, root.agent) @@ -423,16 +443,26 @@ describe('goal tool state transitions', () => { for (let round = 1; round <= 2; round += 1) { turn = openTurn(root, { kind: 'goal', goalId: ref.id, revision: ref.revision, round }) const result = await execute(ctx, 'update_goal', { - goal_id: ref.id, revision: ref.revision, action: 'blocked', + goal_id: ref.id, + revision: ref.revision, + action: 'blocked', + blocked_reason: 'The required credential is still unavailable.', }, root.agent) expect(result.error?.code).toBe('GOAL_TOOL_BLOCK_THRESHOLD') closeTurn(root, turn) } openTurn(root, { kind: 'goal', goalId: ref.id, revision: ref.revision, round: 3 }) const blocked = await execute(ctx, 'update_goal', { - goal_id: ref.id, revision: ref.revision, action: 'blocked', + goal_id: ref.id, + revision: ref.revision, + action: 'blocked', + blocked_reason: 'The required credential is still unavailable.', }, root.agent) - expect(resultGoal(blocked)).toMatchObject({ phase: 'blocked', roundsStarted: 3 }) + expect(resultGoal(blocked)).toMatchObject({ + phase: 'blocked', + blockedReason: { code: 'model-reported', message: 'The required credential is still unavailable.' }, + roundsStarted: 3, + }) }) it('lets direct human authority block before the model threshold', async () => { @@ -440,8 +470,18 @@ describe('goal tool state transitions', () => { openTurn(root, { kind: 'user' }) const created = ctx.goals.create(root.agent, { objective: 'human stop' }) const blocked = await execute(ctx, 'update_goal', { - goal_id: created.id, revision: created.revision, action: 'blocked', + goal_id: created.id, + revision: created.revision, + action: 'blocked', + blocked_reason: 'The user asked to stop until a prerequisite is available.', }, root.agent) - expect(resultGoal(blocked)).toMatchObject({ phase: 'blocked', roundsStarted: 0 }) + expect(resultGoal(blocked)).toMatchObject({ + phase: 'blocked', + blockedReason: { + code: 'model-reported', + message: 'The user asked to stop until a prerequisite is available.', + }, + roundsStarted: 0, + }) }) })