diff --git a/.agents/notes/implemented/feature/2026-07-10-parallel-tool-call-execution.i18n.yaml b/.agents/notes/implemented/feature/2026-07-10-parallel-tool-call-execution.i18n.yaml index 0c7363827e..28ca5e0ae4 100644 --- a/.agents/notes/implemented/feature/2026-07-10-parallel-tool-call-execution.i18n.yaml +++ b/.agents/notes/implemented/feature/2026-07-10-parallel-tool-call-execution.i18n.yaml @@ -1,6 +1,6 @@ # Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: -# pnpm run verify-translation-pairing --write -2026-07-10-parallel-tool-call-execution.md: c67ae61939a3e7974f9bf729058a57f5576308a1 -2026-07-10-parallel-tool-call-execution.zh.md: a80317aa951cbf3a9cae0651348c99712a4193d5 +# pnpm run verify-translation-pairing --write .agents/notes/implemented/feature/2026-07-10-parallel-tool-call-execution.md +2026-07-10-parallel-tool-call-execution.md: 19f5dc189821433052edfa72613980a2e94e2cae +2026-07-10-parallel-tool-call-execution.zh.md: e90e357180ae3684500adf7cba41c5fc1dac5743 diff --git a/.agents/notes/implemented/feature/2026-07-10-parallel-tool-call-execution.md b/.agents/notes/implemented/feature/2026-07-10-parallel-tool-call-execution.md index c67ae61939..19f5dc1898 100644 --- a/.agents/notes/implemented/feature/2026-07-10-parallel-tool-call-execution.md +++ b/.agents/notes/implemented/feature/2026-07-10-parallel-tool-call-execution.md @@ -10,7 +10,7 @@ An assistant message may contain several sibling `tool-call` blocks. Running the Concurrency is a host scheduling concern, not model-facing tool metadata. The loop needs to decide which calls may overlap without hardcoding tool names or exposing scheduler policy in the JSON schema. -The session log remains authoritative: every started call has an audit event, every started call receives a result, and model history observes results in the original call order regardless of completion order. +The session log remains authoritative: every started call has an audit event, ordinary completion and cancellation pair calls with results, and model history observes committed results in the original call order regardless of completion order. ## Decision @@ -46,7 +46,7 @@ Only dispatch and the tool body overlap. `tools/pre-execute` and `tools/post-exe Each started call appends `tool/call` immediately before its pre-execute gate. Completed dispatches occupy model-order slots, and a commit cursor appends `tool/result` and collects `additionalContexts` only when the next slot is ready. Live surfaces may show several pending calls, but results and post-tool context remain model-ordered. -An abort before a group starts records no calls from that group. An abort during a group stops replenishment, waits for already-started calls, commits their results in order, drains accepted batch context after those results, and then ends the step through the existing abort path. Calls that never start have no audit event. +An abort before a group starts records no calls from that group. An abort during a group stops replenishment, waits for already-started calls, commits their results in order, drains accepted batch context after those results, and then ends the step through the existing abort path. Calls that never start have no audit event. An unexpected scheduler failure stops new dispatches, waits for every already-started dispatch to settle, and rethrows the first failure. Because that failure is terminal internal state rather than a tool outcome, the loop does not invent tool results for rejected or uncommitted calls. Code Mode remains outside this scheduler because the model emits one native `run_code` call. `run_code` and its internal dispatch queue remain serial; native sibling calls in `mode: 'both'` use the normal scheduler. @@ -66,7 +66,7 @@ Filesystem read relies on a narrow recorder exception: its synchronous observati ## Verification -Unit coverage pins fail-closed classification, typed argument validation, grouping, barriers, live reclassification after registry replacement, the rolling cap, distinct execution objects, middleware order, ordered results and context, and abort draining. First-party tests pin each parallel declaration. +Unit coverage pins fail-closed classification, typed argument validation, grouping, barriers, live reclassification after registry replacement, the rolling cap, distinct execution objects, middleware order, ordered results and context, abort draining, and scheduler-failure quiescence. First-party tests pin each parallel declaration. Snapshot coverage pins the visible multi-call transcript: pending calls may overlap while completed results remain model-ordered. Code Mode coverage pins its serial boundary. No provider-backed e2e is required because scheduling is deterministic loop behavior. @@ -84,6 +84,8 @@ Snapshot coverage pins the visible multi-call transcript: pending calls may over **Expose staged methods or a scheduling waterfall.** Public `prepare` / `dispatch` / `finalize` methods or a `tools/execution-mode` event add extension surface before another consumer needs it. The loop uses an internal scheduler view, while `executionMode(exec)` leaves an insertion point for a policy seam. +**Convert scheduler failures into tool results.** AgentLoop cannot determine whether a rejected dispatch invoked the tool body; ToolRegistry owns body-invocation state and typed tool outcomes. Internal scheduler failures therefore remain terminal instead of being reclassified as `ABORTED` results. + **Start calls while the model streams.** This may reduce latency further but changes assistant-message authority, replay, and call/result pairing. The scheduler starts only after the assistant message is complete. **Use fixed-size windows.** Waiting for every call in one window before starting the next leaves capacity idle behind a slow call. The rolling pool preserves the cap without that delay. @@ -101,3 +103,5 @@ Ordered commits may hold a fast result behind a slow earlier sibling. This prese Concurrent external calls can compete for quota or process capacity. Providers own their capacity controls; the loop cap only limits calls from one agent step. Tool registration is a scheduling boundary. Registry mutations affect not-yet-started calls because the scheduler reclassifies after each barrier and before every pool replenishment. Already-started calls retain the scheduling decision under which they entered the pool. + +A terminal scheduler failure may leave recorded calls without results before the failed step closes. Waiting for live dispatches preserves quiescence without misreporting those internal failures as tool outcomes. diff --git a/.agents/notes/implemented/feature/2026-07-10-parallel-tool-call-execution.zh.md b/.agents/notes/implemented/feature/2026-07-10-parallel-tool-call-execution.zh.md index a80317aa95..e90e357180 100644 --- a/.agents/notes/implemented/feature/2026-07-10-parallel-tool-call-execution.zh.md +++ b/.agents/notes/implemented/feature/2026-07-10-parallel-tool-call-execution.zh.md @@ -10,7 +10,7 @@ Status: implemented 并发属于宿主调度范畴,不是面向模型的工具元数据。循环需要在不硬编码工具名称、不向 JSON Schema 暴露调度策略的前提下,判断哪些调用可以重叠执行。 -会话日志仍是权威记录:每个已启动的调用都有审计事件,都会获得结果;无论完成顺序如何,模型历史都按原始调用顺序观察结果。 +会话日志仍是权威记录:每个已启动的调用都有审计事件,正常完成和取消都会使调用与结果配对;无论完成顺序如何,模型历史都按原始调用顺序观察已提交的结果。 ## 决策 @@ -46,7 +46,7 @@ Status: implemented 每个已启动的调用都会在进入 pre-execute 门禁之前立即追加 `tool/call`。已完成的派发占据模型顺序的槽位;提交游标只有在下一个槽位就绪时,才会追加 `tool/result` 并收集 `additionalContexts`。实时界面可以显示多个待处理调用,但结果和工具执行后的上下文仍按模型顺序排列。 -如果在一组启动前中止,系统不会记录该组的任何调用。如果在一组执行期间中止,系统会停止补充池,等待已启动的调用,按顺序提交其结果,在这些结果之后排空已接受的批次上下文,然后通过现有中止路径结束该步骤。从未启动的调用没有审计事件。 +如果在一组启动前中止,系统不会记录该组的任何调用。如果在一组执行期间中止,系统会停止补充池,等待已启动的调用,按顺序提交其结果,在这些结果之后排空已接受的批次上下文,然后通过现有中止路径结束该步骤。从未启动的调用没有审计事件。调度器发生意外故障时,会停止新的派发,等待每项已启动的派发结算,并重新抛出第一个故障。由于该故障是内部终态,而非工具结果,循环不会为被拒绝或未提交的调用虚构工具结果。 Code Mode 仍不使用此调度器,因为模型只会发出一个原生 `run_code` 调用。`run_code` 及其内部派发队列仍按串行方式执行;`mode: 'both'` 中的原生并列调用使用常规调度器。 @@ -66,7 +66,7 @@ Code Mode 仍不使用此调度器,因为模型只会发出一个原生 `run_c ## 验证 -单元测试覆盖固定了安全退化的分类、类型化参数验证、分组、屏障、替换注册表后的实时重新分类、滚动上限、独立执行对象、中间件顺序、有序结果与上下文,以及中止排空。第一方测试固定每项并行声明。 +单元测试覆盖固定了安全退化的分类、类型化参数验证、分组、屏障、替换注册表后的实时重新分类、滚动上限、独立执行对象、中间件顺序、有序结果与上下文、中止排空,以及调度器故障后的完全停稳。第一方测试固定每项并行声明。 快照覆盖固定了可见的多调用 transcript(文本记录):待处理调用可以重叠执行,已完成结果仍按模型顺序排列。Code Mode 覆盖固定其串行边界。此调度属于确定性循环行为,因此无需依赖提供方的 e2e 测试。 @@ -84,6 +84,8 @@ Code Mode 仍不使用此调度器,因为模型只会发出一个原生 `run_c **公开分阶段方法或调度 waterfall。** 公开的 `prepare` / `dispatch` / `finalize` 方法或 `tools/execution-mode` 事件,会在出现另一个消费方之前扩大扩展接口。循环使用内部调度器视图,而 `executionMode(exec)` 为策略 seam 保留了插入点。 +**将调度器故障转换为工具结果。** AgentLoop 无法判断被拒绝的派发是否已调用工具主体;ToolRegistry 负责工具主体调用状态和类型化工具结果。因此,内部调度器故障保持为终态,而不会被重新分类为 `ABORTED` 结果。 + **在模型流式输出时启动调用。** 这可能进一步降低延迟,但会改变 assistant 消息的权威性、回放以及调用/结果配对。调度器只在 assistant 消息完成后才启动。 **使用固定大小的窗口。** 如果在启动下一个窗口前等待当前窗口的每个调用,一个缓慢调用就会使容量闲置。滚动池在保持上限的同时避免了这项延迟。 @@ -101,3 +103,5 @@ Code Mode 仍不使用此调度器,因为模型只会发出一个原生 `run_c 并发外部调用可能会争用配额或进程容量。提供方负责自身容量控制;循环上限只限制一个 agent 步骤中的调用数量。 工具注册是调度边界。调度器会在每个屏障之后以及每次补充池之前重新分类,因此注册表变更会影响尚未启动的调用。已启动的调用保留它们进入池时所依据的调度决策。 + +终态调度器故障可能会在故障步骤关闭前留下已记录但没有结果的调用。等待仍在运行的派发可确保完全停稳,而不会将这些内部故障误报为工具结果。 diff --git a/docs/architecture.i18n.yaml b/docs/architecture.i18n.yaml index c296e10fb1..4bc87d1ee1 100644 --- a/docs/architecture.i18n.yaml +++ b/docs/architecture.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/architecture.md -architecture.md: 2ae982eba49b6dbd2365496915f9917071167813 -architecture.zh.md: abaef961504ff64dbcd1e8e8ba9bd002406fa7f4 +architecture.md: bb5414d6bb108056bf2ff25366e5afe261e1803a +architecture.zh.md: 6d39a320019a1bf87141be0874a5a20a51fc3fbb diff --git a/docs/architecture.md b/docs/architecture.md index 2ae982eba4..bb5414d6bb 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -97,10 +97,10 @@ forever: 'assistant/chunk' 'assistant/message' schedule tool calls by ctx.tools.executionMode: - exclusive -> one-call barrier - parallel -> rolling pool, <= maxParallelToolCalls in flight; reclassify before start - each start -> 'tool/call' -> ordered tools/pre-execute -> concurrent tools/execute - each model-order result -> ordered tools/post-execute -> 'tool/result' + exclusive -> barrier + parallel -> rolling pool, <= maxParallelToolCalls; reclassify-at-start; scheduler failure -> stop starts, drain dispatches + start -> 'tool/call' -> ordered tools/pre-execute -> concurrent tools/execute + model-order result -> ordered tools/post-execute -> 'tool/result' drain accepted tool context and steering 'step/end' continue for tools or steering unless a result concluded the turn diff --git a/docs/architecture.zh.md b/docs/architecture.zh.md index abaef96150..6d39a32001 100644 --- a/docs/architecture.zh.md +++ b/docs/architecture.zh.md @@ -97,10 +97,10 @@ forever: 'assistant/chunk' 'assistant/message' schedule tool calls by ctx.tools.executionMode: - exclusive -> one-call barrier - parallel -> rolling pool, <= maxParallelToolCalls in flight; reclassify before start - each start -> 'tool/call' -> ordered tools/pre-execute -> concurrent tools/execute - each model-order result -> ordered tools/post-execute -> 'tool/result' + exclusive -> barrier + parallel -> rolling pool, <= maxParallelToolCalls; reclassify-at-start; scheduler failure -> stop starts, drain dispatches + start -> 'tool/call' -> ordered tools/pre-execute -> concurrent tools/execute + model-order result -> ordered tools/post-execute -> 'tool/result' drain accepted tool context and steering 'step/end' continue for tools or steering unless a result concluded the turn diff --git a/packages/core/agent-loop/README.i18n.yaml b/packages/core/agent-loop/README.i18n.yaml index 938f374862..339c91a2ff 100644 --- a/packages/core/agent-loop/README.i18n.yaml +++ b/packages/core/agent-loop/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/core/agent-loop/README.md -README.md: 6bb8b12af69f54c2a75cd672e4d3802887808c76 -README.zh.md: 80a01f3e3fdddba8c243cad28c43072148af1dd9 +README.md: 16d70cc06498fec1221b7872f988a0126f69f39f +README.zh.md: ce68595072766ebbf1e4cbd9f7c262cee36c5eff diff --git a/packages/core/agent-loop/README.md b/packages/core/agent-loop/README.md index 6bb8b12af6..16d70cc064 100644 --- a/packages/core/agent-loop/README.md +++ b/packages/core/agent-loop/README.md @@ -67,7 +67,7 @@ After `agent/request` returns a provider/model call config, the loop asks `ctx.l Plugin failure ends the current turn, not the loop. Only final adapter dispatch/iteration failures and terminal in-band error or aborted finishes enter `agent/request-error`; middleware, result processing, tools, and other extension failures close directly. Recovery receives the exact live error, immutable provider facts, immutable prior failures, the immutable retry policy of the adapter registration that served the request, and the turn signal after the failed step closes; the policy is absent if no final adapter served it. A handling listener returns `{ kind: 'retry' }`; the loop closes the failed turn with its error and opens one numbered retry turn without an intervening idle notification. Success clears the consecutive history, and an unhandled failure is terminal. AgentLoop owns one cancellation signal for the current admission or turn. An effective `cancel(cause)` clears pending work unless `keepInbox` is set and cooperatively aborts that signal; idle cancellation is a no-op. Durable `turn/end` records `aborted` for `user` and `parent`, while disposal records `disposed`; undispatched model tool calls receive synthetic `tool/call` and `ABORTED_BEFORE_DISPATCH` result pairs. The cancellation cause changes reporting, not how result context finalized after cancellation is handled. Disposal waits for signal-ignoring work before registry removal. The [explicit-cancellation decision](../../../.agents/notes/implemented/architecture/2026-07-16-explicit-turn-cancellation.md) owns the lifecycle and race contract. -Within a step, exclusive calls form barriers; parallel-safe calls use a bounded rolling pool and are reclassified before start. Only dispatch/body overlaps. Policy, durable results, and result context remain model-ordered. Abort stops new calls, drains started results, and retains their finalized result context without distinguishing the cancellation cause. +Within a step, exclusive calls form barriers; parallel-safe calls use a bounded rolling pool and are reclassified before start. Only dispatch/body overlaps. Policy, durable results, and result context remain model-ordered. Abort stops new calls, drains started results, and retains their finalized result context without distinguishing the cancellation cause. An internal scheduler failure stops new dispatches, waits for already-started dispatches, and reaches the turn error boundary without fabricating tool results. ### What belongs to plugins diff --git a/packages/core/agent-loop/README.zh.md b/packages/core/agent-loop/README.zh.md index 80a01f3e3f..ce68595072 100644 --- a/packages/core/agent-loop/README.zh.md +++ b/packages/core/agent-loop/README.zh.md @@ -67,7 +67,7 @@ interface Config { 插件失败会结束当前轮次,而不是结束循环。只有最终适配器分发/迭代失败以及带内的终止错误或中止结束才进入 `agent/request-error`;中间件、结果处理、工具及其他扩展失败会直接关闭轮次。失败步骤关闭后,恢复逻辑会接收确切的实时错误、不可变的提供方事实、不可变的先前失败、为请求提供服务的适配器注册所对应的不可变重试策略,以及轮次信号;如果没有最终适配器为其提供服务,则该策略缺失。处理失败的监听器返回 `{ kind: 'retry' }`;循环用其错误关闭失败轮次,并在不插入空闲通知的情况下开启一个编号重试轮次。成功会清除连续失败历史;未被处理的失败是终态。AgentLoop 为当前接纳或轮次拥有一个取消信号。有效的 `cancel(cause)` 在未设置 `keepInbox` 时清除待处理工作,并以协作方式中止该信号;空闲取消是空操作。持久 `turn/end` 为 `user` 和 `parent` 记录 `aborted`,dispose(资源释放)则记录 `disposed`;未分发的模型工具调用会收到合成的 `tool/call` 与 `ABORTED_BEFORE_DISPATCH` 结果对。取消原因只改变报告方式,不改变对取消后已定案结果上下文的处理。dispose 会等待忽略信号的工作完成,然后才从注册表移除。[显式取消决策](../../../.agents/notes/implemented/architecture/2026-07-16-explicit-turn-cancellation.md)规定生命周期与竞态契约。 -在步骤内,独占调用形成屏障;并行安全调用使用有界滚动池,并在启动前重新分类。只有分发/主体会重叠。策略、持久结果和结果上下文仍保持模型顺序。中止会停止新调用,drain 已启动的结果,并保留其已定案的结果上下文,不区分取消原因。 +在步骤内,独占调用形成屏障;并行安全调用使用有界滚动池,并在启动前重新分类。只有分发/主体会重叠。策略、持久结果和结果上下文仍保持模型顺序。中止会停止新调用,drain 已启动的结果,并保留其已定案的结果上下文,不区分取消原因。内部调度器故障会停止新的分发,等待已启动的分发,然后在不虚构工具结果的情况下到达轮次错误边界。 ### 插件负责的内容 diff --git a/packages/core/agent-loop/src/tool-calls.ts b/packages/core/agent-loop/src/tool-calls.ts index b7ffdaf041..cd57a28f44 100644 --- a/packages/core/agent-loop/src/tool-calls.ts +++ b/packages/core/agent-loop/src/tool-calls.ts @@ -2,10 +2,12 @@ * Schedules one assistant step's tool calls. Exclusive calls form barriers; * parallel calls use a bounded rolling pool and are reclassified before start. * Dispatch may overlap, while policy, results, and result context remain - * model-ordered. Abort stops replenishment and drains started calls. + * model-ordered. Abort or an internal scheduler failure stops replenishment + * and drains started calls. * - * Each advertised call records a balanced `tool/call`/`tool/result` pair. Calls - * skipped after abort receive synthetic error results so replay stays valid. + * Abort records synthetic error results for skipped calls so replay stays + * valid. A terminal scheduler failure preserves already-recorded `tool/call` + * events without fabricating results. * @module dsh-agent-loop/tool-calls */ @@ -37,10 +39,13 @@ interface GroupOutcome { /** * Schedule one assistant step's tool calls by their live concurrency mode. - * Started calls receive ordered results. Abort drains them, records synthetic - * results for unstarted calls, and returns with the signal still aborted after - * accepting started-call context through the caller-supplied acceptor (the - * machine stages it on its outbox for the next step boundary). + * Ordinary completion and abort commit started-call results in order. Abort + * drains them, records synthetic results for unstarted calls, and returns with + * the signal still aborted after accepting started-call context through the + * caller-supplied acceptor (the machine stages it on its outbox for the next + * step boundary). An internal scheduler failure stops new dispatches, drains + * already-started dispatches, and rejects with the first failure without + * fabricating tool results. * The committed step's AgentLoop driver boundary supplies the initiating Agent * that becomes each explicit {@link ToolExecutionInput.agent}. * @@ -110,7 +115,8 @@ function parseArguments(raw: string): unknown { * drain and remains for the caller's next barrier. Results and contexts commit * in model order. Abort stops starts, drains and commits started calls, accepts * their contexts into the owning batch, records results for skipped calls, and - * returns an aborted outcome. + * returns an aborted outcome. Scheduler failure drains dispatches without + * committing synthetic recovery results. */ async function runGroup( ctx: Context, @@ -131,6 +137,10 @@ async function runGroup( let started = 0 let aborted: boolean = signal.aborted let concluded = false + let schedulerFailure: { error: unknown } | undefined + const throwSchedulerFailure = (): void => { + if (schedulerFailure !== undefined) throw schedulerFailure.error + } // `committed` advances only across contiguous model-order slots. const commitReady = async (): Promise => { @@ -157,12 +167,19 @@ async function runGroup( callSeqs[index] = appendToolCall(session, turn, step, call.block) started++ const prepared = await ctx.tools[TOOL_REGISTRY_SCHEDULER].prepare(call.exec) + throwSchedulerFailure() switch (prepared.kind) { case 'dispatch': { - const promise = ctx.tools[TOOL_REGISTRY_SCHEDULER].dispatch(prepared.exec).then((outcome) => { - slots[index] = { exec: prepared.exec, result: outcome.result, needsPost: outcome.kind === 'post-result' } - return index - }) + const promise = ctx.tools[TOOL_REGISTRY_SCHEDULER].dispatch(prepared.exec).then( + (outcome) => { + slots[index] = { exec: prepared.exec, result: outcome.result, needsPost: outcome.kind === 'post-result' } + return index + }, + (error: unknown) => { + schedulerFailure ??= { error } + return index + }, + ) inFlight.set(index, promise) break } @@ -187,24 +204,34 @@ async function runGroup( && ctx.tools.executionMode(nextCall.exec).kind !== 'parallel') break await startCall(nextToStart) nextToStart++ + throwSchedulerFailure() await commitReady() + throwSchedulerFailure() // Abort may arrive while pre-execute awaits. if (signal.aborted) aborted = true } } - // Ordered pre-execute may await; only dispatch/body overlaps. - // TODO: Drain every started call before rethrowing a scheduler error; tool - // bodies must not outlive the failed turn. - await fillPool() - while (inFlight.size > 0) { - const settledIndex = await Promise.race(inFlight.values()) - inFlight.delete(settledIndex) - await commitReady() - // Abort may arrive while a tool or ordered commit awaits. - - if (signal.aborted) aborted = true + // Ordered pre-execute may await; only dispatch/body overlaps. A scheduler + // failure stops new dispatches and reaches the turn boundary after every + // already-started dispatch settles. + try { await fillPool() + while (inFlight.size > 0) { + const settledIndex = await Promise.race(inFlight.values()) + inFlight.delete(settledIndex) + throwSchedulerFailure() + await commitReady() + throwSchedulerFailure() + // Abort may arrive while a tool or ordered commit awaits. + + if (signal.aborted) aborted = true + await fillPool() + } + } catch (error: unknown) { + schedulerFailure ??= { error } + await Promise.allSettled(inFlight.values()) + throw schedulerFailure.error } if (aborted) { diff --git a/packages/core/agent-loop/tests/tool-calls.spec.ts b/packages/core/agent-loop/tests/tool-calls.spec.ts index a0266fd3f8..dd086d8e7f 100644 --- a/packages/core/agent-loop/tests/tool-calls.spec.ts +++ b/packages/core/agent-loop/tests/tool-calls.spec.ts @@ -9,7 +9,7 @@ import { createUserMessage, CallId, StreamChunk } from '@deepseek-ai/dsh-llm' import SessionStore, { SessionEvent, SessionId } from '@deepseek-ai/dsh-session' import SystemPrompt from '@deepseek-ai/dsh-system-prompt' import LlmService from '@deepseek-ai/dsh-llm' -import ToolRegistry, { defineContentToolFixture, TOOL_ABORTED_BEFORE_DISPATCH, type PostToolDecision, type PreToolDecision } from '@deepseek-ai/dsh-tools' +import ToolRegistry, { defineContentToolFixture, TOOL_ABORTED_BEFORE_DISPATCH, TOOL_REGISTRY_SCHEDULER, type PostToolDecision, type PreToolDecision } from '@deepseek-ai/dsh-tools' import AgentRegistry, { type Agent } from '@deepseek-ai/dsh-agent' import AgentLoop, { DEFAULT_MAX_PARALLEL_TOOL_CALLS } from '@deepseek-ai/dsh-agent-loop' import { MockAdapter, textResponse } from './mock-adapter.ts' @@ -613,3 +613,66 @@ describe('tool-call scheduler: abort handling', () => { }) }) }) + +describe('tool-call scheduler: failure quiescence', () => { + it('stops new dispatches and drains started bodies before surfacing the first failure', async () => { + const adapter = new MockAdapter([ + multiCall([ + { id: 'c1', name: 'p', args: { id: '1' } }, + { id: 'c2', name: 'p', args: { id: '2' } }, + { id: 'c3', name: 'p', args: { id: '3' } }, + ]), + ]) + const ctx = await harness(adapter, 3) + const gated = gatedParallelTool('p') + ctx.tools.register(gated.tool) + // The registry contains expected failures as results; replace its internal + // view only to inject the invariant violation this boundary must contain. + const scheduler = ctx.tools[TOOL_REGISTRY_SCHEDULER] + const prepare = scheduler.prepare.bind(scheduler) + const dispatch = scheduler.dispatch.bind(scheduler) + const prepareGate = Promise.withResolvers() + let thirdPrepareEntered = false + scheduler.prepare = async (exec) => { + const prepared = await prepare(exec) + if (exec.callId === CallId('c3')) { + thirdPrepareEntered = true + await prepareGate.promise + } + return prepared + } + const schedulerError = new Error('scheduler exploded') + const drainedError = new Error('sibling failed while draining') + let rejectFirst: ((error: Error) => void) | undefined + scheduler.dispatch = exec => exec.callId === CallId('c1') + ? new Promise((_resolve, reject) => { rejectFirst = reject }) + : dispatch(exec).then(() => { throw drainedError }) + const agent = ctx.agentLoop.create(SessionId('scheduler-failure'), { provider: 'mock', model: 'mock' }) + const errors: unknown[] = [] + ctx.on('agent/error', (subject, _turn, _step, error) => { + if (subject === agent) errors.push(error) + }) + let idle = false + const idlePromise = waitForIdle(ctx, agent).then(() => { idle = true }) + + agent.followup(createUserMessage({ content: [{ type: 'text', text: 'go' }], source: { kind: 'user' } })) + await until(() => gated.started.includes('2') && thirdPrepareEntered && rejectFirst !== undefined) + rejectFirst?.(schedulerError) + await new Promise(resolve => setImmediate(resolve)) + prepareGate.resolve(undefined) + await new Promise(resolve => setImmediate(resolve)) + + const startedBeforeDrain = [...gated.started] + const idleBeforeDrain = idle + const errorsBeforeDrain = [...errors] + for (const id of gated.pending()) gated.release(id) + await idlePromise + + expect(startedBeforeDrain).toEqual(['2']) + expect(idleBeforeDrain).toBe(false) + expect(errorsBeforeDrain).toEqual([]) + expect(gated.pending()).toEqual([]) + expect(errors).toEqual([schedulerError]) + expect(errors[0]).toBe(schedulerError) + }) +})