workflow: host-guarantee the agent-start/agent-end pairing on every stop path

agent-end was worker-authored only, so a start already forwarded to
observers lost its paired end whenever the worker could no longer speak
- the grace force-settle terminating a wedged script, or an unexpected
worker death - stranding progress consumers with agents that never
finish (ds-review-bot finding on #233).

The host now keeps a ledger of forwarded starts and funnels every
agent-end through one gate: worker-reported ends pair (and clear) their
entry, and both termination paths drain the remainder as synthesized
'cancelled' ends BEFORE the run settles, so ends always precede
workflow/end. A real settlement racing the force-settle loses to the
synthesized cancellation - the same first-wins override onResult applies
to the run's own result.
This commit is contained in:
imccyu
2026-07-09 23:05:43 +08:00
parent 053a2b9403
commit 64d0703c40
6 changed files with 164 additions and 11 deletions

View File

@@ -22,7 +22,11 @@
* survivor when the worker dies or is terminated mid-flight. The three
* paths share ONE disposal per child (memoized by callId; the seam's
* dispose() is idempotent anyway, the memo keeps the bookkeeping and the
* containment warn single). On a termination path `agentsStarted` reports the
* containment warn single). Lifecycle pairing is host-guaranteed the same
* way: every forwarded `agent-start` lives in a ledger, and a start the
* dead or terminated worker never paired is closed by a synthesized
* `agent-end` (outcome `'cancelled'`) before the run settles. On a
* termination path `agentsStarted` reports the
* HOST-observed count (accepted `child-start` messages) — `agent()` calls
* still queued worker-side for a concurrency slot are unknowable then; the
* worker's own count rides the result message on every graceful path.
@@ -37,7 +41,7 @@ import type { Context } from 'cordis'
import type { Agent } from '@deepseek-ai/dsh-agent'
import { assertNever } from '@deepseek-ai/dsh-llm'
import type { SubagentRun } from '@deepseek-ai/dsh-subagent'
import type { WorkflowMeta, WorkflowResult, WorkflowRun, WorkflowRunId } from '@deepseek-ai/dsh-workflow'
import type { WorkflowAgentEndInfo, WorkflowAgentInfo, WorkflowMeta, WorkflowResult, WorkflowRun, WorkflowRunId } from '@deepseek-ai/dsh-workflow'
import { renderThrown } from './realm.ts'
import type { ExecutionObserver } from './runtime.ts'
import { HostToWorkerType, WorkerToHostType } from './protocol.ts'
@@ -92,6 +96,8 @@ export class WorkerRun implements WorkflowRun {
private readonly children = new Map<number, SubagentRun>()
/** In-flight child disposals by callId — the memo that gives every path (worker RPC, dispose(), reap) ONE shared disposal per child. */
private readonly childDisposals = new Map<number, Promise<void>>()
/** Started-but-not-ended agents by seq — the pairing ledger the HOST guarantees (see {@link endAgent}). */
private readonly liveAgents = new Map<number, WorkflowAgentInfo>()
private readonly quiescenceWaiters: (() => void)[] = []
/** The per-run abort fanout every child start request carries. */
private readonly controller = new AbortController()
@@ -155,6 +161,10 @@ export class WorkerRun implements WorkflowRun {
// ChildCancel relay (those later RPCs land as idempotent no-ops).
for (const run of this.children.values()) run.cancel(this.cancelReason)
this.graceTimer = setTimeout(() => {
// The worker may no longer speak (it is about to be terminated): pair
// every stranded start before the run settles, so ends precede
// workflow/end.
this.endStrandedAgents()
this.settleResult(this.cancelledResult(this.hostStarted))
void this.worker.terminate()
}, this.disposeGraceMs)
@@ -225,13 +235,15 @@ export class WorkerRun implements WorkflowRun {
if (this.cancelReason === undefined) this.observer.log(message.message)
break
case WorkerToHostType.AgentStart:
this.liveAgents.set(message.info.seq, message.info)
this.observer.agentStart(message.info)
break
case WorkerToHostType.AgentEnd:
// NOT suppressed on cancel: cancelled children report their paired
// agent-end with outcome 'cancelled' (the one-pair-per-started-child
// contract holds on every stop path).
this.observer.agentEnd(message.info)
// agent-end with outcome 'cancelled'. The gate (with the termination
// paths' synthesis) is what makes the one-pair-per-started-child
// contract hold on every stop path.
this.endAgent(message.info)
break
case WorkerToHostType.ChildStart:
this.onChildStart(message.callId, message.request)
@@ -373,6 +385,10 @@ export class WorkerRun implements WorkflowRun {
private onWorkerDeath(message: string): void {
// Whatever the worker left behind must not leak — abort + dispose it all.
if (this.children.size > 0) this.reapChildren('workflow worker gone')
// The thread is gone: no more worker-authored agent-ends can arrive —
// pair every stranded start (a start that crossed between the grace
// force-settle and this exit included) before the run settles.
this.endStrandedAgents()
// settleResult no-ops on an already-settled run (the expected exit after
// a dispose's terminate lands here too).
if (this.cancelReason !== undefined) {
@@ -382,6 +398,34 @@ export class WorkerRun implements WorkflowRun {
this.settleResult({ value: null, stopReason: 'error', error: message, agentsStarted: this.hostStarted })
}
/**
* The single agent-end emission gate: forwards `end` iff its start is still
* unpaired in the ledger, so every forwarded `workflow/agent-start` gets
* EXACTLY one `workflow/agent-end` — the worker's own report where it can
* speak, a host-synthesized one where it cannot ({@link endStrandedAgents}).
* @param end - the settlement to emit (worker-reported or synthesized).
*/
private endAgent(end: WorkflowAgentEndInfo): void {
/* v8 ignore next -- a real end still in flight across the grace force-settle: not orderable in-process */
if (!this.liveAgents.delete(end.seq)) return
this.observer.agentEnd(end)
}
/**
* Synthesize the missing `agent-end` for every started-but-unpaired agent,
* outcome `'cancelled'`: the reap cancels every child, and a real
* settlement racing the force-settle loses to the cancellation — the same
* first-wins override {@link onResult} applies to the run's own result.
* Called where the worker can no longer speak (the grace force-settle,
* worker death), BEFORE settleResult, so the paired ends reach observers
* before `workflow/end`.
*/
private endStrandedAgents(): void {
for (const info of [...this.liveAgents.values()]) {
this.endAgent({ ...info, outcome: 'cancelled' })
}
}
private cancelledResult(agentsStarted: number): WorkflowResult {
// cancel() is the only writer of cancelReason and every caller checks it
// first; the fallback guards the type, not a reachable path.