|
|
|
|
@@ -1,16 +1,53 @@
|
|
|
|
|
/**
|
|
|
|
|
* The host half of one worker-engine run: spawn the Worker, bridge its child RPC onto
|
|
|
|
|
* `ctx.subagents`, fan its observer messages into the engine's events, and own cancellation,
|
|
|
|
|
* the settle-within-grace guarantee, and child cleanup.
|
|
|
|
|
* The host half of one worker-engine run: spawn the Worker, bridge its child
|
|
|
|
|
* RPC onto the holder-bound subagent service, fan its observer messages into
|
|
|
|
|
* the engine's events, and own cancellation, the settle-within-grace
|
|
|
|
|
* guarantee, and child cleanup. The worker's lifetime IS the run's lifetime:
|
|
|
|
|
* `dispose()` always ends with `worker.terminate()`, so no thread outlives its
|
|
|
|
|
* run.
|
|
|
|
|
*
|
|
|
|
|
* The run's `result` promise settles exactly once, from whichever of these
|
|
|
|
|
* lands first: receipt of the worker's `result` message, an unexpected worker
|
|
|
|
|
* death (`error`/`messageerror`/premature `exit` → `stopReason: 'error'`, or
|
|
|
|
|
* `'cancelled'` when a cancel was in flight), or the post-cancel grace timer (a
|
|
|
|
|
* script that never settles is force-settled `cancelled` and its worker
|
|
|
|
|
* terminated — the real kill an in-process engine could not perform). At
|
|
|
|
|
* `result` receipt the host snapshots whether caller/signal/dispose
|
|
|
|
|
* cancellation is already in flight: an earlier cancellation overrides a
|
|
|
|
|
* non-cancelled report; otherwise the report wins before settlement-only child
|
|
|
|
|
* cleanup invokes arbitrary provider callbacks. Worker death uses the same
|
|
|
|
|
* boundary: it claims `error` (or a previously requested `cancelled`) before
|
|
|
|
|
* reaping children, so cleanup callbacks cannot rewrite the outcome. That
|
|
|
|
|
* first signal also closes inbound message admission: Node may emit `error`,
|
|
|
|
|
* then deliver queued messages, then emit `exit`, but those late messages may
|
|
|
|
|
* neither create work nor narrate after settlement. If Result or grace already
|
|
|
|
|
* owns the outcome, death preserves it while still cleaning resources; the
|
|
|
|
|
* eventual exit performs a final disposal-only sweep without repeating child
|
|
|
|
|
* cancellation.
|
|
|
|
|
*
|
|
|
|
|
* Provider starts and published children are tracked separately. Every start
|
|
|
|
|
* receives one shared per-run abort signal; the provider owns partial setup
|
|
|
|
|
* until its promise fulfills. If admission closes while a start is pending,
|
|
|
|
|
* the signal aborts it; a late fulfillment is disposed without publication to
|
|
|
|
|
* the worker. Ready runs enter a callId registry whose memoized disposal is
|
|
|
|
|
* shared by graceful worker RPC, public disposal, normal-settlement reap, and
|
|
|
|
|
* worker-death cleanup. Quiescence requires both pending starts and published
|
|
|
|
|
* children to drain. Lifecycle pairing is host-guaranteed independently:
|
|
|
|
|
* every forwarded `agent-start` enters a ledger, and a dead or terminated
|
|
|
|
|
* worker's missing `agent-end` is synthesized exactly once as cancelled. On a
|
|
|
|
|
* termination path `agentsStarted` reports the host-observed child-start count;
|
|
|
|
|
* calls still queued worker-side for a concurrency slot are unknowable.
|
|
|
|
|
*
|
|
|
|
|
* @module @deepseek-ai/dsh-workflow-workerthread/host
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
import { fileURLToPath } from 'node:url'
|
|
|
|
|
import { Worker } from 'node:worker_threads'
|
|
|
|
|
import type { WorkerOptions } from 'node:worker_threads'
|
|
|
|
|
import type { Context } from 'cordis'
|
|
|
|
|
import type { Agent } from '@deepseek-ai/dsh-agent'
|
|
|
|
|
import { assertNever } from '@deepseek-ai/dsh-llm'
|
|
|
|
|
import { snapshotJsonValue } from '@deepseek-ai/dsh-session'
|
|
|
|
|
import type SubagentService from '@deepseek-ai/dsh-subagent'
|
|
|
|
|
import type { SubagentRun } from '@deepseek-ai/dsh-subagent'
|
|
|
|
|
import type { WorkflowAgentEndInfo, WorkflowAgentInfo, WorkflowMeta, WorkflowResult, WorkflowRun, WorkflowRunId } from '@deepseek-ai/dsh-workflow'
|
|
|
|
|
import { renderThrown } from './realm.ts'
|
|
|
|
|
@@ -19,9 +56,36 @@ import { HostToWorkerType, WorkerToHostType } from './protocol.ts'
|
|
|
|
|
import type { HostToWorkerPayloads, WorkerToHostMessage } from './protocol.ts'
|
|
|
|
|
import type { ChildResult, ChildStartRequest, WorkerInit } from './types.ts'
|
|
|
|
|
|
|
|
|
|
/** One published child and its shared quiescent-disposal transaction. */
|
|
|
|
|
interface ChildRecord {
|
|
|
|
|
readonly run: SubagentRun
|
|
|
|
|
disposal?: Promise<void>
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Resolve the worker entry and spawn options for the current runtime shape.
|
|
|
|
|
* Unbuilt (tsx demos, vitest — `import.meta.url` points into `src/`), the
|
|
|
|
|
* entry is a JavaScript data-URL bootstrap. That bootstrap runs INSIDE the
|
|
|
|
|
* user worker, registers tsx's ESM AND CommonJS transforms there, and only
|
|
|
|
|
* then imports the TypeScript sibling. The whole mixed-module source graph
|
|
|
|
|
* therefore receives TypeScript transformation and the tsconfig paths map in
|
|
|
|
|
* the worker's own module-loader realm. A worker inherits no
|
|
|
|
|
* transform pipeline from vitest (vite transforms in-process), and a parent
|
|
|
|
|
* `--import tsx` registration is not a contract that user workers share on
|
|
|
|
|
* every supported Node line. Built (`lib/index.js`), the entry is the sibling
|
|
|
|
|
* bundle the package tsdown config emits and no loader is needed (`execArgv`
|
|
|
|
|
* pinned empty in both shapes — hermetic, like the environment).
|
|
|
|
|
*
|
|
|
|
|
* Both shapes spawn with an EMPTY environment (`env: {}`): the documented vm
|
|
|
|
|
* escape reaches `process`, and the harness's ambient credentials
|
|
|
|
|
* (`DEEPSEEK_API_KEY` et al.) must not ride along — the same stance as
|
|
|
|
|
* `dsh-code-runtime-worker`, stronger than the scrubbed env the
|
|
|
|
|
* defensive-patterns rule requires for spawned commands (a shell needs PATH;
|
|
|
|
|
* this worker needs nothing). Sole exception: the unbuilt shape forwards
|
|
|
|
|
* `TSX_TSCONFIG_PATH` when the parent carries it (loader plumbing the paths
|
|
|
|
|
* map depends on outside the repo cwd, not a secret). This closes the
|
|
|
|
|
* AMBIENT channel only — an escapee still holds process-wide privileges
|
|
|
|
|
* like fs access (the README's trust premise stands).
|
|
|
|
|
* @param init - the run payload, passed as `workerData`.
|
|
|
|
|
* @returns the entry URL and the Worker options to spawn it with.
|
|
|
|
|
*/
|
|
|
|
|
@@ -30,14 +94,31 @@ function resolveWorkerSpawn(init: WorkerInit): { entry: URL; options: WorkerOpti
|
|
|
|
|
if (!import.meta.url.endsWith('.ts')) {
|
|
|
|
|
return { entry: new URL('./worker.js', import.meta.url), options: { workerData: init, env: {}, execArgv: [] } }
|
|
|
|
|
}
|
|
|
|
|
// Lazy tsx resolution: only the unbuilt shape needs it, so the built bundle never requires
|
|
|
|
|
// tsx to be installed.
|
|
|
|
|
// Resolve tsx lazily: only the unbuilt shape executes this arm, so a built
|
|
|
|
|
// consumer never needs the dev-only loader installed. A JavaScript entry is
|
|
|
|
|
// essential — it can install tsx's ESM and CommonJS hooks from INSIDE the
|
|
|
|
|
// user worker before any TypeScript enters Node's native strip-only parser.
|
|
|
|
|
// Both hooks are load-bearing because the source graph crosses both module
|
|
|
|
|
// shapes on supported Node lines. TSX_TSCONFIG_PATH is
|
|
|
|
|
// the one variable forwarded through the scrub: a parent running outside
|
|
|
|
|
// the repo cwd (the ACP snapshot harness is the real case) pins the paths
|
|
|
|
|
// map through it. Loader plumbing, not a secret.
|
|
|
|
|
const workerEntry = new URL('./worker.ts', import.meta.url)
|
|
|
|
|
const tsxEsmApiEntry = import.meta.resolve('tsx/esm/api')
|
|
|
|
|
const tsxCjsApiEntry = import.meta.resolve('tsx/cjs/api')
|
|
|
|
|
const bootstrap = [
|
|
|
|
|
`import { register as registerEsm } from ${JSON.stringify(tsxEsmApiEntry)}`,
|
|
|
|
|
`import { register as registerCjs } from ${JSON.stringify(tsxCjsApiEntry)}`,
|
|
|
|
|
'registerCjs()',
|
|
|
|
|
'registerEsm()',
|
|
|
|
|
`await import(${JSON.stringify(workerEntry.href)})`,
|
|
|
|
|
].join('\n')
|
|
|
|
|
return {
|
|
|
|
|
entry: new URL('./worker.ts', import.meta.url),
|
|
|
|
|
entry: new URL(`data:text/javascript,${encodeURIComponent(bootstrap)}`),
|
|
|
|
|
options: {
|
|
|
|
|
workerData: init,
|
|
|
|
|
env: process.env.TSX_TSCONFIG_PATH === undefined ? {} : { TSX_TSCONFIG_PATH: process.env.TSX_TSCONFIG_PATH },
|
|
|
|
|
execArgv: ['--import', fileURLToPath(import.meta.resolve('tsx'))],
|
|
|
|
|
execArgv: [],
|
|
|
|
|
},
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
@@ -45,15 +126,21 @@ function resolveWorkerSpawn(init: WorkerInit): { entry: URL; options: WorkerOpti
|
|
|
|
|
/**
|
|
|
|
|
* One live worker-engine run — the seam's {@link WorkflowRun}, returned by
|
|
|
|
|
* `start()` directly. Owns the Worker, the child registry, and the result
|
|
|
|
|
* settlement; `result` never rejects. `meta` is this handle's OWN clone
|
|
|
|
|
* (event payloads carry separate clones), so a consumer mutating it corrupts
|
|
|
|
|
* nothing.
|
|
|
|
|
* settlement; `result` never rejects. `meta` is trusted same-process data
|
|
|
|
|
* borrowed as immutable by the handle and lifecycle events. The holder-bound
|
|
|
|
|
* SubagentService handle is captured before the
|
|
|
|
|
* engine returns this run, so unloading the engine removes only the ability to
|
|
|
|
|
* start another workflow; this run can still start and clean up its children.
|
|
|
|
|
*/
|
|
|
|
|
export class WorkerRun implements WorkflowRun {
|
|
|
|
|
/** Settles exactly once with the run's outcome; never rejects. */
|
|
|
|
|
readonly result: Promise<WorkflowResult>
|
|
|
|
|
private settleResolve!: (result: WorkflowResult) => void
|
|
|
|
|
private settled = false
|
|
|
|
|
/** A Result/death/grace outcome atomically won before teardown callbacks. */
|
|
|
|
|
private terminalClaimed = false
|
|
|
|
|
/** The first death signal closes worker-message admission and owns failure-time cleanup. */
|
|
|
|
|
private workerDeathObserved = false
|
|
|
|
|
private cancelReason: string | undefined
|
|
|
|
|
private graceTimer: NodeJS.Timeout | undefined
|
|
|
|
|
private readonly worker: Worker
|
|
|
|
|
@@ -61,19 +148,23 @@ export class WorkerRun implements WorkflowRun {
|
|
|
|
|
private workerGone = false
|
|
|
|
|
/** Accepted `child-start` messages — the terminate-path `agentsStarted` (see module doc). */
|
|
|
|
|
private hostStarted = 0
|
|
|
|
|
/** Live children by callId; an entry leaves ONLY after its dispose settles (quiescence = empty). */
|
|
|
|
|
private readonly children = new Map<number, SubagentRun>()
|
|
|
|
|
/** In-flight child disposals by callId — the memo that gives every path (worker RPC, dispose(), reap) ONE shared disposal per child. */
|
|
|
|
|
private readonly childDisposals = new Map<number, Promise<void>>()
|
|
|
|
|
/** Published children by callId; an entry leaves only after disposal settles. */
|
|
|
|
|
private readonly children = new Map<number, ChildRecord>()
|
|
|
|
|
/** Provider starts that have not yet fulfilled or rejected. */
|
|
|
|
|
private readonly pendingStarts = new Set<Promise<void>>()
|
|
|
|
|
/** Started-but-not-ended agents by seq — the pairing ledger the HOST guarantees (see {@link endAgent}). */
|
|
|
|
|
private readonly liveAgents = new Map<number, WorkflowAgentInfo>()
|
|
|
|
|
private readonly quiescenceWaiters: (() => void)[] = []
|
|
|
|
|
/** The per-run abort fanout every child start request carries. */
|
|
|
|
|
private readonly controller = new AbortController()
|
|
|
|
|
/** External start signal and the exact callback installed on it, retained only until first settle/teardown. */
|
|
|
|
|
private inputSignal: AbortSignal | undefined
|
|
|
|
|
private inputSignalAbort: (() => void) | undefined
|
|
|
|
|
private disposed: Promise<void> | undefined
|
|
|
|
|
|
|
|
|
|
constructor(
|
|
|
|
|
private readonly ctx: Context,
|
|
|
|
|
private readonly subagents: SubagentService,
|
|
|
|
|
readonly id: WorkflowRunId,
|
|
|
|
|
readonly meta: WorkflowMeta,
|
|
|
|
|
private readonly parent: Agent,
|
|
|
|
|
@@ -90,34 +181,53 @@ export class WorkerRun implements WorkflowRun {
|
|
|
|
|
const { entry, options } = resolveWorkerSpawn(init)
|
|
|
|
|
this.worker = new Worker(entry, options)
|
|
|
|
|
this.worker.on('message', (message: WorkerToHostMessage) => { this.onMessage(message) })
|
|
|
|
|
this.worker.on('error', (error) => { this.onWorkerDeath(`workflow worker failed: ${renderThrown(error)}`) })
|
|
|
|
|
this.worker.on('error', (error) => { this.onWorkerDeath(`workflow worker failed: ${renderThrown(error)}`, false) })
|
|
|
|
|
/* v8 ignore next -- messageerror: not constructible from the engine's own protocol (every payload is JSON data) */
|
|
|
|
|
this.worker.on('messageerror', (error) => { this.onWorkerDeath(`workflow worker message failed to deserialize: ${renderThrown(error)}`) })
|
|
|
|
|
this.worker.on('messageerror', (error) => { this.onWorkerDeath(`workflow worker message failed to deserialize: ${renderThrown(error)}`, false) })
|
|
|
|
|
this.worker.on('exit', (code) => {
|
|
|
|
|
this.workerGone = true
|
|
|
|
|
this.onWorkerDeath(`workflow worker exited before the run settled (exit code ${code})`)
|
|
|
|
|
this.onWorkerDeath(`workflow worker exited before the run settled (exit code ${code})`, true)
|
|
|
|
|
})
|
|
|
|
|
if (signal?.aborted) {
|
|
|
|
|
this.cancel('workflow start signal already aborted')
|
|
|
|
|
} else {
|
|
|
|
|
signal?.addEventListener('abort', () => { this.cancel('workflow signal aborted') }, { once: true })
|
|
|
|
|
} else if (signal !== undefined) {
|
|
|
|
|
const onAbort = (): void => {
|
|
|
|
|
this.detachInputSignal()
|
|
|
|
|
this.cancel('workflow signal aborted')
|
|
|
|
|
}
|
|
|
|
|
this.inputSignal = signal
|
|
|
|
|
this.inputSignalAbort = onAbort
|
|
|
|
|
signal.addEventListener('abort', onAbort, { once: true })
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Cancel the worker and host-owned children, then arm forced settlement.
|
|
|
|
|
* Cancel the run: the worker is told (its hooks start throwing and the
|
|
|
|
|
* script dies at its next await), the required signal shared by every child
|
|
|
|
|
* start is aborted, and the grace timer
|
|
|
|
|
* arms: a run still unsettled `disposeGraceMs` later force-settles
|
|
|
|
|
* `cancelled` and its worker is TERMINATED. Idempotent; the first reason
|
|
|
|
|
* wins.
|
|
|
|
|
* @param reason - human-readable cause (default `'workflow cancelled'`).
|
|
|
|
|
*/
|
|
|
|
|
cancel(reason?: string): void {
|
|
|
|
|
// Do not arm a grace timer after settlement.
|
|
|
|
|
if (this.settled || this.cancelReason !== undefined) return
|
|
|
|
|
// A settled run has nothing left to cancel, and a terminal source claimed
|
|
|
|
|
// before its cleanup callbacks must exclude cancellation reentered by one
|
|
|
|
|
// of those callbacks. Without the settled guard the
|
|
|
|
|
// ordinary consumer path (await result, then dispose -> cancel) would arm
|
|
|
|
|
// a grace timer nothing ever clears, pinning the run and its Worker
|
|
|
|
|
// closure until the grace expires - a bounded leak per completed run.
|
|
|
|
|
if (this.settled || this.terminalClaimed || this.cancelReason !== undefined) return
|
|
|
|
|
this.cancelReason = reason ?? 'workflow cancelled'
|
|
|
|
|
this.post(HostToWorkerType.Cancel, { reason: this.cancelReason })
|
|
|
|
|
this.controller.abort(this.cancelReason)
|
|
|
|
|
// Host-side cancellation still reaches children when the worker is wedged.
|
|
|
|
|
for (const run of this.children.values()) run.cancel(this.cancelReason)
|
|
|
|
|
this.abortChildren(this.cancelReason)
|
|
|
|
|
this.graceTimer = setTimeout(() => {
|
|
|
|
|
// Pair stranded child starts before terminal workflow events.
|
|
|
|
|
// Cancellation already owns the race through cancelReason; close the
|
|
|
|
|
// terminal boundary explicitly before observer teardown callbacks.
|
|
|
|
|
this.terminalClaimed = true
|
|
|
|
|
// The worker may no longer speak (it is about to be terminated): pair
|
|
|
|
|
// every stranded start before the run settles, so ends precede
|
|
|
|
|
// workflow/end.
|
|
|
|
|
this.endStrandedAgents()
|
|
|
|
|
this.settleResult(this.cancelledResult(this.hostStarted))
|
|
|
|
|
void this.worker.terminate()
|
|
|
|
|
@@ -127,14 +237,36 @@ export class WorkerRun implements WorkflowRun {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Cancel + bounded settle + termination.
|
|
|
|
|
*
|
|
|
|
|
* Cancel + bounded settle + termination. Host-drives every registered
|
|
|
|
|
* child's disposal IMMEDIATELY — a wedged worker can relay no dispose RPC,
|
|
|
|
|
* and deferring child teardown to the post-terminate reap would spend the
|
|
|
|
|
* whole grace waiting for a quiescence that cannot start, then return with
|
|
|
|
|
* the disposals still in flight — so child disposal overlaps the same
|
|
|
|
|
* grace the worker gets to settle (the worker's own dispose RPCs join the
|
|
|
|
|
* shared per-child disposal). Waits (at most the grace) for the result and
|
|
|
|
|
* child quiescence, then terminates the worker unconditionally — the
|
|
|
|
|
* thread never outlives its run — and reaps whatever children remain
|
|
|
|
|
* (their disposal is contained, not awaited past the grace, the same
|
|
|
|
|
* abandonment the seam documents for a slow-disposing child). Idempotent;
|
|
|
|
|
* safe on every path.
|
|
|
|
|
* @returns resolves when the run's resources are released or abandoned.
|
|
|
|
|
*/
|
|
|
|
|
dispose(): Promise<void> {
|
|
|
|
|
this.disposed ??= (async () => {
|
|
|
|
|
if (this.disposed !== undefined) return this.disposed
|
|
|
|
|
// Claim the public transaction BEFORE its body invokes child/provider
|
|
|
|
|
// disposal. A raw provider callback can reenter handle.dispose(); it must
|
|
|
|
|
// join this promise rather than start a second traversal.
|
|
|
|
|
const claimed = Promise.withResolvers<undefined>()
|
|
|
|
|
this.disposed = claimed.promise
|
|
|
|
|
void (async () => {
|
|
|
|
|
this.detachInputSignal()
|
|
|
|
|
this.cancel('workflow disposed')
|
|
|
|
|
for (const [callId, run] of [...this.children]) void this.disposeChild(callId, run)
|
|
|
|
|
// cancel() deliberately becomes a no-op after terminal settlement, but
|
|
|
|
|
// disposal still owns every registered child. Reap independently so an
|
|
|
|
|
// already-settled workflow cannot wait on child quiescence before it has
|
|
|
|
|
// started the surviving children's disposals. On an unsettled run this
|
|
|
|
|
// joins the cancel path through the per-call cancellation/disposal gates.
|
|
|
|
|
this.reapChildren('workflow disposed')
|
|
|
|
|
await Promise.race([
|
|
|
|
|
(async () => {
|
|
|
|
|
await this.result
|
|
|
|
|
@@ -144,13 +276,17 @@ export class WorkerRun implements WorkflowRun {
|
|
|
|
|
])
|
|
|
|
|
await this.worker.terminate()
|
|
|
|
|
this.reapChildren('workflow disposed')
|
|
|
|
|
})()
|
|
|
|
|
})().then(
|
|
|
|
|
() => { claimed.resolve(undefined) },
|
|
|
|
|
/* v8 ignore next -- result/quiescence never reject and Worker.terminate is the only external promise */
|
|
|
|
|
(error: unknown) => { claimed.reject(error) },
|
|
|
|
|
)
|
|
|
|
|
return this.disposed
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** Post one message to the worker (payload looked up from the tag's map entry), tolerating a thread that is already gone. */
|
|
|
|
|
private post<T extends HostToWorkerType>(type: T, payload: HostToWorkerPayloads[T]): void {
|
|
|
|
|
if (this.workerGone) return
|
|
|
|
|
if (this.workerGone || this.workerDeathObserved) return
|
|
|
|
|
try {
|
|
|
|
|
this.worker.postMessage({ type, ...payload })
|
|
|
|
|
} catch (error: unknown) {
|
|
|
|
|
@@ -163,6 +299,11 @@ export class WorkerRun implements WorkflowRun {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private onMessage(message: WorkerToHostMessage): void {
|
|
|
|
|
// Node may emit `error`, then deliver an already-queued `message`, then
|
|
|
|
|
// emit `exit`. The first death signal is the host's logical delivery
|
|
|
|
|
// barrier: nothing arriving afterward may create a child, narrate after
|
|
|
|
|
// workflow/end, or compete with the chosen outcome.
|
|
|
|
|
if (this.workerDeathObserved) return
|
|
|
|
|
switch (message.type) {
|
|
|
|
|
case WorkerToHostType.Ready:
|
|
|
|
|
this.post(HostToWorkerType.Go, {})
|
|
|
|
|
@@ -192,9 +333,6 @@ export class WorkerRun implements WorkflowRun {
|
|
|
|
|
case WorkerToHostType.ChildStart:
|
|
|
|
|
this.onChildStart(message.callId, message.request)
|
|
|
|
|
break
|
|
|
|
|
case WorkerToHostType.ChildCancel:
|
|
|
|
|
this.children.get(message.callId)?.cancel(message.reason)
|
|
|
|
|
break
|
|
|
|
|
case WorkerToHostType.ChildDispose:
|
|
|
|
|
this.onChildDispose(message.callId)
|
|
|
|
|
break
|
|
|
|
|
@@ -207,18 +345,44 @@ export class WorkerRun implements WorkflowRun {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private onChildStart(callId: number, request: ChildStartRequest): void {
|
|
|
|
|
/** Why a ready provider result may no longer be admitted to the worker. */
|
|
|
|
|
private childAdmissionFailure(): { reason: string; rendered: string } | undefined {
|
|
|
|
|
if (this.cancelReason !== undefined) {
|
|
|
|
|
// The worker's start raced our cancel: refuse — a child must never
|
|
|
|
|
// start on an already-aborted signal (a provider subscribing only to
|
|
|
|
|
// future abort events would never observe it).
|
|
|
|
|
this.post(HostToWorkerType.ChildStartError, { callId, rendered: `workflow run cancelled: ${this.cancelReason}` })
|
|
|
|
|
return { reason: this.cancelReason, rendered: `workflow run cancelled: ${this.cancelReason}` }
|
|
|
|
|
}
|
|
|
|
|
if (this.workerDeathObserved) {
|
|
|
|
|
return { reason: 'workflow worker gone', rendered: 'workflow worker is no longer available' }
|
|
|
|
|
}
|
|
|
|
|
if (this.terminalClaimed) {
|
|
|
|
|
return { reason: 'workflow settled', rendered: 'workflow run already settled' }
|
|
|
|
|
}
|
|
|
|
|
return undefined
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private onChildStart(callId: number, request: ChildStartRequest): void {
|
|
|
|
|
const initialFailure = this.childAdmissionFailure()
|
|
|
|
|
if (initialFailure !== undefined) {
|
|
|
|
|
// Refuse after a terminal boundary: a child must never start on an
|
|
|
|
|
// already-aborted signal (a provider subscribing only to future abort
|
|
|
|
|
// events would never observe it).
|
|
|
|
|
this.post(HostToWorkerType.ChildStartError, { callId, rendered: initialFailure.rendered })
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
this.hostStarted += 1
|
|
|
|
|
const task = this.startChild(callId, request)
|
|
|
|
|
this.pendingStarts.add(task)
|
|
|
|
|
void task.then(
|
|
|
|
|
() => { this.finishPendingStart(task) },
|
|
|
|
|
/* v8 ignore next -- startChild contains provider and cleanup failures */
|
|
|
|
|
() => { this.finishPendingStart(task) },
|
|
|
|
|
)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** Await one provider-owned startup transaction and publish only while admitted. */
|
|
|
|
|
private async startChild(callId: number, request: ChildStartRequest): Promise<void> {
|
|
|
|
|
let run: SubagentRun
|
|
|
|
|
try {
|
|
|
|
|
run = this.ctx.subagents.start(this.provider, {
|
|
|
|
|
run = await this.subagents.start(this.provider, {
|
|
|
|
|
prompt: [{ type: 'text', text: request.prompt }],
|
|
|
|
|
parent: this.parent,
|
|
|
|
|
signal: this.controller.signal,
|
|
|
|
|
@@ -226,21 +390,38 @@ export class WorkerRun implements WorkflowRun {
|
|
|
|
|
...request.model !== undefined ? { agentOptions: { model: request.model } } : {},
|
|
|
|
|
})
|
|
|
|
|
} catch (error: unknown) {
|
|
|
|
|
this.post(HostToWorkerType.ChildStartError, { callId, rendered: renderThrown(error) })
|
|
|
|
|
const failure = this.childAdmissionFailure()
|
|
|
|
|
this.post(HostToWorkerType.ChildStartError, {
|
|
|
|
|
callId,
|
|
|
|
|
rendered: failure?.rendered ?? renderThrown(error),
|
|
|
|
|
})
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
const failure = this.childAdmissionFailure()
|
|
|
|
|
if (failure !== undefined) {
|
|
|
|
|
this.post(HostToWorkerType.ChildStartError, { callId, rendered: failure.rendered })
|
|
|
|
|
try {
|
|
|
|
|
await run.dispose()
|
|
|
|
|
} catch (error: unknown) {
|
|
|
|
|
this.ctx.logger.warn(`workflow-workerthread: refused child dispose failed: ${renderThrown(error)}`)
|
|
|
|
|
}
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
this.children.set(callId, run)
|
|
|
|
|
const childId = run.id
|
|
|
|
|
|
|
|
|
|
// Observe settlement IMMEDIATELY, before readiness.
|
|
|
|
|
const record: ChildRecord = { run }
|
|
|
|
|
this.children.set(callId, record)
|
|
|
|
|
// Attach result forwarding before publishing the child handle. Because the
|
|
|
|
|
// callback itself runs in a later microtask, ChildStarted is still posted
|
|
|
|
|
// first even for an already-settled scripted provider.
|
|
|
|
|
const forwardResult = run.result.then<() => void, () => void>(
|
|
|
|
|
(result) => {
|
|
|
|
|
try {
|
|
|
|
|
const snapshot: ChildResult = structuredClone({
|
|
|
|
|
const snapshot = snapshotJsonValue<ChildResult>({
|
|
|
|
|
output: result.output,
|
|
|
|
|
...result.structured !== undefined ? { structured: result.structured } : {},
|
|
|
|
|
stopReason: result.stopReason,
|
|
|
|
|
})
|
|
|
|
|
if (snapshot === undefined) throw new TypeError('child result is not losslessly JSON-serializable')
|
|
|
|
|
return () => { this.post(HostToWorkerType.ChildSettled, { callId, result: snapshot }) }
|
|
|
|
|
} catch (error: unknown) {
|
|
|
|
|
const rendered = `workflow child result could not cross the worker boundary: ${renderThrown(error)}`
|
|
|
|
|
@@ -252,88 +433,104 @@ export class WorkerRun implements WorkflowRun {
|
|
|
|
|
return () => { this.post(HostToWorkerType.ChildFailed, { callId, rendered }) }
|
|
|
|
|
},
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
// The provider owns the publication boundary.
|
|
|
|
|
void run.started.then(
|
|
|
|
|
() => {
|
|
|
|
|
this.post(HostToWorkerType.ChildStarted, { callId, childId })
|
|
|
|
|
void forwardResult.then((forward) => { forward() })
|
|
|
|
|
},
|
|
|
|
|
(error: unknown) => {
|
|
|
|
|
this.post(HostToWorkerType.ChildStartError, { callId, rendered: renderThrown(error) })
|
|
|
|
|
if (this.children.get(callId) === run) void this.disposeChild(callId, run)
|
|
|
|
|
},
|
|
|
|
|
)
|
|
|
|
|
this.post(HostToWorkerType.ChildStarted, { callId, childId: run.id })
|
|
|
|
|
void forwardResult.then((forward) => { forward() })
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private onChildDispose(callId: number): void {
|
|
|
|
|
const run = this.children.get(callId)
|
|
|
|
|
if (run === undefined) {
|
|
|
|
|
const record = this.children.get(callId)
|
|
|
|
|
if (record === undefined) {
|
|
|
|
|
// Already disposed host-side (a dispose() drive or a death reap beat
|
|
|
|
|
// the RPC) — the ack is still owed (the worker-side wrapper awaits it).
|
|
|
|
|
this.post(HostToWorkerType.ChildDisposed, { callId })
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
// disposeChild never rejects (containment is inside), so the ack always follows.
|
|
|
|
|
void this.disposeChild(callId, run).then(() => { this.post(HostToWorkerType.ChildDisposed, { callId }) })
|
|
|
|
|
void this.disposeChild(callId, record).then(() => { this.post(HostToWorkerType.ChildDisposed, { callId }) })
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Start (or join) one registered child's disposal; the registry entry leaves when it
|
|
|
|
|
* settles.
|
|
|
|
|
*
|
|
|
|
|
* Start (or join) one registered child's disposal; the registry entry
|
|
|
|
|
* leaves when it settles. Memoized per callId: the worker's dispose RPC,
|
|
|
|
|
* the dispose() host drive, and the reap can all land on the same child —
|
|
|
|
|
* the child's `dispose()` runs once and every caller awaits that one
|
|
|
|
|
* settlement. A rejection is contained (the subagent seam's dispose() is
|
|
|
|
|
* not supposed to reject, but a backend that does anyway must not break
|
|
|
|
|
* quiescence): logged, and the child still leaves the registry.
|
|
|
|
|
* @param callId - the child's registry key.
|
|
|
|
|
* @param run - the registered child (the caller looked it up).
|
|
|
|
|
* @param record - the registered child (the caller looked it up).
|
|
|
|
|
* @returns resolves when the disposal settled either way; never rejects.
|
|
|
|
|
*/
|
|
|
|
|
private disposeChild(callId: number, run: SubagentRun): Promise<void> {
|
|
|
|
|
let disposal = this.childDisposals.get(callId)
|
|
|
|
|
if (disposal === undefined) {
|
|
|
|
|
// The seam promises a Promise, but invoke inside an async boundary so a
|
|
|
|
|
// contract-violating synchronous throw is contained exactly like a
|
|
|
|
|
// rejected disposal and cannot break host quiescence.
|
|
|
|
|
disposal = (async () => { await run.dispose() })().then(
|
|
|
|
|
() => { this.finishChild(callId) },
|
|
|
|
|
(error: unknown) => {
|
|
|
|
|
this.ctx.logger.warn(`workflow-workerthread: child dispose failed: ${renderThrown(error)}`)
|
|
|
|
|
this.finishChild(callId)
|
|
|
|
|
},
|
|
|
|
|
)
|
|
|
|
|
this.childDisposals.set(callId, disposal)
|
|
|
|
|
}
|
|
|
|
|
return disposal
|
|
|
|
|
private disposeChild(callId: number, record: ChildRecord): Promise<void> {
|
|
|
|
|
if (record.disposal !== undefined) return record.disposal
|
|
|
|
|
record.disposal = Promise.resolve()
|
|
|
|
|
.then(() => record.run.dispose())
|
|
|
|
|
.catch((error: unknown) => {
|
|
|
|
|
this.ctx.logger.warn(`workflow-workerthread: child dispose failed: ${renderThrown(error)}`)
|
|
|
|
|
})
|
|
|
|
|
.then(() => { this.finishChild(callId) })
|
|
|
|
|
return record.disposal
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** Drop a child from the registry (and its disposal memo), releasing quiescence waiters at zero. */
|
|
|
|
|
/** Drop a child record and release quiescence waiters when all work ends. */
|
|
|
|
|
private finishChild(callId: number): void {
|
|
|
|
|
this.children.delete(callId)
|
|
|
|
|
this.childDisposals.delete(callId)
|
|
|
|
|
if (this.children.size === 0) {
|
|
|
|
|
for (const waiter of this.quiescenceWaiters.splice(0)) waiter()
|
|
|
|
|
}
|
|
|
|
|
this.notifyChildQuiescence()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** Resolves once the child registry is empty (every disposal settled). */
|
|
|
|
|
/** Retire one provider startup transaction. */
|
|
|
|
|
private finishPendingStart(task: Promise<void>): void {
|
|
|
|
|
this.pendingStarts.delete(task)
|
|
|
|
|
this.notifyChildQuiescence()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** Release waiters only after both pending starts and published children end. */
|
|
|
|
|
private notifyChildQuiescence(): void {
|
|
|
|
|
if (this.children.size !== 0 || this.pendingStarts.size !== 0) return
|
|
|
|
|
for (const waiter of this.quiescenceWaiters.splice(0)) waiter()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** Resolves once every pending start and published child has reached quiescence. */
|
|
|
|
|
private childQuiescence(): Promise<void> {
|
|
|
|
|
if (this.children.size === 0) return Promise.resolve()
|
|
|
|
|
if (this.children.size === 0 && this.pendingStarts.size === 0) return Promise.resolve()
|
|
|
|
|
return new Promise((resolve) => { this.quiescenceWaiters.push(resolve) })
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** Abort + dispose every registered child (worker death / final teardown); disposal is contained, not awaited. */
|
|
|
|
|
private reapChildren(reason: string): void {
|
|
|
|
|
this.controller.abort(this.cancelReason ?? reason)
|
|
|
|
|
for (const [callId, run] of [...this.children]) {
|
|
|
|
|
run.cancel(this.cancelReason ?? reason)
|
|
|
|
|
void this.disposeChild(callId, run)
|
|
|
|
|
this.abortChildren(this.cancelReason ?? reason)
|
|
|
|
|
for (const [callId, record] of [...this.children]) {
|
|
|
|
|
void this.disposeChild(callId, record)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** Abort the one canonical signal shared by pending and published children. */
|
|
|
|
|
private abortChildren(reason: string): void {
|
|
|
|
|
if (!this.controller.signal.aborted) this.controller.abort(reason)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private onResult(result: WorkflowResult): void {
|
|
|
|
|
// The worker's settle-reap already child-cancel()s every stray; this
|
|
|
|
|
// abort fires the seam signal too, for providers that only honor the
|
|
|
|
|
// request signal (both channels, on every path).
|
|
|
|
|
if (this.cancelReason === undefined) this.controller.abort('workflow settled')
|
|
|
|
|
if (this.cancelReason !== undefined && result.stopReason !== 'cancelled') {
|
|
|
|
|
// The owned worker session sends one Result. Keep a late duplicate or a
|
|
|
|
|
// Result queued behind another terminal source completely side-effect-free.
|
|
|
|
|
if (this.terminalClaimed) return
|
|
|
|
|
// First-wins is decided when the Result message reaches the host. If no
|
|
|
|
|
// external cancellation was already in flight, this result won. Reaping a
|
|
|
|
|
// stray child below may synchronously reenter cancel() through provider
|
|
|
|
|
// callbacks, but that internal post-result cleanup must not retroactively
|
|
|
|
|
// rewrite the worker result that arrived first.
|
|
|
|
|
const cancellationWasRequested = this.cancelReason !== undefined
|
|
|
|
|
// Claim before settlement cleanup invokes provider disposal. Once Result
|
|
|
|
|
// won, a later cancellation cannot rewrite it.
|
|
|
|
|
this.terminalClaimed = true
|
|
|
|
|
// Abort pending starts and begin disposing published children before the
|
|
|
|
|
// workflow becomes externally settled. Cleanup remains independently
|
|
|
|
|
// tracked by childQuiescence and the holder's dispose().
|
|
|
|
|
this.reapChildren('workflow settled')
|
|
|
|
|
if (!cancellationWasRequested) {
|
|
|
|
|
this.settleResult(result)
|
|
|
|
|
return
|
|
|
|
|
}
|
|
|
|
|
if (result.stopReason !== 'cancelled') {
|
|
|
|
|
// The script settled while our cancel was crossing the thread boundary
|
|
|
|
|
// — the seam-visible result had NOT settled when cancellation was
|
|
|
|
|
// requested, so report cancelled (the vm drive()'s post-settle check,
|
|
|
|
|
@@ -344,21 +541,39 @@ export class WorkerRun implements WorkflowRun {
|
|
|
|
|
this.settleResult(result)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** An unexpected worker death (or the expected exit after termination). */
|
|
|
|
|
private onWorkerDeath(message: string): void {
|
|
|
|
|
// Whatever the worker left behind must not leak — abort + dispose it all.
|
|
|
|
|
if (this.children.size > 0) this.reapChildren('workflow worker gone')
|
|
|
|
|
// The thread is gone: no more worker-authored agent-ends can arrive —
|
|
|
|
|
// pair every stranded start (a start that crossed between the grace
|
|
|
|
|
// force-settle and this exit included) before the run settles.
|
|
|
|
|
this.endStrandedAgents()
|
|
|
|
|
// settleResult no-ops on an already-settled run (the expected exit after
|
|
|
|
|
// a dispose's terminate lands here too).
|
|
|
|
|
if (this.cancelReason !== undefined) {
|
|
|
|
|
this.settleResult(this.cancelledResult(this.hostStarted))
|
|
|
|
|
return
|
|
|
|
|
/** Process an error/messageerror/exit signal; `exit` also performs the final disposal sweep. */
|
|
|
|
|
private onWorkerDeath(message: string, isExit: boolean): void {
|
|
|
|
|
if (!this.workerDeathObserved) {
|
|
|
|
|
// Close message admission BEFORE cleanup callbacks: Node can deliver a
|
|
|
|
|
// message queued before the crash after its `error` event. Treating the
|
|
|
|
|
// first death signal as a logical barrier prevents that late message
|
|
|
|
|
// from creating work or narrating after workflow/end.
|
|
|
|
|
this.workerDeathObserved = true
|
|
|
|
|
const outcomeWasClaimed = this.terminalClaimed
|
|
|
|
|
const cancellationWasRequested = this.cancelReason !== undefined
|
|
|
|
|
// When death is itself the terminal source, claim BEFORE child reap or
|
|
|
|
|
// synthesized observer callbacks. Either can reenter cancel(); a death
|
|
|
|
|
// that arrived first remains an error, while a cancellation already
|
|
|
|
|
// accepted before death remains cancelled. If Result/grace already won,
|
|
|
|
|
// preserve it while still performing prompt failure-time cleanup.
|
|
|
|
|
if (!outcomeWasClaimed) this.terminalClaimed = true
|
|
|
|
|
if (this.children.size > 0 || this.pendingStarts.size > 0) this.reapChildren('workflow worker gone')
|
|
|
|
|
this.endStrandedAgents()
|
|
|
|
|
if (!outcomeWasClaimed) {
|
|
|
|
|
if (cancellationWasRequested) {
|
|
|
|
|
this.settleResult(this.cancelledResult(this.hostStarted))
|
|
|
|
|
} else {
|
|
|
|
|
this.settleResult({ value: null, stopReason: 'error', error: message, agentsStarted: this.hostStarted })
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
this.settleResult({ value: null, stopReason: 'error', error: message, agentsStarted: this.hostStarted })
|
|
|
|
|
if (!isExit) return
|
|
|
|
|
// `error` is not Node's physical delivery barrier: a queued message may
|
|
|
|
|
// precede `exit`. Admission is already closed, so this final sweep only
|
|
|
|
|
// joins/starts disposal for registry survivors; it deliberately does not
|
|
|
|
|
// repeat explicit provider cancellation.
|
|
|
|
|
for (const [callId, record] of [...this.children]) void this.disposeChild(callId, record)
|
|
|
|
|
this.endStrandedAgents()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
@@ -377,11 +592,14 @@ export class WorkerRun implements WorkflowRun {
|
|
|
|
|
/**
|
|
|
|
|
* Synthesize the missing `agent-end` for every started-but-unpaired agent,
|
|
|
|
|
* outcome `'cancelled'`: the reap cancels every child, and a real
|
|
|
|
|
* settlement racing the force-settle loses to the cancellation — the same
|
|
|
|
|
* first-wins override {@link onResult} applies to the run's own result.
|
|
|
|
|
* settlement racing the force-settle loses to that already-started external
|
|
|
|
|
* cancellation. The atomic terminal boundaries in {@link onResult} and
|
|
|
|
|
* {@link onWorkerDeath} deliberately exclude teardown callbacks as contenders.
|
|
|
|
|
* Called where the worker can no longer speak (the grace force-settle,
|
|
|
|
|
* worker death), BEFORE settleResult, so the paired ends reach observers
|
|
|
|
|
* before `workflow/end`.
|
|
|
|
|
* worker death, physical exit). When grace/death is the terminal source it
|
|
|
|
|
* runs before settleResult, so already-known pairs precede `workflow/end`;
|
|
|
|
|
* after an earlier Result, exit cleanup may close a survivor afterward.
|
|
|
|
|
* The ledger preserves exactly-once pairing in both orders.
|
|
|
|
|
*/
|
|
|
|
|
private endStrandedAgents(): void {
|
|
|
|
|
for (const info of [...this.liveAgents.values()]) {
|
|
|
|
|
@@ -397,10 +615,25 @@ export class WorkerRun implements WorkflowRun {
|
|
|
|
|
return { value: null, stopReason: 'cancelled', error: `workflow run cancelled: ${reason}`, agentsStarted }
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** First settle wins; disarms the grace timer. */
|
|
|
|
|
/** Remove the exact abort callback installed on the caller's start signal. */
|
|
|
|
|
private detachInputSignal(): void {
|
|
|
|
|
const signal = this.inputSignal
|
|
|
|
|
const onAbort = this.inputSignalAbort
|
|
|
|
|
if (signal === undefined || onAbort === undefined) return
|
|
|
|
|
this.inputSignal = undefined
|
|
|
|
|
this.inputSignalAbort = undefined
|
|
|
|
|
signal.removeEventListener('abort', onAbort)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** First settle wins; disarms the grace timer and releases the caller signal. */
|
|
|
|
|
private settleResult(result: WorkflowResult): void {
|
|
|
|
|
// Every current terminal source claims ownership before calling here; keep
|
|
|
|
|
// the fallback local so a future caller cannot resolve twice.
|
|
|
|
|
/* v8 ignore next -- defensive fallback outside the claimed state machine */
|
|
|
|
|
if (this.settled) return
|
|
|
|
|
this.terminalClaimed = true
|
|
|
|
|
this.settled = true
|
|
|
|
|
this.detachInputSignal()
|
|
|
|
|
clearTimeout(this.graceTimer)
|
|
|
|
|
this.settleResolve(result)
|
|
|
|
|
}
|
|
|
|
|
|