diff --git a/AGENTS.md b/AGENTS.md index eb34d404a0..5602b244ce 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,7 +18,7 @@ packages/ Harness packages at packages///, all named @deepseek-ai web/ web seam + search/fetch providers + model-facing web tools compact/ compaction seam + basic backend subagent/ subagent seam + spawn/fork/ACP backends + delegation tool - workflow/ workflow seam + node:vm script engine + the workflow tool + workflow/ workflow seam + worker-thread script engine + the workflow tool todo/ the todo_write tool hooks/ Claude Code / Codex hook bridges + shared wire-protocol library session-persistence/ persistence seam + JSONL/SQLite backends diff --git a/docs/capability-seams.md b/docs/capability-seams.md index caa9905e46..1a13bd2060 100644 --- a/docs/capability-seams.md +++ b/docs/capability-seams.md @@ -145,6 +145,6 @@ flowchart LR | `ctx.compact` | `seam` | [`compact`](../packages/compact/compact) | [`compact-basic`](../packages/compact/compact-basic) | [`compact-basic`](../packages/compact/compact-basic) | - | The basic backend currently consumes the pre-step event directly; a model-facing compact tool remains deferred. | | `ctx.subagents` | `seam` | [`subagent`](../packages/subagent/subagent) | [`subagent-spawn`](../packages/subagent/subagent-spawn), [`subagent-fork`](../packages/subagent/subagent-fork), [`subagent-acp`](../packages/subagent/subagent-acp), [`subagent-mock`](../packages/support/subagent-mock) | [`tool-subagent`](../packages/subagent/tool-subagent) | - | Providers implement transports; tool-subagent exposes one configured provider as a model-facing tool name. | | `ctx.web` | `seam` | [`web`](../packages/web/web) | [`web-search-exa`](../packages/web/web-search-exa), [`web-search-perplexity`](../packages/web/web-search-perplexity), [`web-search-deepseek`](../packages/web/web-search-deepseek), [`web-fetch-local`](../packages/web/web-fetch-local) | [`tool-web`](../packages/web/tool-web) | - | Search and fetch providers register into one ctx.web seam; tool-web owns the stable model-facing names. | -| `ctx.workflows` | `seam` | [`workflow`](../packages/workflow/workflow) | [`workflow-vm`](../packages/workflow/workflow-vm) | [`tool-workflow`](../packages/workflow/tool-workflow) | - | One engine per context (bash shape, no named-provider registry); the vm engine fans agent() calls out through ctx.subagents. | +| `ctx.workflows` | `seam` | [`workflow`](../packages/workflow/workflow) | [`workflow-vm`](../packages/workflow/workflow-vm) | [`tool-workflow`](../packages/workflow/tool-workflow) | - | One engine per context (bash shape, no named-provider registry); the worker-thread engine fans agent() calls out through ctx.subagents. | Maintenance mode: hybrid: services are discovered from Cordis declarations; interface/implementation/consumer roles are classified in `scripts/gen-doc-graphs.ts` with a completeness guard. diff --git a/docs/config-catalog.md b/docs/config-catalog.md index 8ffaefdabd..300468dd70 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -815,18 +815,18 @@ export interface Config { maxTotalAgents?: number /** Items accepted by a single `parallel()`/`pipeline()` call (default 4096). */ maxItemsPerCall?: number - /** vm timeout for the script's initial synchronous slice AND the meta-literal evaluation (default 5000 ms). */ + /** vm timeout for the initial synchronous slice (inside the worker) AND the host-side meta evaluation (default 5000 ms). */ syncTimeoutMs?: number /** * How long after a cancellation an unsettled script may keep running before - * it is abandoned and `result` force-settles `cancelled` (default 5000 ms); - * also bounds `dispose()`. + * the run force-settles `cancelled` and its worker is TERMINATED (default + * 5000 ms); also bounds `dispose()`. */ disposeGraceMs?: number } ``` -Source: [`packages/workflow/workflow-vm/src/index.ts:58`](../packages/workflow/workflow-vm/src/index.ts) +Source: [`packages/workflow/workflow-vm/src/index.ts:69`](../packages/workflow/workflow-vm/src/index.ts) ## Loadable plugins with no config diff --git a/docs/cordis-catalog/services.md b/docs/cordis-catalog/services.md index 027fa054c1..13c9f57733 100644 --- a/docs/cordis-catalog/services.md +++ b/docs/cordis-catalog/services.md @@ -238,12 +238,13 @@ Semantics every implementation must honor: - start throws synchronously for a request that cannot begin (an unparseable script, an invalid meta block). Once it returns a WorkflowRun, `result` NEVER rejects — every failure resolves with `stopReason: 'error'` (or `'cancelled'`) — and once the run is cancelled, `result` SETTLES within the implementation's bounded grace even if the script itself never settles (a consumer awaiting `result` must never be wedged past a cancellation). - The `workflow/*` events fire through emitWorkflowEvent (data snapshots, per-listener containment); `workflow/end` fires exactly once per started run, after `result` is settled or as it settles. - `dispose()` reaches quiescence within a bounded grace: it cancels, waits for the script to settle AND its started children to finish disposing, and abandons whatever is left rather than hanging its caller (the engine documents what abandonment leaves behind). +- Runs are HOLDER-OWNED: the engine hands control (`cancel`/`dispose`) to the `start()` caller and does not track its live runs — disposing the engine's own fiber mid-run deliberately leaves those runs to their holders' teardown, so an engine reload cannot yank a run out from under the consumer awaiting it. ```ts cordis-catalog abstract start(request: WorkflowStartRequest): WorkflowRun ``` -Source: [`packages/workflow/workflow/src/index.ts:202`](../../packages/workflow/workflow/src/index.ts) +Source: [`packages/workflow/workflow/src/index.ts:207`](../../packages/workflow/workflow/src/index.ts) ## Inherited `ctx` members (cordis core + loader/hmr/timer) diff --git a/docs/core-data-structures/core.md b/docs/core-data-structures/core.md index 7d1110d7a8..2f1e5031e9 100644 --- a/docs/core-data-structures/core.md +++ b/docs/core-data-structures/core.md @@ -24,6 +24,7 @@ Everything else is documented on a **sub-page**, not here. The rule that draws t | [compaction.md](compaction.md) | the compaction seam: the `compact/*` session events, `CompactionResult`, the `CompactService` interface | | [subagent.md](subagent.md) | the subagent seam: the named-provider registry, `SubagentStartRequest`/`Result`/`Run`, the start-time-vs-runtime capability split | | [web.md](web.md) | the web access seam: `WebSearchRequest`/`Result`, `WebFetchRequest`/`Result`, `WebFetchBody`, provider/capability status, `WebError` | +| [workflow.md](workflow.md) | the workflow seam: `WorkflowStartRequest`, `WorkflowMeta`, `WorkflowRun`/`Result`, the `workflow/*` event payloads, `WorkflowError` fatality | > Type definitions on this page are pasted **verbatim** from source and drift-checked by `pnpm run verify-type-equiv` (see [development.md](../development.md#documenting-types-verbatim-ts-type-equiv)). Inline JSDoc is omitted for readability; follow the source link for the full contracts. diff --git a/docs/core-data-structures/workflow.md b/docs/core-data-structures/workflow.md index 8c7916a6c3..4d880ea948 100644 --- a/docs/core-data-structures/workflow.md +++ b/docs/core-data-structures/workflow.md @@ -2,7 +2,7 @@ The workflow seam — an agent running a model-written orchestration SCRIPT that fans out subagents. Like [subagent](subagent.md) it is **one optional capability**, not part of the agent-loop spine, so its vocabulary lives here rather than in [core.md](core.md). Unlike the subagent registry it takes the bash shape: ONE engine implementation per context provides `ctx.workflows`; there is no named-provider registry (a second engine is a plugin swap, not a co-resident). -Interface: [dsh-workflow](../../packages/workflow/workflow) (`ctx.workflows` + the vocabulary below). The implementation is [dsh-workflow-vm](../../packages/workflow/workflow-vm) (an in-process `node:vm` engine); the model-facing consumer is [dsh-tool-workflow](../../packages/workflow/tool-workflow). The proposal and rationale: [the dynamic-workflows RFC](../rfc/implemented/feature/2026-07-05-dynamic-workflows.md). +Interface: [dsh-workflow](../../packages/workflow/workflow) (`ctx.workflows` + the vocabulary below). The implementation is [dsh-workflow-vm](../../packages/workflow/workflow-vm) (a `node:worker_threads` engine — one worker per run, the script's vm context inside it); the model-facing consumer is [dsh-tool-workflow](../../packages/workflow/tool-workflow). The proposal and rationale: [the dynamic-workflows RFC](../rfc/implemented/feature/2026-07-05-dynamic-workflows.md). Source: [`packages/workflow/workflow/src/types.ts`](../../packages/workflow/workflow/src/types.ts) @@ -47,7 +47,7 @@ interface WorkflowResult { ## A live run: `WorkflowRun` -The handle the consumer holds while a script executes. The consumer awaits `result`, may `cancel` mid-flight, and MUST `dispose` on every path. `result` does NOT reject — a script failure resolves with `stopReason: 'error'` — and once the run is cancelled it SETTLES within the engine's bounded grace even if the script itself never settles (the engine abandons the script and reports `cancelled`), so a consumer awaiting `result` is never wedged past a cancellation. `dispose()` = cancel + that bounded settle + child quiescence (the engine documents what abandonment leaves behind); it never hangs on a stuck script. +The handle the consumer holds while a script executes. The consumer awaits `result`, may `cancel` mid-flight, and MUST `dispose` on every path. `result` does NOT reject — a script failure resolves with `stopReason: 'error'` — and once the run is cancelled it SETTLES within the engine's bounded grace even if the script itself never settles (the engine force-settles `cancelled`; the worker-thread engine then terminates the script's worker), so a consumer awaiting `result` is never wedged past a cancellation. `dispose()` = cancel + that bounded settle + child quiescence; it never hangs on a stuck script. ```ts type-equiv interface WorkflowRun { diff --git a/docs/event-producer-consumer.md b/docs/event-producer-consumer.md index ec1fc468e3..a8618022a5 100644 --- a/docs/event-producer-consumer.md +++ b/docs/event-producer-consumer.md @@ -34,11 +34,11 @@ This matrix shows which packages dispatch each harness-owned event and which pac | `tools/change` | `emit` | [`packages/core/tools/src/index.ts:97`](../packages/core/tools/src/index.ts) | [`tools`](../packages/core/tools) (`emit`) | - | | `tools/post-execute` | `waterfall` | [`packages/core/tools/src/index.ts:92`](../packages/core/tools/src/index.ts) | [`tools`](../packages/core/tools) (`waterfall`) | [`hooks-claude`](../packages/hooks/hooks-claude), [`hooks-codex`](../packages/hooks/hooks-codex) | | `tools/pre-execute` | `waterfall` | [`packages/core/tools/src/index.ts:76`](../packages/core/tools/src/index.ts) | [`tools`](../packages/core/tools) (`waterfall`) | [`hooks-claude`](../packages/hooks/hooks-claude), [`hooks-codex`](../packages/hooks/hooks-codex) | -| `workflow/agent-end` | `emit` | [`packages/workflow/workflow/src/index.ts:93`](../packages/workflow/workflow/src/index.ts) | - | - | -| `workflow/agent-start` | `emit` | [`packages/workflow/workflow/src/index.ts:85`](../packages/workflow/workflow/src/index.ts) | - | - | -| `workflow/end` | `emit` | [`packages/workflow/workflow/src/index.ts:103`](../packages/workflow/workflow/src/index.ts) | - | - | -| `workflow/log` | `emit` | [`packages/workflow/workflow/src/index.ts:77`](../packages/workflow/workflow/src/index.ts) | - | - | -| `workflow/phase` | `emit` | [`packages/workflow/workflow/src/index.ts:70`](../packages/workflow/workflow/src/index.ts) | - | - | -| `workflow/start` | `emit` | [`packages/workflow/workflow/src/index.ts:62`](../packages/workflow/workflow/src/index.ts) | - | - | +| `workflow/agent-end` | `emit` | [`packages/workflow/workflow/src/index.ts:93`](../packages/workflow/workflow/src/index.ts) | [`workflow`](../packages/workflow/workflow) (`events.dispatch`) | - | +| `workflow/agent-start` | `emit` | [`packages/workflow/workflow/src/index.ts:85`](../packages/workflow/workflow/src/index.ts) | [`workflow`](../packages/workflow/workflow) (`events.dispatch`) | - | +| `workflow/end` | `emit` | [`packages/workflow/workflow/src/index.ts:103`](../packages/workflow/workflow/src/index.ts) | [`workflow`](../packages/workflow/workflow) (`events.dispatch`) | - | +| `workflow/log` | `emit` | [`packages/workflow/workflow/src/index.ts:77`](../packages/workflow/workflow/src/index.ts) | [`workflow`](../packages/workflow/workflow) (`events.dispatch`) | - | +| `workflow/phase` | `emit` | [`packages/workflow/workflow/src/index.ts:70`](../packages/workflow/workflow/src/index.ts) | [`workflow`](../packages/workflow/workflow) (`events.dispatch`) | - | +| `workflow/start` | `emit` | [`packages/workflow/workflow/src/index.ts:62`](../packages/workflow/workflow/src/index.ts) | [`workflow`](../packages/workflow/workflow) (`events.dispatch`) | - | Maintenance mode: hybrid generated: Cordis event declarations and most producer/listener edges are AST-scanned; dynamic dispatch sites are classified in `scripts/gen-doc-graphs.ts`. diff --git a/docs/rfc/implemented/feature/2026-07-05-dynamic-workflows.md b/docs/rfc/implemented/feature/2026-07-05-dynamic-workflows.md index f7749efa56..d0e436b17e 100644 --- a/docs/rfc/implemented/feature/2026-07-05-dynamic-workflows.md +++ b/docs/rfc/implemented/feature/2026-07-05-dynamic-workflows.md @@ -20,19 +20,19 @@ One deliberate strictness DIVERGENCE from CC: hook misuse — unknown or deferre `ctx.workflows` is an abstract `WorkflowService` in the bash shape — one engine per context, no named-provider registry (engines are deployment swaps, not co-residents). `start(request)` throws synchronously for a script that cannot begin; a returned `WorkflowRun`'s `result` NEVER rejects (failures resolve as `stopReason: 'error' | 'cancelled'`). The `workflow/*` events are observe-only emits carrying DATA SNAPSHOTS (id + meta; `workflow/end` omits the result value), per-listener contained, mirroring `subagent/start`/`subagent/end` — control stays with the run's holder. Vocabulary details: [core-data-structures/workflow.md](../../../core-data-structures/workflow.md). -### The engine (dsh-workflow-vm): in-process node:vm +### The engine (dsh-workflow-vm): one worker thread per run -**Trust premise (governs every engine decision below)**: workflow scripts are MODEL-WRITTEN — the same trust level as the model's existing bash access — so the engine defends against BUGGY scripts, never hostile ones. In scope: `result` never rejects, no unhandled rejections from dropped hook promises, loud rejection of values JSON cannot carry, fatal-vs-null hook discipline, cancellation that always frees the caller. Out of scope, deliberately: adversarial values (throwing/spinning accessors, proxies with hostile traps, prototype forgery, `prepareStackTrace` hijack) AND Node-API escape from the context — the vm context shares object machinery with the host, so a script can reach the host `Function` constructor (`globalThis.constructor.constructor`) and from it `process` and every Node builtin; the absent globals are API surface, not containment. Host code MAY run script code while reading script values, and that is accepted, because a hostile script can already occupy the event loop forever with a synchronous spin past its first await; containing its error VALUES while conceding it the event loop would be cost without a threat model. Genuine hardening is an engine swap behind the seam (worker/isolated-vm gets value isolation by serialization for free), not incremental host-side defenses. +**Trust premise (governs every engine decision below)**: workflow scripts are MODEL-WRITTEN — the same trust level as the model's existing bash access — so the engine defends against BUGGY scripts, never hostile ones. In scope: `result` never rejects, no unhandled rejections from dropped hook promises, loud rejection of values JSON cannot carry, fatal-vs-null hook discipline, cancellation that always frees the caller. Out of scope, deliberately: adversarial values (throwing/spinning accessors, proxies with hostile traps, prototype forgery, `prepareStackTrace` hijack) AND Node-API escape from the script's context — the vm context shares object machinery with its surrounding realm, so a script can reach the `Function` constructor (`globalThis.constructor.constructor`) and from it `process` and every Node builtin; the absent globals are API surface, not containment, and a worker thread is NOT a security boundary (an escapee holds process-wide privileges — Node's permission model is per-process). Worker-side code MAY run script code while reading script values, and that is accepted: a synchronous spin costs the script its OWN thread (terminated at the post-cancel grace), never the host loop, so containing error VALUES would be cost without a threat model. Genuine sandboxing (isolated-vm, a separate process) remains an engine swap behind the seam, not incremental defenses here. -**Why node:vm and not isolated-vm/worker threads**: isolated-vm is in maintenance mode, needs `--no-node-snapshot` on EVERY consumer process (including the published bins) on Node ≥ 20, and falls back to node-gyp source builds; a worker-thread engine turns every hook into RPC and complicates the per-file coverage gate. Under the trust premise, in-process is enough. Accepted, documented limitations: `start()` blocks the caller for the script's initial synchronous slice (bounded by the vm timeout); that timeout covers ONLY the initial slice, so a synchronous spin past it (an await continuation, a thenable's `then` invoked by promise resolution — a returned thenable resolves per JavaScript semantics, which is what makes an un-awaited `return agent('x')` work — or script code the host runs while rendering a thrown value) cannot be killed in-process; `dispose()` cancels, waits a bounded grace for the script to settle and its children to finish disposing, then abandons. +**Why node:worker_threads**: one run = one worker thread, no pooling — a run is heavyweight (many children), so thread spin-up (~tens of ms) is noise. The script runs in a vm context INSIDE the worker, keeping the script-visible surface exactly the hook contract above (a bare worker realm would leak `setTimeout`/`fetch`/`process` as accidental API), and every `agent()` bridges to `ctx.subagents` by message-port RPC — children are I/O-bound LLM loops and stay on the host loop; the thread isolates the SCRIPT, the only part that can spin. What the thread buys: `start()` never blocks the host (an in-process engine runs the initial synchronous slice inline and cannot kill a spin past the first await — it could only ABANDON such a script, leaving the spin on the host loop), the post-cancel grace ends in a REAL `worker.terminate()`, and the value boundary is serialization by construction. isolated-vm was rejected for actual sandboxing: maintenance mode, `--no-node-snapshot` on EVERY consumer process (including published bins) on Node ≥ 20, node-gyp source-build fallback. Key mechanics (details in the package README): meta extraction and a body pre-parse stay HOST-side (preserving the seam's synchronous throws), a ready→go handshake keeps a run cancelled before start from ever executing the body, `cancel()` drives both child-cancel channels host-side (the shared request signal AND each child's explicit `cancel()` — a wedged worker cannot relay its own cancel RPCs), a host-side child registry backs worker-death reaping and `dispose()` quiescence, the wire protocol is enum-keyed payload maps private to the package, and on a termination path `agentsStarted` degrades to the host-observed count. Coverage puts the worker-side session on an in-process `MessageChannel` (real-Worker code is invisible to main-process v8) and proves the built `lib/worker.js` — a second tsdown entry, sanctioned in the workspace-constraints gate by the `"./worker"` subpath export — under plain node in the built-bin smoke gate. **Meta extraction**: a string/comment-aware brace scanner (template interpolation rejected) finds the literal; it is evaluated ALONE in an empty, timed vm context; the result must materialize to plain JSON data and pass shape validation (unknown fields rejected loud); the statement is blanked line-preservingly so stacks keep script line numbers. -**Value boundary**: values entering the host (meta, hook options, schemas, the return value) go through `materializeFromRealm` — a plain recursive walk that rejects loud everything JSON cannot carry (exotic prototypes, functions, symbols, cycles, sparse arrays, non-finite numbers, nested `undefined`), copying via `Object.defineProperty` so a `"__proto__"` key becomes a data property, never a prototype mutation; getters are read ordinarily and their RESULT crosses (a throwing read fails loud). Values entering the realm (`args`, `agent()` results, hook promises and failures, combinator arrays) are handed over directly as host values — the script is trusted, so host prototypes are not a leak; `args` is host-`structuredClone`d once so a script cannot mutate the caller's object. Hook failures are host `WorkflowError`s: the combinators recognize fatality by host `instanceof` (unforgeable from the realm), and the script-visible consequence — in-script `instanceof Error` is `false` for hook errors; branch on `e.name`/`e.code` — is documented in the engine README. Realm functions (stages, thunks) are called, never materialized. Thrown script values are rendered by a total host-side renderer (stack → message → `String()`, fixed label if rendering throws), so `result` cannot reject. Caps (`maxConcurrentAgents` auto = `min(16, max(1, availableParallelism() - 2))`, `maxTotalAgents` 1000, `maxItemsPerCall` 4096) and timeouts are validated Config, not literals. +**Value boundary**: values leaving the script (meta, hook options, schemas, the return value) go through `materializeFromRealm` — a plain recursive walk that rejects loud everything JSON cannot carry (exotic prototypes, functions, symbols, cycles, sparse arrays, non-finite numbers, nested `undefined`), copying via `Object.defineProperty` so a `"__proto__"` key becomes a data property, never a prototype mutation; getters are read ordinarily and their RESULT crosses (a throwing read fails loud) — which is also what makes every later postMessage hop total. Values entering the realm (`args`, `agent()` results, hook promises and failures, combinator arrays) are handed over directly as worker-realm values — the script is trusted, so outer prototypes are not a leak; `args` rides the `workerData` structured clone (the caller-isolation copy) and is cloned once more so a script scribbling on it cannot mutate the session's init object. Hook failures are `WorkflowError`s built OUTSIDE the script's context: the combinators recognize fatality by `instanceof` against the engine's own class (unforgeable from the script), and the script-visible consequence — in-script `instanceof Error` is `false` for hook errors; branch on `e.name`/`e.code` — is documented in the engine README. Realm functions (stages, thunks) are called, never materialized. Thrown script values are rendered by a total renderer (stack → message → `String()`, fixed label if rendering throws), so `result` cannot reject. Caps (`maxConcurrentAgents` auto = `min(16, max(1, availableParallelism() - 2))`, `maxTotalAgents` 1000, `maxItemsPerCall` 4096) and timeouts are validated Config, not literals. ### The consumer (dsh-tool-workflow) -A `workflow` tool mirroring `dsh-tool-subagent`'s synchronous shape: start, await, `try/finally` dispose, abort-bridge `exec.signal`, non-`completed` → `isError`. Render intent: a `generic` card titled by a textual `meta.name` sniff (presentation is a pure function of args). The tool description IS the model-facing authoring spec. Examples load it with guidance to use workflows only on explicit user request — the harness has no ultracode-style effort gate. +A `workflow` tool mirroring `dsh-tool-subagent`'s synchronous shape: start, await, `try/finally` dispose, abort-bridge `exec.signal`, non-`completed` → `isError`. Render intent: a `generic` card titled by a textual `meta.name` sniff (presentation is a pure function of args). The tool description IS the model-facing authoring spec. The usage policy ships with the tool as its own `tool:` prompt section (explicit-ask-only guidance — tool guidance lives in tool plugins, never in the deployment persona); the harness has no ultracode-style effort gate. ### The foundation: structured output on the subagent seam @@ -45,13 +45,14 @@ A `workflow` tool mirroring `dsh-tool-subagent`'s synchronous shape: start, awai - **Saved/bundled workflows** (a `.deepseek/workflows/` registry, slash-command surface) and **script persistence to a run directory** (the tool-call event already records the script durably). - **Nested `workflow()`**, **token `budget`**, and the `effort`/`isolation`/`agentType` agent options (each rejects loud with a message naming it deferred). - **An overall run wall-clock timeout** — cancellation always frees the caller (result settles within the grace), so a cap on total run time is a policy knob for the background redesign, not a correctness need here. -- **Engine hardening**: a worker-thread or isolated-vm engine behind the same seam (kills synchronous spins; adds memory limits). +- **Engine hardening beyond worker threads**: an isolated-vm or separate-process engine behind the same seam (actual sandboxing; memory limits). - **ACP progress UI** over the `workflow/*` events (a `/workflows`-style view); the events exist for it. - **ACP-backend structured output** and **`toolFilter`** (both still capability-gated `false`). ## Alternatives considered -- **Hostile-value containment in the host** (trap-free proxy rejection, accessor-never-invoked descriptor walks, realm-side pre-rendering of thrown values, realm-built promises/arrays/error clones with structural fatal recognition): an earlier revision built all of it, and review showed the cost was real while the threat model was not — every one of those defenses guards against an author the premise already trusts, who retains an accepted unkillable event-loop spin regardless. Removed in favor of the plain boundary above; the hardened engine deletes such machinery anyway (serialization by construction). +- **Hostile-value containment in the host** (trap-free proxy rejection, accessor-never-invoked descriptor walks, realm-side pre-rendering of thrown values, realm-built promises/arrays/error clones with structural fatal recognition): an earlier revision built all of it, and review showed the cost was real while the threat model was not — every one of those defenses guards against an author the premise already trusts. Removed in favor of the plain boundary above; the thread boundary makes such machinery redundant anyway (serialization by construction). +- **In-process `node:vm` execution** (the first cut of this RFC shipped it): mechanically simplest — no RPC, no thread — but `start()` blocks the caller for the script's initial synchronous slice, a synchronous spin past the first await cannot be killed in-process (the vm `timeout` covers only that first slice), and `dispose()` could only ABANDON an unsettling script, leaving the spin on the host loop. Superseded by the worker-thread engine, which keeps the same vm-context script surface while unblocking the host and making termination real. - **Background execution as the default** (CC's shape): deferred; foreground-synchronous matches `dsh-tool-subagent`'s cut, and background semantics should be designed ONCE across bash/subagent/workflow rather than per-tool. - **Workflow-layer JSON parsing for `agent({schema})`**: duplicating a seam concern at one consumer while the seam's capability flag stayed dishonestly `false`. - **Meta as tool parameters instead of `export const meta`**: zero parsing, but scripts stop being self-contained artifacts and CC-authored scripts stop being drop-in. @@ -62,4 +63,4 @@ A `workflow` tool mirroring `dsh-tool-subagent`'s synchronous shape: start, awai ## Consequences -The harness gains CC-compatible script orchestration: fan-out plans live in a rerunnable artifact instead of the parent context, and the structured-output half of the subagent seam is now real (the vocabulary stopped lying about `outputSchema`). What it cost, all bounded by the trust premise: the in-process engine blocks its caller for a script's initial synchronous slice, cannot kill a synchronous spin past that slice, and does not isolate host values from the script — acceptable because scripts share the model's trust level, and each limitation names its exit (the engine swap behind the seam). The fatal-vs-null strictness divergence from CC means a CC-authored script that RELIES on option typos dissolving to `null` behaves differently here — judged worth it to keep the repo's no-accepted-then-ignored rule. Consumers must hold the run handle for control (`cancel`/`dispose`); observers get data snapshots only, so no listener can extend a run's lifetime or corrupt another's view. +The harness gains CC-compatible script orchestration: fan-out plans live in a rerunnable artifact instead of the parent context, and the structured-output half of the subagent seam is now real (the vocabulary stopped lying about `outputSchema`). What it cost, all bounded by the trust premise: a worker thread per run (~tens-of-ms spin-up), every hook crossing a message port as RPC, and a termination-path `agentsStarted` that degrades to the host-observed count; in exchange `start()` never blocks the host, a post-cancel grace ends in a real `worker.terminate()`, and the value boundary is serialization by construction. A worker thread is still NOT a security boundary — scripts share the model's trust level, and actual sandboxing names its exit (the isolated-vm/separate-process engine swap behind the seam). The fatal-vs-null strictness divergence from CC means a CC-authored script that RELIES on option typos dissolving to `null` behaves differently here — judged worth it to keep the repo's no-accepted-then-ignored rule. Consumers must hold the run handle for control (`cancel`/`dispose`); observers get data snapshots only, so no listener can extend a run's lifetime or corrupt another's view. diff --git a/examples/acp-agent/cordis.yml b/examples/acp-agent/cordis.yml index eeb78a7962..f2015bc7c2 100644 --- a/examples/acp-agent/cordis.yml +++ b/examples/acp-agent/cordis.yml @@ -82,10 +82,10 @@ toolName: subagent_fork -# Dynamic workflows: the node:vm engine (ctx.workflows) over the spawn subagent -# backend above, plus the model-facing `workflow` tool. The model writes a -# JavaScript orchestration script (meta + body); the engine runs it in-process -# and fans agent() calls out as spawn children. +# Dynamic workflows: the worker-thread engine (ctx.workflows) over the spawn +# subagent backend above, plus the model-facing `workflow` tool. The model +# writes a JavaScript orchestration script (meta + body); the engine runs it +# in its own worker thread and fans agent() calls out as spawn children. - id: workflow-vm name: '@deepseek-ai/dsh-workflow-vm' config: diff --git a/examples/coding-agent/cordis.yml b/examples/coding-agent/cordis.yml index 28643e0d73..ee55bced0b 100644 --- a/examples/coding-agent/cordis.yml +++ b/examples/coding-agent/cordis.yml @@ -103,10 +103,10 @@ toolName: subagent_fork -# Dynamic workflows: the node:vm engine (ctx.workflows) over the spawn subagent -# backend above, plus the model-facing `workflow` tool. The model writes a -# JavaScript orchestration script (meta + body); the engine runs it in-process -# and fans agent() calls out as spawn children. +# Dynamic workflows: the worker-thread engine (ctx.workflows) over the spawn +# subagent backend above, plus the model-facing `workflow` tool. The model +# writes a JavaScript orchestration script (meta + body); the engine runs it +# in its own worker thread and fans agent() calls out as spawn children. - id: workflow-vm name: '@deepseek-ai/dsh-workflow-vm' config: diff --git a/packages/README.md b/packages/README.md index 3c38eda671..43b446eeb5 100644 --- a/packages/README.md +++ b/packages/README.md @@ -14,7 +14,7 @@ Packages are grouped by modular role at `packages///`. The group dir | [`fs/`](fs/README.md) | Filesystem capability family: the abstract seam, a local impl, and the model-facing file tools | Product — stable surface | | [`compact/`](compact/README.md) | Compaction capability family: the abstract seam + a basic backend (tool deferred) | Product — stable surface | | [`subagent/`](subagent/README.md) | Subagent capability family: the provider-registry seam and the model-facing delegation tool | Product — stable surface | -| [`workflow/`](workflow/README.md) | Workflow capability family: the script-engine seam, the node:vm engine, and the model-facing `workflow` tool | Product — stable surface | +| [`workflow/`](workflow/README.md) | Workflow capability family: the script-engine seam, the worker-thread engine, and the model-facing `workflow` tool | Product — stable surface | | [`web/`](web/README.md) | Web capability family: the abstract seam, search/fetch provider impls, and the model-facing web tools | Product — stable surface | | [`todo/`](todo/README.md) | Todo/planning family: the model-facing `todo_write` tool (whole-list task tracking on the session log) | Product — stable surface | | [`hooks/`](hooks/README.md) | Hook bridges + the shared Claude Code / Codex wire-protocol library | Product — stable surface | diff --git a/packages/workflow/README.md b/packages/workflow/README.md index f28a325ea8..a0750d9fdf 100644 --- a/packages/workflow/README.md +++ b/packages/workflow/README.md @@ -5,9 +5,9 @@ The workflow seam: a model-written JavaScript orchestration script that fans out | Package | Role | ctx key | |---|---|---| | `workflow/` | Abstract workflow seam: service base class + run vocabulary + `workflow/*` events | `ctx.workflows` | -| `workflow-vm/` | In-process `node:vm` engine: parses the script, injects the hooks, drives `ctx.subagents` | (provides `ctx.workflows`) | +| `workflow-vm/` | `node:worker_threads` engine: one worker per run; the script's vm context lives inside the worker, `agent()` bridges to `ctx.subagents` over the message port | (provides `ctx.workflows`) | | `tool-workflow/` | Model-facing `workflow` tool over `ctx.workflows` | (registers on `ctx.tools`) | -The interface lives at `workflow/workflow/`. The engine's `agent()` hook rides the [subagent seam](../subagent/README.md) (any registered provider; the shipped examples use `spawn`), and `agent({ schema })` rides the structured-output support the in-process backends implement. The seam split exists for engine hardening: `node:vm` is in-process and cannot kill a pathological synchronous spin — a worker-thread or isolated-vm engine swaps in behind the same interface if that ever matters. +The interface lives at `workflow/workflow/`. The engine's `agent()` hook rides the [subagent seam](../subagent/README.md) (any registered provider; the shipped examples use `spawn`), and `agent({ schema })` rides the structured-output support the in-process backends implement. The worker thread isolates the SCRIPT — the host never blocks on it, and a cancelled run's post-grace termination is real — but it is NOT a security boundary; an isolated-vm/separate-process engine (actual sandboxing) swaps in behind the same interface if that ever matters. The proposal, decisions, and deferred work: [docs/rfc/implemented/feature/2026-07-05-dynamic-workflows.md](../../docs/rfc/implemented/feature/2026-07-05-dynamic-workflows.md). diff --git a/packages/workflow/workflow/README.md b/packages/workflow/workflow/README.md index 076f5443ac..96add5017e 100644 --- a/packages/workflow/workflow/README.md +++ b/packages/workflow/workflow/README.md @@ -1,10 +1,10 @@ # @deepseek-ai/dsh-workflow -The **workflow seam** (`ctx.workflows`): an abstract service defining WHAT a workflow engine does — execute a model-written orchestration script that fans out subagents — without saying HOW. The bash-shaped third of the [workflow family](../README.md): implementations subclass `WorkflowService` and register as the `workflows` service (one per context); [`dsh-workflow-vm`](../workflow-vm/README.md) is the first, and [`dsh-tool-workflow`](../tool-workflow/README.md) is the model-facing consumer. +The **workflow seam** (`ctx.workflows`): an abstract service defining WHAT a workflow engine does — execute a model-written orchestration script that fans out subagents — without saying HOW. The bash-shaped third of the [workflow family](../README.md): implementations subclass `WorkflowService` and register as the `workflows` service (one per context); [`dsh-workflow-vm`](../workflow-vm/README.md) (one worker thread per run) is the implementation, and [`dsh-tool-workflow`](../tool-workflow/README.md) is the model-facing consumer. ## Service: `WorkflowService` (abstract) -`start(request: WorkflowStartRequest): WorkflowRun` — parse and execute a script. Throws synchronously (`SCRIPT_PARSE`/`META_INVALID`) for a script that cannot begin; once a run is returned, its `result` NEVER rejects — every failure resolves with `stopReason: 'error'` (or `'cancelled'`) — and once the run is cancelled, `result` settles within the implementation's bounded grace even if the script itself never settles (a consumer awaiting `result` must never be wedged past a cancellation). `dispose()` must reach quiescence within a bounded grace (cancel → wait for the script to settle and its children to finish disposing → abandon), never hanging its caller. +`start(request: WorkflowStartRequest): WorkflowRun` — parse and execute a script. Throws synchronously (`SCRIPT_PARSE`/`META_INVALID`) for a script that cannot begin; once a run is returned, its `result` NEVER rejects — every failure resolves with `stopReason: 'error'` (or `'cancelled'`) — and once the run is cancelled, `result` settles within the implementation's bounded grace even if the script itself never settles (a consumer awaiting `result` must never be wedged past a cancellation). `dispose()` must reach quiescence within a bounded grace (cancel → wait for the script to settle and its children to finish disposing → abandon), never hanging its caller. Runs are HOLDER-owned: the engine does not track its live runs, so disposing the engine's fiber mid-run leaves each run to its holder's teardown. The protected `emitWorkflowEvent` helper dispatches the `workflow/*` events with PER-LISTENER containment and PER-LISTENER payload snapshots (a throwing subscriber is logged, never propagated, and cannot starve later listeners; each subscriber gets its own clone of the payload, so mutating it corrupts neither the engine nor other listeners) — the same containment guarantee as the subagent seam's lifecycle emits. @@ -22,7 +22,7 @@ All observe-only emits carrying DATA SNAPSHOTS (`WorkflowRunInfo` = id + meta) - `workflow/start`(info) / `workflow/end`(info, resultInfo) — run lifecycle; `resultInfo` deliberately omits the value. - `workflow/phase`(info, title) / `workflow/log`(info, message) — script narration. -- `workflow/agent-start`(info, agent) / `workflow/agent-end`(info, agent + outcome) — one pair per `agent()` call, correlated by `seq`. +- `workflow/agent-start`(info, agent) / `workflow/agent-end`(info, agent + outcome) — one pair per `agent()` call that STARTED a child run (a call rejected at validation or caps, refused at start, or cancelled while queued for a slot emits no pair), correlated by `seq`. ## Non-goals (this cut) diff --git a/packages/workflow/workflow/src/index.ts b/packages/workflow/workflow/src/index.ts index 0e5a03c438..2a57d143b6 100644 --- a/packages/workflow/workflow/src/index.ts +++ b/packages/workflow/workflow/src/index.ts @@ -4,10 +4,10 @@ * that fans out subagents — without saying HOW. Implementations subclass * {@link WorkflowService} and register as the `workflows` service (one * implementation per context, cordis' standard duplicate-service behavior); - * `@deepseek-ai/dsh-workflow-vm` (an in-process `node:vm` engine) is the - * first. Future engines (a worker-thread or isolated-vm sandbox) swap in - * without touching the model-facing tool that consumes them - * (`@deepseek-ai/dsh-tool-workflow`). + * the implementation is `@deepseek-ai/dsh-workflow-vm`, which runs each + * script in its own worker thread. Hardened engines (an isolated-vm or + * separate-process sandbox) swap in without touching the model-facing tool + * that consumes them (`@deepseek-ai/dsh-tool-workflow`). * * The `workflow/*` lifecycle events are OBSERVE-ONLY data snapshots: they * carry {@link WorkflowRunInfo} (id + meta), never the live {@link WorkflowRun} @@ -198,6 +198,11 @@ export function isFatalWorkflowError(error: unknown): boolean { * for the script to settle AND its started children to finish disposing, * and abandons whatever is left rather than hanging its caller (the engine * documents what abandonment leaves behind). + * - Runs are HOLDER-OWNED: the engine hands control (`cancel`/`dispose`) to + * the `start()` caller and does not track its live runs — disposing the + * engine's own fiber mid-run deliberately leaves those runs to their + * holders' teardown, so an engine reload cannot yank a run out from under + * the consumer awaiting it. */ export abstract class WorkflowService extends Service { constructor(ctx: Context) { diff --git a/packages/workflow/workflow/src/types.ts b/packages/workflow/workflow/src/types.ts index 767fb8257b..8552e32454 100644 --- a/packages/workflow/workflow/src/types.ts +++ b/packages/workflow/workflow/src/types.ts @@ -89,7 +89,13 @@ export interface WorkflowResult { stopReason: WorkflowStopReason /** The failure message (present iff `stopReason` is not `completed`). */ error?: string - /** How many `agent()` calls the run accepted (whole lifetime, including calls still queued for a slot when the run was cancelled). */ + /** + * How many `agent()` calls the run accepted over its whole lifetime. On a + * graceful settlement this is the script-side count (calls still queued for + * a concurrency slot included); on a termination path (grace force-settle, + * worker death) it degrades to the host-observed count — calls queued + * inside a terminated script are unknowable then. + */ agentsStarted: number } @@ -98,18 +104,19 @@ export interface WorkflowResult { * `result`, may `cancel` mid-flight, and MUST `dispose` on every path. * `result` does NOT reject — a script failure resolves with `stopReason: * 'error'` — and once the run is cancelled it SETTLES within the engine's - * bounded grace even if the script itself never settles (the engine abandons - * the script and reports `cancelled`), so a consumer awaiting `result` is - * never wedged past a cancellation. `dispose()` = cancel + that bounded - * settle + child quiescence; it never hangs on a stuck script and is safe to - * call on every path (idempotent). + * bounded grace even if the script itself never settles (the engine + * force-settles `cancelled`; what becomes of the script is engine-documented + * — the worker-thread engine terminates its worker), so a consumer awaiting + * `result` is never wedged past a cancellation. `dispose()` = cancel + that + * bounded settle + child quiescence; it never hangs on a stuck script and is + * safe to call on every path (idempotent). */ export interface WorkflowRun { readonly id: WorkflowRunId /** The validated meta block (available before the body runs). */ readonly meta: WorkflowMeta readonly result: Promise - /** Cancel the run: children abort, pending hooks reject, the script dies at its next await (or is abandoned at the grace). */ + /** Cancel the run: children abort, pending hooks reject, the script dies at its next await (or is force-settled at the grace). */ cancel(reason?: string): void /** Cancel + bounded-grace settle; safe to call on every path (idempotent). */ dispose(): Promise diff --git a/scripts/gen-doc-graphs.ts b/scripts/gen-doc-graphs.ts index 1fabb98707..0abe1a34cf 100644 --- a/scripts/gen-doc-graphs.ts +++ b/scripts/gen-doc-graphs.ts @@ -193,7 +193,7 @@ const SERVICE_ROLES: ServiceRole[] = [ mode: 'seam', implementations: ['workflow-vm'], consumers: ['tool-workflow'], - note: 'One engine per context (bash shape, no named-provider registry); the vm engine fans agent() calls out through ctx.subagents.', + note: 'One engine per context (bash shape, no named-provider registry); the worker-thread engine fans agent() calls out through ctx.subagents.', }, ] @@ -203,6 +203,14 @@ const DYNAMIC_EVENT_DISPATCHERS: Array<{ event: string; pkg: string; method: str // listeners or strand an already-started child run. { event: 'subagent/start', pkg: 'subagent', method: 'events.dispatch' }, { event: 'subagent/end', pkg: 'subagent', method: 'events.dispatch' }, + // The workflow/* lifecycle events dispatch the same way, for the same + // per-listener-containment reason (WorkflowService.emitWorkflowEvent). + { event: 'workflow/start', pkg: 'workflow', method: 'events.dispatch' }, + { event: 'workflow/phase', pkg: 'workflow', method: 'events.dispatch' }, + { event: 'workflow/log', pkg: 'workflow', method: 'events.dispatch' }, + { event: 'workflow/agent-start', pkg: 'workflow', method: 'events.dispatch' }, + { event: 'workflow/agent-end', pkg: 'workflow', method: 'events.dispatch' }, + { event: 'workflow/end', pkg: 'workflow', method: 'events.dispatch' }, ] function generatedHeader(title: string): string[] {