From a2c53a885b0c5ad6e93ea0d53baa9acf44cb68bf Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Mon, 6 Jul 2026 17:33:06 +0800 Subject: [PATCH 01/24] docs(rfc): propose tool result retention library --- docs/rfc/INDEX.md | 1 + ...026-07-06-tool-result-retention-library.md | 175 ++++++++++++++++++ 2 files changed, 176 insertions(+) create mode 100644 docs/rfc/proposed/architecture/2026-07-06-tool-result-retention-library.md diff --git a/docs/rfc/INDEX.md b/docs/rfc/INDEX.md index 5307c381c8..a7e0506cad 100644 --- a/docs/rfc/INDEX.md +++ b/docs/rfc/INDEX.md @@ -26,6 +26,7 @@ Generated by `pnpm run gen-rfc-index` from the RFC tree — never edit by hand; |---|---| | [Runtime schemas for the event vocabulary (Zod vs the merge-extensible-map pattern)](proposed/architecture/2026-06-16-typed-event-schemas.md) | 2026-06-16 | | [Extract a generic long-running tool runtime](proposed/architecture/2026-06-20-generic-long-running-tool-runtime.md) | 2026-06-20 | +| [Tool result retention library](proposed/architecture/2026-07-06-tool-result-retention-library.md) | 2026-07-06 | ### Process diff --git a/docs/rfc/proposed/architecture/2026-07-06-tool-result-retention-library.md b/docs/rfc/proposed/architecture/2026-07-06-tool-result-retention-library.md new file mode 100644 index 0000000000..b8f16f6bb1 --- /dev/null +++ b/docs/rfc/proposed/architecture/2026-07-06-tool-result-retention-library.md @@ -0,0 +1,175 @@ +# RFC: Tool result retention library + +Status: proposed + +## Problem + +Several model-facing tools already bound the amount of context they return, but each one owns a different local mechanism and vocabulary: bash keeps a tail plus spill files, web search caps source lists, web fetch caps body content, and `glob` / `grep` discovery needs `cap + 1` early stop while reading ripgrep output. A single post-hoc `truncate(text)` helper cannot cover those cases: by the time `grep` or `glob` has collected every result, the expensive traversal has already happened and the process may have emitted more output than the harness intended to buffer. + +The shared abstraction the tools need is **retention**, not generic collection. A caller feeds items or text chunks into a bounded object, receives a per-push decision about whether the upstream can stop, and later receives the retained content plus exact or partial omission metadata. Tool-specific code still owns business semantics: file grouping, line numbering, exit codes, provider error states, and model-facing prose. The common library owns only the mechanical question "what did we keep, what did we omit, and may the caller stop reading now?" + +## Proposal + +Add a small, dependency-light retention library under `packages/util/retention` (package name `@deepseek-ai/dsh-retention`). It exports pure item and text retainers plus notice helpers. It is not a Cordis service and registers no plugin; tool packages import it directly when they need bounded model-facing output. + +The library has two independent retainers: + +- `ItemRetainer` handles ordered logical units such as paths, grep matches, or search sources. It supports `head` retention only in v1. +- `TextRetainer` handles byte-oriented text streams such as bash stdout/stderr or web response bodies. It supports `head`, `tail`, and `headTail` retention while preserving UTF-8 boundaries at `finish()`. + +Both retainers return a `PushDecision` after each `push()`. `shouldStop` is the critical control-flow field: `glob` / `grep` use it to kill ripgrep once the probe item proves truncation, while bash ignores it because tail/head-tail retention must read to process exit to know the true suffix and to avoid pipe backpressure. + +```ts ignore-check +/** + * How much content the retainer omitted. + * + * `atLeast` is the early-stop shape: `glob` / `grep` see the first item past the cap, + * stop the upstream process, and know only that at least one item was omitted. + */ +type Omitted = + | { kind: 'none' } + | { kind: 'exact'; count: number } + | { kind: 'atLeast'; count: number } + | { kind: 'unknown' } + +/** + * The caller receives this after each `push()`. + * + * `shouldStop` is advisory, not automatic: the tool owns how to stop its upstream + * source, such as aborting an HTTP body, breaking a file scan, or killing ripgrep. + */ +interface PushDecision { + kept: boolean + truncated: boolean + shouldStop: boolean +} + +/** + * Final result for ordered logical units. + * + * `seen` means units observed by the retainer, not necessarily total units in the + * upstream source; with early stop, total is intentionally unknown. + */ +interface RetainedItems { + items: T[] + truncated: boolean + seen: number + kept: number + omitted: Omitted +} + +/** + * Final result for text streams. + * + * The returned `text` is safe to send to a formatter; the retainer does not add + * tool-specific headers, exit markers, XML tags, or recovery instructions. + */ +interface RetainedText { + text: string + truncated: boolean + omittedBytes: Omitted +} +``` + +### Strategies + +The strategy names are caller-facing and avoid implementation phrases such as "overflow". `stopWhenFull` means the retainer should ask the caller to stop once keeping more would exceed the budget. `readToEnd` means the retainer must keep accepting input even after the retained output is full, usually to preserve a true tail, count exact omission, or drain an upstream process. + +```ts ignore-check +type StopMode = 'stopWhenFull' | 'readToEnd' + +type ItemRetentionStrategy = + | { + /** Keep the first `maxItems` units. Use for `glob`, `grep`, and web sources. */ + kind: 'head' + maxItems: number + stop: StopMode + } + +type TextRetentionStrategy = + | { + /** Keep the first `maxBytes` bytes. May stop an upstream body early. */ + kind: 'head' + maxBytes: number + stop: StopMode + } + | { + /** Keep the final `maxBytes` bytes. Requires reading to the end. */ + kind: 'tail' + maxBytes: number + } + | { + /** Keep a stable prefix and suffix, omitting the middle. Requires reading to the end. */ + kind: 'headTail' + headBytes: number + tailBytes: number + } +``` + +### Tool mapping + +`read` is intentionally outside the v1 retention library. Its `read-render` helper owns a file-specific pagination contract: `offset` / `limit`, line numbers, `totalLines`, offset-out-of-range errors, per-line preview truncation, and a selected-output byte cap that can stop scanning mid-window. That is a line-window renderer, not a generic retention primitive. It may share future neutral notice helpers, but it should not pass its already-selected window through `ItemRetainer`. + +`FsGlobEntry` and `FlatGrepMatch` below are the intended discovery-tool item shapes, not existing retention-library exports. `FsGlobEntry` is one backend-derived path, and `FlatGrepMatch` is one ungrouped grep match before the backend groups retained matches by file. + +`glob` uses `ItemRetainer` with `{ kind: 'head', maxItems: globMaxResults, stop: 'stopWhenFull' }` inside the backend or executor that is consuming traversal output. The `(maxItems + 1)`th valid path is the probe item: it is not retained, it sets `truncated: true`, and `shouldStop: true` tells the caller to stop ripgrep, cancel a remote stream, or stop whatever upstream is producing candidates. `omitted` is `{ kind: 'atLeast', count: 1 }` because the traversal stopped before the full count was known. Path mapping, skipped candidates, and `incomplete` stay outside the retainer. + +`grep` uses `ItemRetainer` with `{ kind: 'head', maxItems: grepMaxMatches, stop: 'stopWhenFull' }` before grouping. The backend parses a ripgrep match record, maps the path, applies per-line preview truncation, then pushes a flat match. After `finish()`, the backend groups retained matches by file and sorts the returned subset. Grouping is not part of the retainer because the cap is total matches, not files; per-match preview truncation and `incomplete` are also separate from result-level retention. + +`bash` uses `TextRetainer` with `tail` or `headTail` and reads to process completion. It does not stop when full: stopping the read would lose the real tail and can create pipe backpressure. The bash executor still owns spill files, exit status, signal, timeout, and background-task behavior; the retention helper only replaces ad hoc in-memory head/tail accounting where that behavior is desired. Long-running task ownership remains orthogonal to the [generic long-running tool runtime](2026-06-20-generic-long-running-tool-runtime.md) proposal. + +`web_fetch` can use `TextRetainer` with `head` when the provider exposes a stream, or keep provider-owned body caps when the provider must read and decode internally. Either way, the fetch result's `truncated` remains a provider/tool fact, and the library only supplies retained text and omission metadata. + +`web_search` can use `ItemRetainer` with `head`. Current providers often return an array, so this is post-hoc but still standardizes notices; a streaming provider can use the same strategy with `stopWhenFull`. + +### Notices + +The library exposes a neutral notice shape and a tiny formatter hook, but tools provide the user-facing words. A grep footer says "Narrow the pattern, path, or include"; a web fetch footer says "Fetch a more specific URL or section"; bash may point to a spill file. The retainer cannot know those recovery actions. + +```ts ignore-check +interface RetentionNotice { + scope: string + strategy: 'head' | 'tail' | 'headTail' + unit: 'items' | 'bytes' | 'chars' | 'lines' + limit: number | { head: number; tail: number } + kept: number + omitted: Omitted +} + +const formatGrepNotice = (notice: RetentionNotice): string => + formatRetentionNotice( + notice, + ({ kept }) => `Results capped at ${kept}. Narrow the pattern, path, or include to see more.`, + ) +``` + +The formatter hook is deliberately small: a tool turns a `RetentionNotice` into its own footer text. The helper may standardize omission wording, but it does not own recovery guidance. + +`truncated` means the retainer omitted otherwise-available content because of a budget. It does not mean the upstream was incomplete. Tools keep separate fields for permission failures, skipped binary files, provider partial failures, unreadable candidates, invalid UTF-8, and any other "could not inspect" condition. + +## Alternatives considered + +**Post-hoc `truncate(text)` only.** Rejected: it matches Codex's history/tool-output truncation use case but fails the `glob` / `grep` resource model. The tool must stop ripgrep once the probe result proves truncation; collecting all output and trimming afterward defeats the point and can exceed the command runner's in-memory output cap. + +**One generic `Collector` with pluggable callbacks.** Rejected for v1: it hides the two important resource modes. Logical item retention can ask the caller to stop after a probe item; text tail/head-tail retention usually must read to the end. Separate `ItemRetainer` and `TextRetainer` names make that difference explicit while keeping the API small. + +**Put `read` windowing behind `ItemRetainer`.** Rejected for v1: `read` is the only current window consumer, and its semantics are file pagination rather than generic retention. A single `Omitted` count cannot represent both sides of a line window, and `read` also carries `totalLines`, offset-range errors, per-line preview truncation, and a byte cap over selected output. Keeping `read-render` tool-owned avoids growing the shared library around one special case. + +**Make truncation part of `ToolExecutionResult`.** Rejected: the tool registry would have to understand tool-specific recovery guidance, grouping, line numbering, exit status, and provider semantics. Retention is a library used before a tool returns `ContentBlock[]`; the model-facing result remains tool-owned. + +**Expose limits in every model-facing tool schema.** Rejected as the default: Claude Code's grep exposes `head_limit` / `offset`, but this harness keeps routine budgets as deployment config unless the model genuinely needs pagination control. A future read-like continuation field can be added per tool; it does not belong in the shared retention primitive. + +## Acceptance criteria + +- A new `@deepseek-ai/dsh-retention` utility package exports `ItemRetainer`, `TextRetainer`, `RetainedItems`, `RetainedText`, the strategy types, `Omitted`, `PushDecision`, and neutral notice helpers without depending on Cordis or any tool package. +- Unit tests cover item-head early stop with a probe item, item-head read-to-end with exact omission counts, text-head early stop, text-tail retention with exact omission counts, head-tail byte retention, zero budgets, UTF-8 boundary handling, and the difference between `{ kind: 'atLeast', count: 1 }` and exact omission. +- `glob`, `grep`, `bash`, `web_fetch`, and `web_search` have documented mappings to the library before any broad migration begins; each mapping states whether it may stop upstream early. `read` is documented as intentionally out of scope for v1. +- Existing tool-specific states such as `incomplete`, provider failures, binary skips, and bash spill-path recovery remain outside the retention library. +- If the first implementation migrates an existing tool, that package's README and tests prove the model-facing result text is unchanged except for deliberate notice wording. + +## Risks + +- **Over-generalizing the v1 surface.** A generic callback-heavy collector would be harder to reason about than the duplicated code it replaces. The v1 surface deliberately supports only item `head` retention and text `head` / `tail` / `headTail`; windows, grouped budgets, and sort-aware caps can wait until a second consumer proves it needs them. +- **Conflating truncation with incomplete execution.** The library name may invite callers to stuff permission or provider partial failures into `truncated`. Tests and README examples must keep the rule explicit: retention budgets omit available content; incomplete inspection is a tool-domain state. +- **Byte-vs-character confusion.** Text retainers count bytes for process/body safety, while some model-facing previews care about characters or lines. The v1 API should make byte retention explicit and leave character-level preview helpers as separate functions. +- **False precision after early stop.** `glob` and `grep` cannot report exact omitted counts when they stop the upstream at the first overflow item. The `Omitted.atLeast` variant exists so formatters do not claim "omitted 1" when the true count may be much larger. From 7a1c2779a701737f8527e7291801dfeaca35ca0d Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Tue, 7 Jul 2026 09:31:53 +0800 Subject: [PATCH 02/24] feat(retention): add dsh-retention bounded-output library MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ship @deepseek-ai/dsh-retention under packages/util/: pure ItemRetainer / TextRetainer plus neutral notice helpers, so tools that cap model-facing output share one "what did we keep, what did we omit, may we stop reading" mechanic while keeping grouping, exit codes, provider errors, and recovery prose tool-owned. The two retainers are separate names because they differ in resource model: item-head can stop the upstream on the first over-cap probe (shouldStop), while text tail/head-tail must read to the end. The library documents glob/grep/bash/web_fetch/web_search mappings but migrates no tool yet — glob/grep don't exist, and migration is deliberately separate work. Flips the RFC to implemented/ and rewrites its skeleton to shipped reality. --- docs/module-graph.md | 2 + docs/rfc/INDEX.md | 2 +- ...026-07-06-tool-result-retention-library.md | 33 +- knip.json | 5 + packages/README.md | 2 +- packages/util/README.md | 3 + packages/util/retention/README.md | 83 +++ packages/util/retention/package.json | 30 ++ packages/util/retention/src/index.ts | 474 ++++++++++++++++++ .../util/retention/tests/retention.spec.ts | 348 +++++++++++++ packages/util/retention/tsconfig.json | 11 + pnpm-lock.yaml | 6 + tsconfig.build.json | 1 + tsconfig.json | 1 + 14 files changed, 980 insertions(+), 21 deletions(-) rename docs/rfc/{proposed => implemented}/architecture/2026-07-06-tool-result-retention-library.md (75%) create mode 100644 packages/util/retention/README.md create mode 100644 packages/util/retention/package.json create mode 100644 packages/util/retention/src/index.ts create mode 100644 packages/util/retention/tests/retention.spec.ts create mode 100644 packages/util/retention/tsconfig.json diff --git a/docs/module-graph.md b/docs/module-graph.md index a4e729484d..d2537028f6 100644 --- a/docs/module-graph.md +++ b/docs/module-graph.md @@ -9,6 +9,7 @@ Inter-package dependencies among the `@deepseek-ai/dsh-*` harness packages, deri flowchart TD subgraph group_util["packages/util"] pkg_brand["brand"] + pkg_retention["retention"] pkg_timeout["timeout"] end subgraph group_llm["packages/llm"] @@ -208,6 +209,7 @@ flowchart TD | Package | Group | Depends on | | --- | --- | --- | | [`brand`](../packages/util/brand) | `util` | — | +| [`retention`](../packages/util/retention) | `util` | — | | [`timeout`](../packages/util/timeout) | `util` | — | | [`app-boot`](../packages/ui/app-boot) | `ui` | — | | [`llm`](../packages/llm/llm) | `llm` | [`brand`](../packages/util/brand) | diff --git a/docs/rfc/INDEX.md b/docs/rfc/INDEX.md index a7e0506cad..1ce999d071 100644 --- a/docs/rfc/INDEX.md +++ b/docs/rfc/INDEX.md @@ -26,7 +26,6 @@ Generated by `pnpm run gen-rfc-index` from the RFC tree — never edit by hand; |---|---| | [Runtime schemas for the event vocabulary (Zod vs the merge-extensible-map pattern)](proposed/architecture/2026-06-16-typed-event-schemas.md) | 2026-06-16 | | [Extract a generic long-running tool runtime](proposed/architecture/2026-06-20-generic-long-running-tool-runtime.md) | 2026-06-20 | -| [Tool result retention library](proposed/architecture/2026-07-06-tool-result-retention-library.md) | 2026-07-06 | ### Process @@ -120,6 +119,7 @@ Generated by `pnpm run gen-rfc-index` from the RFC tree — never edit by hand; | [Prompt variables and tool-guidance ownership](implemented/architecture/2026-07-05-prompt-variables-and-tool-guidance-ownership.md) | 2026-07-05 | | [Subagent provider-lifecycle events — `subagent/provider-added` / `subagent/provider-removed`](implemented/architecture/2026-07-05-subagent-provider-lifecycle-events.md) | 2026-07-05 | | [A shared timeout/deadline primitive, with hard-kill left to each capability](implemented/architecture/2026-07-06-timeout-deadline-library.md) | 2026-07-06 | +| [Tool result retention library](implemented/architecture/2026-07-06-tool-result-retention-library.md) | 2026-07-06 | ### Process diff --git a/docs/rfc/proposed/architecture/2026-07-06-tool-result-retention-library.md b/docs/rfc/implemented/architecture/2026-07-06-tool-result-retention-library.md similarity index 75% rename from docs/rfc/proposed/architecture/2026-07-06-tool-result-retention-library.md rename to docs/rfc/implemented/architecture/2026-07-06-tool-result-retention-library.md index b8f16f6bb1..7ea954b0a2 100644 --- a/docs/rfc/proposed/architecture/2026-07-06-tool-result-retention-library.md +++ b/docs/rfc/implemented/architecture/2026-07-06-tool-result-retention-library.md @@ -1,6 +1,6 @@ # RFC: Tool result retention library -Status: proposed +Status: implemented ## Problem @@ -8,9 +8,9 @@ Several model-facing tools already bound the amount of context they return, but The shared abstraction the tools need is **retention**, not generic collection. A caller feeds items or text chunks into a bounded object, receives a per-push decision about whether the upstream can stop, and later receives the retained content plus exact or partial omission metadata. Tool-specific code still owns business semantics: file grouping, line numbering, exit codes, provider error states, and model-facing prose. The common library owns only the mechanical question "what did we keep, what did we omit, and may the caller stop reading now?" -## Proposal +## Decision -Add a small, dependency-light retention library under `packages/util/retention` (package name `@deepseek-ai/dsh-retention`). It exports pure item and text retainers plus notice helpers. It is not a Cordis service and registers no plugin; tool packages import it directly when they need bounded model-facing output. +`@deepseek-ai/dsh-retention` lives under `packages/util/` (peer to `dsh-brand` and `dsh-timeout`) and owns bounded model-facing output. It is a library of pure classes and functions, **not** a Cordis service or plugin: it takes no `ctx`, registers nothing, holds no cross-call state, and emits no events. Tool packages import it directly when they need bounded output. The library has two independent retainers: @@ -116,7 +116,7 @@ type TextRetentionStrategy = `grep` uses `ItemRetainer` with `{ kind: 'head', maxItems: grepMaxMatches, stop: 'stopWhenFull' }` before grouping. The backend parses a ripgrep match record, maps the path, applies per-line preview truncation, then pushes a flat match. After `finish()`, the backend groups retained matches by file and sorts the returned subset. Grouping is not part of the retainer because the cap is total matches, not files; per-match preview truncation and `incomplete` are also separate from result-level retention. -`bash` uses `TextRetainer` with `tail` or `headTail` and reads to process completion. It does not stop when full: stopping the read would lose the real tail and can create pipe backpressure. The bash executor still owns spill files, exit status, signal, timeout, and background-task behavior; the retention helper only replaces ad hoc in-memory head/tail accounting where that behavior is desired. Long-running task ownership remains orthogonal to the [generic long-running tool runtime](2026-06-20-generic-long-running-tool-runtime.md) proposal. +`bash` uses `TextRetainer` with `tail` or `headTail` and reads to process completion. It does not stop when full: stopping the read would lose the real tail and can create pipe backpressure. The bash executor still owns spill files, exit status, signal, timeout, and background-task behavior; the retention helper only replaces ad hoc in-memory head/tail accounting where that behavior is desired. Long-running task ownership remains orthogonal to the [generic long-running tool runtime](../../proposed/architecture/2026-06-20-generic-long-running-tool-runtime.md) proposal. `web_fetch` can use `TextRetainer` with `head` when the provider exposes a stream, or keep provider-owned body caps when the provider must read and decode internally. Either way, the fetch result's `truncated` remains a provider/tool fact, and the library only supplies retained text and omission metadata. @@ -147,6 +147,16 @@ The formatter hook is deliberately small: a tool turns a `RetentionNotice` into `truncated` means the retainer omitted otherwise-available content because of a budget. It does not mean the upstream was incomplete. Tools keep separate fields for permission failures, skipped binary files, provider partial failures, unreadable candidates, invalid UTF-8, and any other "could not inspect" condition. +## Consequences + +**What shipped.** `@deepseek-ai/dsh-retention` exports `ItemRetainer`, `TextRetainer`, the result types (`RetainedItems`, `RetainedText`), the strategy types (`ItemRetentionStrategy`, `TextRetentionStrategy`, `StopMode`), `Omitted`, `PushDecision`, `RetentionNotice`, and the neutral notice helpers `describeOmitted` / `formatRetentionNotice` — with no dependency on Cordis or any tool package. Unit tests cover item-head early stop with a probe item, item-head read-to-end with exact omission counts, text-head early stop, text-tail retention with exact omission counts, head-tail byte retention, zero budgets, UTF-8 boundary handling (2-, 3-, and 4-byte codepoints and invalid lead bytes at each cut), and the difference between `{ kind: 'atLeast', count: 1 }` and exact omission. + +**What is documented but not yet migrated.** `glob`, `grep`, `bash`, `web_fetch`, and `web_search` have their mappings documented in the [package README](../../../../packages/util/retention/README.md) — each stating whether it may stop upstream early — but no tool has been migrated onto the library in this change; migration is deliberately separate follow-up work. `glob` / `grep` do not yet exist as tools, so the `shouldStop` early-stop path has no in-repo caller until they land. `read` is documented as intentionally out of scope: its `read-render` line-window contract (`offset`/`limit`, `totalLines`, offset-range errors, per-line preview truncation, a byte cap over the selected window) is not generic retention, and one `Omitted` count cannot represent both sides of a line window. + +**Boundaries the library holds.** `truncated` means the retainer omitted otherwise-available content because of a budget; it never means the upstream was incomplete. Tool-specific states — `incomplete`, permission failures, provider partial failures, binary skips, bash spill-path recovery, invalid UTF-8 — stay in tool-domain fields, outside the retainer. When a future change migrates a tool, that package's README and tests must prove the model-facing result text is unchanged except for deliberate notice wording. + +**Tradeoffs accepted.** The v1 surface deliberately supports only item `head` retention and text `head` / `tail` / `headTail`; windows, grouped budgets, and sort-aware caps wait until a second consumer proves the need (the generic-collector alternative is why). Text retention counts bytes for process/body safety, leaving character- and line-level preview budgets as separate tool-owned concerns. `glob` / `grep` cannot report an exact omitted count once they stop the upstream at the first overflow item, so `Omitted.atLeast` exists and `describeOmitted` prints no number for it — formatters never claim "omitted 1" when the true count may be far larger. + ## Alternatives considered **Post-hoc `truncate(text)` only.** Rejected: it matches Codex's history/tool-output truncation use case but fails the `glob` / `grep` resource model. The tool must stop ripgrep once the probe result proves truncation; collecting all output and trimming afterward defeats the point and can exceed the command runner's in-memory output cap. @@ -158,18 +168,3 @@ The formatter hook is deliberately small: a tool turns a `RetentionNotice` into **Make truncation part of `ToolExecutionResult`.** Rejected: the tool registry would have to understand tool-specific recovery guidance, grouping, line numbering, exit status, and provider semantics. Retention is a library used before a tool returns `ContentBlock[]`; the model-facing result remains tool-owned. **Expose limits in every model-facing tool schema.** Rejected as the default: Claude Code's grep exposes `head_limit` / `offset`, but this harness keeps routine budgets as deployment config unless the model genuinely needs pagination control. A future read-like continuation field can be added per tool; it does not belong in the shared retention primitive. - -## Acceptance criteria - -- A new `@deepseek-ai/dsh-retention` utility package exports `ItemRetainer`, `TextRetainer`, `RetainedItems`, `RetainedText`, the strategy types, `Omitted`, `PushDecision`, and neutral notice helpers without depending on Cordis or any tool package. -- Unit tests cover item-head early stop with a probe item, item-head read-to-end with exact omission counts, text-head early stop, text-tail retention with exact omission counts, head-tail byte retention, zero budgets, UTF-8 boundary handling, and the difference between `{ kind: 'atLeast', count: 1 }` and exact omission. -- `glob`, `grep`, `bash`, `web_fetch`, and `web_search` have documented mappings to the library before any broad migration begins; each mapping states whether it may stop upstream early. `read` is documented as intentionally out of scope for v1. -- Existing tool-specific states such as `incomplete`, provider failures, binary skips, and bash spill-path recovery remain outside the retention library. -- If the first implementation migrates an existing tool, that package's README and tests prove the model-facing result text is unchanged except for deliberate notice wording. - -## Risks - -- **Over-generalizing the v1 surface.** A generic callback-heavy collector would be harder to reason about than the duplicated code it replaces. The v1 surface deliberately supports only item `head` retention and text `head` / `tail` / `headTail`; windows, grouped budgets, and sort-aware caps can wait until a second consumer proves it needs them. -- **Conflating truncation with incomplete execution.** The library name may invite callers to stuff permission or provider partial failures into `truncated`. Tests and README examples must keep the rule explicit: retention budgets omit available content; incomplete inspection is a tool-domain state. -- **Byte-vs-character confusion.** Text retainers count bytes for process/body safety, while some model-facing previews care about characters or lines. The v1 API should make byte retention explicit and leave character-level preview helpers as separate functions. -- **False precision after early stop.** `glob` and `grep` cannot report exact omitted counts when they stop the upstream at the first overflow item. The `Omitted.atLeast` variant exists so formatters do not claim "omitted 1" when the true count may be much larger. diff --git a/knip.json b/knip.json index 2e3e101ae1..586528db74 100644 --- a/knip.json +++ b/knip.json @@ -26,6 +26,11 @@ "project": ["src/**/*.ts", "tests/**/*.ts"], "ignoreDependencies": ["cordis"] }, + "packages/util/retention": { + "entry": ["tests/**/*.spec.ts"], + "project": ["src/**/*.ts", "tests/**/*.ts"], + "ignoreDependencies": ["cordis"] + }, "packages/llm/llm-deepseek": { "entry": ["tests/**/*.spec.ts", "tests/**/*.e2e.ts"], "project": ["src/**/*.ts", "tests/**/*.ts"] diff --git a/packages/README.md b/packages/README.md index f07e0d5a36..1f7cb9121b 100644 --- a/packages/README.md +++ b/packages/README.md @@ -20,7 +20,7 @@ Packages are grouped by modular role at `packages///`. The group dir | [`session-persistence/`](session-persistence/README.md) | Persistence capability family: the seam + JSONL/SQLite backends | Product — stable surface | | [`ui/`](ui/README.md) | Editor/client integration surfaces (the ACP bridge) + the app packages | Product — stable surface | | [`support/`](support/README.md) | Dev/test/example infrastructure (invariants, replay adapter, subagent mock) | Support — lower compatibility expectations | -| [`util/`](util/README.md) | Low-level zero-dependency utilities shared across groups (the `Branded` primitive) | Support — small, stable, harness-dep-free | +| [`util/`](util/README.md) | Low-level zero-dependency primitives shared across groups (branding, timeout, retention) | Support — small, stable, harness-dep-free | The split is the point: a package's group says whether it is part of the product API or support/test/example infrastructure, so release and removal decisions do not treat every package as an equal public contract. New packages join an existing group; adding a new top-level group is a deliberate act (extend the group READMEs and this table). diff --git a/packages/util/README.md b/packages/util/README.md index 45afe7b0a9..6477523861 100644 --- a/packages/util/README.md +++ b/packages/util/README.md @@ -6,7 +6,10 @@ Zero-dependency primitives shared across the other groups. A package lands here |---|---| | `brand/` | The type-only `Branded` nominal-typing primitive (no runtime code, no harness deps) | | `timeout/` | The timing/classification half of a timeout — `clampTimeout`/`deadline`/`timeoutOf`/`TimeoutReason` (pure functions, no harness deps); termination stays in each capability | +| `retention/` | Bounded model-facing output — `ItemRetainer`/`TextRetainer` + neutral notice helpers (pure, no harness deps); business semantics stay in each tool | `dsh-brand` is the canonical case: it owns ONLY the `Branded` helper, so a capability package can brand the ids it owns (`dsh-bash`'s `BashTaskId`/`OwnerToken`, `dsh-session`'s `SessionId`, …) by depending on `dsh-brand` alone, without pulling in an unrelated package just to reach `Branded`. `dsh-timeout` follows the same shape for the timeout family: `dsh-bash` and `dsh-web-fetch-local` each fuse a caller's cancellation with a deadline and later classify "timed out" vs "cancelled" by depending on `dsh-timeout` alone. It deliberately owns only the timing/classification half — the *termination* (SIGKILL a process group, tear down a fetch socket) stays in each capability, because no shared layer can own every capability's kill (see [the timeout-library RFC](../../docs/rfc/implemented/architecture/2026-07-06-timeout-deadline-library.md)). + +`dsh-retention` is the same split for bounded tool output: a tool (`glob`/`grep`/`bash`/`web_fetch`/`web_search`) feeds items or text into a retainer and gets back "what we kept, what we omitted, may you stop reading" — while grouping, exit codes, provider errors, and recovery prose stay tool-owned. It deliberately owns only the retention mechanic; `truncated` is a budget fact, never an "incomplete inspection" state (see [the retention-library RFC](../../docs/rfc/implemented/architecture/2026-07-06-tool-result-retention-library.md)). diff --git a/packages/util/retention/README.md b/packages/util/retention/README.md new file mode 100644 index 0000000000..50bac13828 --- /dev/null +++ b/packages/util/retention/README.md @@ -0,0 +1,83 @@ +# dsh-retention + +A dependency-light **retention** library: bounded model-facing output for tools that must cap how much context they return. A caller feeds items or text chunks into a bounded object, gets a per-push decision about whether the upstream may stop, and later gets the retained content plus exact or partial omission metadata. + +The library owns **only** the mechanical question *"what did we keep, what did we omit, and may the caller stop reading now?"*. Tool-specific code keeps its business semantics: file grouping, line numbering, exit codes, provider error states, per-line preview truncation, spill files, and the model-facing prose. This is the boundary the [RFC](../../../docs/rfc/implemented/architecture/2026-07-06-tool-result-retention-library.md) draws. + +It is a **library, not a service or plugin**: no `ctx`, registers nothing, emits no events. The only state is per-retainer (one accumulation), never cross-call. Tool packages import it directly. + +## Surface + +```ts +import { + ItemRetainer, TextRetainer, + describeOmitted, formatRetentionNotice, +} from '@deepseek-ai/dsh-retention' +import type { + Omitted, PushDecision, RetainedItems, RetainedText, + ItemRetentionStrategy, TextRetentionStrategy, StopMode, RetentionNotice, +} from '@deepseek-ai/dsh-retention' +``` + +| Export | Role | +|---|---| +| `ItemRetainer` | Bounds ordered logical units (paths, grep matches, sources). `head` only in v1. `push()` → `PushDecision`; `finish()` → `RetainedItems`. | +| `TextRetainer` | Bounds a byte-oriented text stream. `head` / `tail` / `headTail`, UTF-8 boundaries preserved at `finish()`. `push()` → `PushDecision`; `finish()` → `RetainedText`. | +| `describeOmitted(omitted, unit)` | Standardized, false-precision-safe omission clause (`exact` prints a count; `atLeast`/`unknown` do not). | +| `formatRetentionNotice(notice, recovery)` | Joins the standardized omission clause with the tool's own recovery guidance. | +| `Omitted` | `none` / `exact` / `atLeast` / `unknown` — how much was omitted, and whether the count is a lower bound. | +| `PushDecision` | `{ kept, truncated, shouldStop }` — the per-push control-flow result. | + +## The two resource modes + +The two retainers are separate names, not one generic collector, because they differ in **resource model** — and that difference is the whole point of the `shouldStop` field. + +- **`ItemRetainer` can stop the upstream early.** With `stop: 'stopWhenFull'`, the first over-cap unit is a *probe*: it is not retained, sets `truncated`, and returns `shouldStop: true`. A discovery tool uses that to kill ripgrep / cancel a stream the moment truncation is proven, instead of collecting everything and trimming afterward. Because it stopped before the true total was known, `omitted` is `{ kind: 'atLeast', count: 1 }` — a lower bound, never a false-precise exact count. +- **`TextRetainer` tail/headTail must read to the end.** A true tail is unknowable until the stream closes, and draining avoids pipe backpressure on a child process, so `tail` and `headTail` never set `shouldStop` and report an `exact` omitted byte count. Only `head` + `stopWhenFull` can stop a text stream early. + +`shouldStop` is **advisory**: the retainer cannot reach the upstream. The tool owns the actual stop — abort the HTTP body, break the scan, kill the process group. + +## `truncated` is a budget fact, never "incomplete" + +`truncated` means *the retainer omitted otherwise-available content because of a budget*. It does **not** mean the upstream was incomplete. Permission failures, skipped binary files, provider partial failures, unreadable candidates, and invalid UTF-8 stay in tool-domain fields — never folded into `truncated`. Conflating the two is the bug this library's naming most invites; keep them separate. + +## Bytes, not characters + +Text caps and `omittedBytes` count **bytes**, for process/body safety (a child's pipe and an HTTP body are byte streams). A chunk that straddles a codepoint is handled: `finish()` trims a partial codepoint at each cut so the returned text never introduces a replacement char at the boundary, and the two sides are decoded separately so a codepoint is never reconstructed across the omitted middle. Character- or line-level preview budgets are a separate, tool-owned concern. + +## Tool mappings + +Every current retention consumer maps to the library below; each row states whether it may stop its upstream early. A broad migration is out of scope for the library's first landing — these are the intended shapes. + +| Tool | Retainer & strategy | Stops upstream early? | Notes | +|---|---|---|---| +| `glob` | `ItemRetainer`, `head` + `stopWhenFull` | **Yes** — the `(maxItems+1)`th path is the probe; `shouldStop` kills ripgrep. | Path mapping, skipped candidates, `incomplete` stay outside. `omitted` is `atLeast`. | +| `grep` | `ItemRetainer`, `head` + `stopWhenFull` | **Yes** — cap is total matches; stop on the probe match. | Per-match preview truncation, then push a flat match; group + sort the retained subset *after* `finish()`. | +| `bash` | `TextRetainer`, `tail` or `headTail`, reads to completion | No — stopping would lose the true tail and risk pipe backpressure. | Executor still owns spill files, exit status, signal, timeout, background tasks. | +| `web_fetch` | `TextRetainer`, `head` (streaming provider) | Optional — a streaming body can stop; a decode-internally provider keeps its own cap. | The fetch result's `truncated` remains a provider/tool fact. | +| `web_search` | `ItemRetainer`, `head` | Post-hoc today (providers return arrays); a streaming provider can use `stopWhenFull`. | Standardizes the "sources capped" notice. | + +`read` is **intentionally out of scope for v1.** Its `read-render` helper owns a file-specific pagination contract — `offset`/`limit`, line numbers, `totalLines`, offset-out-of-range errors, per-line preview truncation, a byte cap over the selected window — which is a line-window renderer, not generic retention. A single `Omitted` count cannot represent both sides of a line window. + +## Usage shape + +```ts ignore-check +// glob: stop ripgrep the moment truncation is proven. +const retainer = new ItemRetainer({ kind: 'head', maxItems: globMaxResults, stop: 'stopWhenFull' }) +for await (const entry of candidates) { + const { shouldStop } = retainer.push(entry) + if (shouldStop) { killRipgrep(); break } // the tool owns the actual stop +} +const { items, truncated, omitted } = retainer.finish() + +// bash: keep a head + tail, read to process exit. +const out = new TextRetainer({ kind: 'headTail', headBytes: headCap, tailBytes: tailCap }) +child.stdout.on('data', (chunk: Buffer) => { out.push(chunk) }) // shouldStop ignored: must drain +const { text, omittedBytes } = out.finish() + +// A footer: the library standardizes the omission clause; the tool owns recovery words. +const footer = formatRetentionNotice( + { scope: 'grep', strategy: 'head', unit: 'items', limit: grepMaxMatches, kept: items.length, omitted }, + ({ kept }) => `Results capped at ${kept}. Narrow the pattern, path, or include to see more.`, +) +``` diff --git a/packages/util/retention/package.json b/packages/util/retention/package.json new file mode 100644 index 0000000000..2926144ace --- /dev/null +++ b/packages/util/retention/package.json @@ -0,0 +1,30 @@ +{ + "name": "@deepseek-ai/dsh-retention", + "description": "Zero-dependency bounded-retention primitive: ItemRetainer/TextRetainer + neutral notice helpers (what did we keep, what did we omit, may the caller stop reading)", + "version": "0.0.1", + "private": true, + "type": "module", + "main": "lib/index.js", + "types": "lib/types/index.d.ts", + "exports": { + ".": { + "types": "./lib/types/index.d.ts", + "default": "./lib/index.js" + }, + "./src/*": "./src/*", + "./package.json": "./package.json" + }, + "files": [ + "lib/index.js", + "lib/types/**/*.d.ts", + "lib/types/**/*.d.ts.map", + "src" + ], + "license": "BSD-3-Clause", + "peerDependencies": { + "cordis": "^4.0.0-rc.6" + }, + "devDependencies": { + "cordis": "^4.0.0-rc.6" + } +} diff --git a/packages/util/retention/src/index.ts b/packages/util/retention/src/index.ts new file mode 100644 index 0000000000..bf754e2a8b --- /dev/null +++ b/packages/util/retention/src/index.ts @@ -0,0 +1,474 @@ +/** + * A dependency-light **retention** library: bounded model-facing output for + * tools that must cap how much context they return. A caller feeds items or + * text chunks into a bounded object, gets a per-push {@link PushDecision} about + * whether the upstream may stop, and later gets the retained content plus exact + * or partial omission metadata ({@link RetainedItems} / {@link RetainedText}). + * + * The library owns ONLY the mechanical question "what did we keep, what did we + * omit, and may the caller stop reading now?". Tool-specific code still owns + * business semantics: file grouping, line numbering, exit codes, provider error + * states, per-line preview truncation, spill files, and the model-facing prose. + * In particular {@link RetainedText.truncated}/{@link RetainedItems.truncated} + * means "the retainer omitted otherwise-available content because of a budget" — + * NOT "the upstream was incomplete". Permission failures, skipped binaries, + * provider partial failures, and unreadable candidates stay in tool-domain + * fields, never folded into `truncated`. + * + * This is deliberately a library, not a cordis service or plugin: it takes no + * `ctx`, registers nothing, and emits no events. The two retainers are the only + * stateful pieces and their state is per-instance (one accumulation), never + * cross-call. Tool packages import it directly when they need bounded output. + * + * The two retainers differ in resource model, which is why they are two names + * rather than one generic collector: + * - {@link ItemRetainer} bounds ordered logical units (paths, grep matches, + * search sources). `head` retention only in v1. With `stopWhenFull` it can ask + * the caller to stop the upstream after the first over-cap probe item. + * - {@link TextRetainer} bounds byte-oriented text streams (bash stdout/stderr, + * web bodies). `head` / `tail` / `headTail`, preserving UTF-8 boundaries at + * {@link TextRetainer.finish}. Only `head` can stop early; `tail`/`headTail` + * must read to the end to know the true suffix and exact omission. + * + * @module @deepseek-ai/dsh-retention + */ + +/** + * How much content the retainer omitted. + * + * `atLeast` is the early-stop shape: an {@link ItemRetainer}/{@link TextRetainer} + * with `stopWhenFull` sees the first unit/chunk past the cap, asks the caller to + * stop the upstream, and therefore knows only a LOWER bound — reporting an exact + * count there would be false precision when the true total may be much larger. + * `exact` is the read-to-end shape (`tail`, `headTail`, or `head` with + * `readToEnd`), where every unit/byte was observed. `unknown` is reserved for a + * caller that omits without a count; the retainers themselves never return it. + */ +export type Omitted = + | { kind: 'none' } + | { kind: 'exact'; count: number } + | { kind: 'atLeast'; count: number } + | { kind: 'unknown' } + +/** + * The caller receives this after each `push()`. + * + * `shouldStop` is ADVISORY, not automatic: the tool owns how to stop its upstream + * source — aborting an HTTP body, breaking a file scan, killing ripgrep. The + * retainer cannot reach the upstream; it only reports that keeping more would + * exceed the budget. A `readToEnd` / `tail` / `headTail` retainer never sets it + * (those must drain to the end). + */ +export interface PushDecision { + /** Was this whole unit / all of this chunk's bytes retained (nothing dropped)? */ + kept: boolean + /** Cumulative: has the retainer omitted anything due to the budget yet? */ + truncated: boolean + /** Advisory: keeping more would exceed the budget — the tool may stop its upstream. */ + shouldStop: boolean +} + +/** + * Final result for ordered logical units. + * + * `seen` means units OBSERVED by the retainer, not necessarily the total in the + * upstream source; with an early stop, the true total is intentionally unknown + * (hence {@link Omitted.atLeast}). `kept` is `items.length`, surfaced explicitly + * so a notice formatter need not re-count. + */ +export interface RetainedItems { + items: T[] + truncated: boolean + seen: number + kept: number + omitted: Omitted +} + +/** + * Final result for text streams. + * + * The returned `text` is safe to hand to a formatter: the retainer adds no + * tool-specific headers, exit markers, XML tags, or recovery instructions, and + * `omittedBytes` counts BYTES (not characters or lines) — text retention is + * byte-oriented for process/body safety. UTF-8 boundaries at each cut are + * preserved, so `text` never carries a replacement char introduced by the cut + * itself. + */ +export interface RetainedText { + text: string + truncated: boolean + omittedBytes: Omitted +} + +/** + * Whether a retainer asks the caller to stop the upstream once keeping more + * would exceed the budget (`stopWhenFull`), or must keep accepting input even + * after the retained output is full (`readToEnd`) — usually to preserve a true + * tail, count exact omission, or drain an upstream process to avoid pipe + * backpressure. Names avoid implementation phrases like "overflow". + */ +export type StopMode = 'stopWhenFull' | 'readToEnd' + +/** Item retention strategy. Only `head` in v1; windows/grouped budgets wait for a second consumer. */ +export type ItemRetentionStrategy = { + /** Keep the first `maxItems` units. Use for `glob`, `grep`, and web sources. */ + kind: 'head' + maxItems: number + stop: StopMode +} + +/** Text retention strategy: keep a prefix, a suffix, or both, counted in bytes. */ +export type TextRetentionStrategy = + | { + /** Keep the first `maxBytes` bytes. May stop an upstream body early. */ + kind: 'head' + maxBytes: number + stop: StopMode + } + | { + /** Keep the final `maxBytes` bytes. Requires reading to the end. */ + kind: 'tail' + maxBytes: number + } + | { + /** Keep a stable prefix and suffix, omitting the middle. Requires reading to the end. */ + kind: 'headTail' + headBytes: number + tailBytes: number + } + +/** + * A neutral, tool-agnostic description of one retention outcome — the input to + * {@link formatRetentionNotice}. It carries the mechanical facts (strategy, + * unit, limit, kept count, {@link Omitted}); the tool supplies the recovery + * words, because only the tool knows the recovery action ("narrow the pattern", + * "fetch a more specific URL", "read the spill file"). + */ +export interface RetentionNotice { + /** Tool/scope label, e.g. `grep`, `web_fetch`, `bash stdout`. */ + scope: string + strategy: 'head' | 'tail' | 'headTail' + unit: 'items' | 'bytes' | 'chars' | 'lines' + limit: number | { head: number; tail: number } + kept: number + omitted: Omitted +} + +/** Assert a budget field is a non-negative integer (the retainer request contract). */ +function assertBudget(value: number, name: string): void { + if (!Number.isInteger(value) || value < 0) { + throw new Error(`${name} must be a non-negative integer`) + } +} + +/** + * Bounds an ordered stream of logical units, keeping the first `maxItems` + * ({@link ItemRetentionStrategy} `head`). `push()` reports, per unit, whether it + * was kept and — under `stopWhenFull` — whether the caller should stop the + * upstream now that the first over-cap probe unit has been seen. + * + * Grouping, sorting, path mapping, per-unit preview truncation, and any + * `incomplete` state stay OUTSIDE the retainer: it counts and keeps, nothing + * more. The caller pushes already-shaped units and, after {@link finish}, + * groups/sorts the retained subset itself. + */ +export class ItemRetainer { + private readonly maxItems: number + private readonly stop: StopMode + private readonly items: T[] = [] + private seen = 0 + private omittedCount = 0 + + /** @param strategy Head strategy: `maxItems` (non-negative integer) and the {@link StopMode}. */ + constructor(strategy: ItemRetentionStrategy) { + assertBudget(strategy.maxItems, 'maxItems') + this.maxItems = strategy.maxItems + this.stop = strategy.stop + } + + /** + * Offer one unit. Kept when the retainer is below `maxItems`; otherwise dropped + * and counted as omitted. Under `stopWhenFull` the first dropped unit is the + * probe: `shouldStop` is `true` so the caller can kill ripgrep / cancel the + * stream, and the final {@link Omitted} stays `atLeast` (the true total is + * unknown). Under `readToEnd` the caller keeps pushing so omission is `exact`. + * + * @param item The already-shaped logical unit (path, flat match, source). + * @returns The per-push {@link PushDecision}. + */ + push(item: T): PushDecision { + this.seen++ + if (this.items.length < this.maxItems) { + // Reached only below the cap, before any omission (items only grow, the + // cap is fixed), so nothing has been dropped yet: truncated is always false. + this.items.push(item) + return { kept: true, truncated: false, shouldStop: false } + } + this.omittedCount++ + return { + kept: false, + truncated: true, + // Only ask to stop when the caller opted into it; readToEnd must keep + // draining to reach an exact omission count. + shouldStop: this.stop === 'stopWhenFull', + } + } + + /** + * Finalize and report what was kept and omitted. `omitted` is `atLeast` under + * `stopWhenFull` (a lower bound — the caller was asked to stop before the true + * total was known) and `exact` under `readToEnd`. + * + * @returns The {@link RetainedItems} snapshot (safe to group/sort downstream). + */ + finish(): RetainedItems { + const truncated = this.omittedCount > 0 + return { + items: this.items, + truncated, + seen: this.seen, + kept: this.items.length, + omitted: truncated + ? { kind: this.stop === 'stopWhenFull' ? 'atLeast' : 'exact', count: this.omittedCount } + : { kind: 'none' }, + } + } +} + +const encoder = new TextEncoder() +const decoder = new TextDecoder() // utf-8, non-fatal: internal malformed bytes → U+FFFD + +/** + * Drop a trailing incomplete UTF-8 sequence so a prefix cut never emits a + * replacement char at the boundary. Walks back over continuation bytes + * (`10xxxxxx`) to the lead byte; if fewer bytes follow it than the lead byte's + * length declares, the sequence is incomplete and is trimmed. A complete tail, + * or a run too long/short to be a valid lead, is returned untouched (any + * genuinely malformed interior is left for the decoder to replace). + */ +function trimTrailingPartialUtf8(bytes: Uint8Array): Uint8Array { + let i = bytes.length - 1 + // Continuation bytes are 0b10xxxxxx; scan back at most 3 (max sequence is 4). + // Indices are bounds-checked by the loop guard, so the reads are in range (a + // cast, not `!`, per the repo's no-non-null-assertion rule). + while (i >= 0 && ((bytes[i] as number) & 0xc0) === 0x80 && bytes.length - i <= 3) i-- + if (i < 0) return bytes + const lead = bytes[i] as number + const expected = lead < 0x80 ? 1 : lead < 0xe0 ? 2 : lead < 0xf0 ? 3 : lead < 0xf8 ? 4 : 0 + // expected 0 → not a lead byte (stray continuation / invalid): leave it. + if (expected === 0) return bytes + return bytes.length - i < expected ? bytes.subarray(0, i) : bytes +} + +/** + * Drop leading continuation bytes (`10xxxxxx`) so a suffix cut starts on a + * lead/ASCII byte instead of mid-codepoint. + */ +function trimLeadingContinuationUtf8(bytes: Uint8Array): Uint8Array { + let i = 0 + // i < length guards the read; cast rather than `!` (no-non-null-assertion). + while (i < bytes.length && ((bytes[i] as number) & 0xc0) === 0x80) i++ + return bytes.subarray(i) +} + +/** + * Bounds a byte-oriented text stream, keeping a prefix, a suffix, or both + * ({@link TextRetentionStrategy}). All three strategies share one prefix/suffix + * accumulator: `head` is prefix-only, `tail` is suffix-only, `headTail` is both. + * Only `head` with `stopWhenFull` sets `shouldStop`; `tail`/`headTail` must read + * to the end to know the true suffix and the exact omitted byte count. + * + * Bytes, not characters: caps and `omittedBytes` are byte counts for process/ + * body safety. Chunks that straddle a codepoint are handled — {@link finish} + * trims a partial codepoint at each cut so the returned text never introduces a + * replacement char at the boundary. The retainer holds at most + * `prefixCap + tailBytes + one chunk` in memory (old suffix chunks are dropped + * as they slide out), so a large stream does not accumulate unbounded. + */ +export class TextRetainer { + private readonly prefixCap: number + private readonly suffixCap: number + private readonly allowStop: boolean + private readonly prefixChunks: Uint8Array[] = [] + private prefixHeld = 0 + private readonly suffixChunks: Uint8Array[] = [] + private suffixHeld = 0 + private total = 0 + + /** @param strategy One of the {@link TextRetentionStrategy} shapes; byte budgets must be non-negative integers. */ + constructor(strategy: TextRetentionStrategy) { + switch (strategy.kind) { + case 'head': + assertBudget(strategy.maxBytes, 'maxBytes') + this.prefixCap = strategy.maxBytes + this.suffixCap = 0 + this.allowStop = strategy.stop === 'stopWhenFull' + break + case 'tail': + assertBudget(strategy.maxBytes, 'maxBytes') + this.prefixCap = 0 + this.suffixCap = strategy.maxBytes + this.allowStop = false + break + case 'headTail': + assertBudget(strategy.headBytes, 'headBytes') + assertBudget(strategy.tailBytes, 'tailBytes') + this.prefixCap = strategy.headBytes + this.suffixCap = strategy.tailBytes + this.allowStop = false + break + } + } + + /** + * Offer one chunk (a `Uint8Array`, or a `string` encoded as UTF-8). Prefix + * bytes fill up to the prefix cap then stop; suffix bytes roll so only the + * last `suffixCap` bytes are retained. `kept` is `true` only when no byte of + * this chunk was dropped. Under `head` + `stopWhenFull`, `shouldStop` turns + * `true` on the chunk that first drops a byte (the caller may then abort the + * body); other strategies never set it. + * + * @param chunk The next bytes of the stream (`Uint8Array` or UTF-8 `string`). + * @returns The per-push {@link PushDecision}. + */ + push(chunk: Uint8Array | string): PushDecision { + const bytes = typeof chunk === 'string' ? encoder.encode(chunk) : chunk + const before = this.total + this.total += bytes.length + + // Prefix: take only up to the cap; the rest of this chunk is "not prefixed". + const room = this.prefixCap - this.prefixHeld + const take = Math.max(0, Math.min(room, bytes.length)) + if (take > 0) { + this.prefixChunks.push(bytes.subarray(0, take)) + this.prefixHeld += take + } + + // Suffix: append the whole chunk, then drop whole leading chunks that have + // fully slid out of the last `suffixCap` bytes (bounded memory). + if (this.suffixCap > 0) { + this.suffixChunks.push(bytes) + this.suffixHeld += bytes.length + let head = this.suffixChunks[0] + while (head !== undefined && this.suffixHeld - head.length >= this.suffixCap) { + this.suffixChunks.shift() + this.suffixHeld -= head.length + head = this.suffixChunks[0] + } + } + + // Dropped = bytes that no side can keep. Compute cumulative omission the + // SAME way finish() does (via omittedAt), so push and finish never disagree; + // per-push we only need whether THIS chunk pushed the total past what the + // two caps hold, and — for head+stopWhenFull — whether to stop. + const droppedThisChunk = this.omittedAt(this.total) > this.omittedAt(before) + return { + kept: !droppedThisChunk, + truncated: this.omittedAt(this.total) > 0, + shouldStop: this.allowStop && droppedThisChunk, + } + } + + /** Bytes omitted once `total` bytes have been seen: `total − keptPrefix − keptSuffix`. */ + private omittedAt(total: number): number { + const prefixLen = Math.min(total, this.prefixCap) + const suffixLen = Math.min(total - prefixLen, this.suffixCap) + return total - prefixLen - suffixLen + } + + /** + * Finalize: decode the retained prefix and suffix (each trimmed to a UTF-8 + * boundary at its cut) and report the exact or lower-bound omitted byte count. + * `head` + `stopWhenFull` yields `atLeast` (a lower bound — the caller was + * asked to stop before the true size was known); every other case reads to the + * end and yields `exact`. + * + * @returns The {@link RetainedText} snapshot (safe to hand to a formatter). + */ + finish(): RetainedText { + const prefixLen = Math.min(this.total, this.prefixCap) + const suffixLen = Math.min(this.total - prefixLen, this.suffixCap) + const omitted = this.omittedAt(this.total) + const truncated = omitted > 0 + + // A cut exists at the prefix end only if content followed it (moved to the + // suffix or omitted); likewise the suffix start is a cut only if content + // preceded it. When the whole stream fits in one side, pass bytes through + // untrimmed so valid output is never altered. + let prefix = concat(this.prefixChunks) + if (suffixLen > 0 || omitted > 0) prefix = trimTrailingPartialUtf8(prefix) + + const suffixBuf = concat(this.suffixChunks) + let suffix = suffixBuf.subarray(this.suffixHeld - suffixLen) + if (prefixLen > 0 || omitted > 0) suffix = trimLeadingContinuationUtf8(suffix) + + return { + // Decode the two sides separately so a codepoint is never reconstructed + // across the omitted middle. + text: decoder.decode(prefix) + decoder.decode(suffix), + truncated, + omittedBytes: truncated + ? { kind: this.allowStop ? 'atLeast' : 'exact', count: omitted } + : { kind: 'none' }, + } + } +} + +/** Concatenate chunks into one contiguous buffer (their exact total length). */ +function concat(chunks: readonly Uint8Array[]): Uint8Array { + let length = 0 + for (const chunk of chunks) length += chunk.length + const out = new Uint8Array(length) + let offset = 0 + for (const chunk of chunks) { + out.set(chunk, offset) + offset += chunk.length + } + return out +} + +/** + * Standardized, false-precision-safe wording for one {@link Omitted} value — + * the "may standardize omission wording" half the library owns. `exact` prints + * the count (`Omitted 3 items`); `atLeast`/`unknown` print NO count, because an + * early stop knows only that more was dropped, not how much (claiming "omitted + * 1" when the true total may be huge is the false-precision trap the `atLeast` + * variant exists to avoid). `none` is the empty string. + * + * @param omitted The omission metadata from a retainer result. + * @param unit The noun for the omitted quantity (`items`, `bytes`, `chars`, `lines`). + * @returns A neutral clause (no trailing space), or `''` when nothing was omitted. + */ +export function describeOmitted(omitted: Omitted, unit: RetentionNotice['unit']): string { + switch (omitted.kind) { + case 'none': + return '' + case 'exact': + return `Omitted ${omitted.count} ${unit}.` + case 'atLeast': + case 'unknown': + return `More ${unit} were omitted.` + } +} + +/** + * Turn a {@link RetentionNotice} into a one-line footer: the library-owned + * standardized omission clause ({@link describeOmitted}) followed by the tool's + * own recovery guidance. The library never owns recovery words — only the tool + * knows the action ("narrow the pattern", "fetch a more specific URL", "read the + * spill file") — so `recovery` supplies them and receives the full notice to + * phrase from (`kept`, `limit`, `omitted`, …). Either half may be empty; the two + * are joined with a single space. + * + * @param notice The neutral retention outcome. + * @param recovery Tool-supplied guidance builder; receives the notice, returns a sentence (or `''`). + * @returns The combined footer line. + */ +export function formatRetentionNotice( + notice: RetentionNotice, + recovery: (notice: RetentionNotice) => string, +): string { + return [describeOmitted(notice.omitted, notice.unit), recovery(notice)] + .filter(part => part.length > 0) + .join(' ') +} diff --git a/packages/util/retention/tests/retention.spec.ts b/packages/util/retention/tests/retention.spec.ts new file mode 100644 index 0000000000..b3401cf50a --- /dev/null +++ b/packages/util/retention/tests/retention.spec.ts @@ -0,0 +1,348 @@ +import { describe, expect, it } from 'vitest' +import { + describeOmitted, + formatRetentionNotice, + ItemRetainer, + type Omitted, + type RetentionNotice, + TextRetainer, +} from '@deepseek-ai/dsh-retention' + +/** Decode a RetainedText via a round-trip helper for readable UTF-8 assertions. */ +const utf8 = (s: string): Uint8Array => new TextEncoder().encode(s) + +describe('ItemRetainer — head, stopWhenFull (glob/grep early stop)', () => { + it('keeps the first maxItems and asks to stop on the probe item', () => { + const r = new ItemRetainer({ kind: 'head', maxItems: 2, stop: 'stopWhenFull' }) + expect(r.push('a')).toEqual({ kept: true, truncated: false, shouldStop: false }) + expect(r.push('b')).toEqual({ kept: true, truncated: false, shouldStop: false }) + // The (maxItems + 1)th valid item is the probe: not retained, sets truncated, + // and shouldStop tells the caller to kill the upstream. + expect(r.push('c')).toEqual({ kept: false, truncated: true, shouldStop: true }) + + const result = r.finish() + expect(result.items).toEqual(['a', 'b']) + expect(result.kept).toBe(2) + expect(result.seen).toBe(3) + expect(result.truncated).toBe(true) + // Early stop knows only a lower bound, never an exact total. + expect(result.omitted).toEqual({ kind: 'atLeast', count: 1 }) + }) + + it('reports none when everything fits', () => { + const r = new ItemRetainer({ kind: 'head', maxItems: 3, stop: 'stopWhenFull' }) + r.push(1) + r.push(2) + const result = r.finish() + expect(result.items).toEqual([1, 2]) + expect(result.truncated).toBe(false) + expect(result.omitted).toEqual({ kind: 'none' }) + }) +}) + +describe('ItemRetainer — head, readToEnd (exact omission)', () => { + it('keeps draining past the cap and reports an exact omitted count', () => { + const r = new ItemRetainer({ kind: 'head', maxItems: 1, stop: 'readToEnd' }) + expect(r.push('a')).toEqual({ kept: true, truncated: false, shouldStop: false }) + // readToEnd never asks to stop — the caller must keep pushing to count exactly. + expect(r.push('b')).toEqual({ kept: false, truncated: true, shouldStop: false }) + expect(r.push('c')).toEqual({ kept: false, truncated: true, shouldStop: false }) + + const result = r.finish() + expect(result.items).toEqual(['a']) + expect(result.seen).toBe(3) + expect(result.omitted).toEqual({ kind: 'exact', count: 2 }) + }) +}) + +describe('ItemRetainer — zero budget', () => { + it('keeps nothing; first item is the probe under stopWhenFull', () => { + const r = new ItemRetainer({ kind: 'head', maxItems: 0, stop: 'stopWhenFull' }) + expect(r.push('a')).toEqual({ kept: false, truncated: true, shouldStop: true }) + const result = r.finish() + expect(result.items).toEqual([]) + expect(result.kept).toBe(0) + expect(result.omitted).toEqual({ kind: 'atLeast', count: 1 }) + }) + + it('rejects a non-integer / negative maxItems', () => { + expect(() => new ItemRetainer({ kind: 'head', maxItems: -1, stop: 'readToEnd' })) + .toThrow(/maxItems must be a non-negative integer/) + expect(() => new ItemRetainer({ kind: 'head', maxItems: 1.5, stop: 'readToEnd' })) + .toThrow(/maxItems must be a non-negative integer/) + }) +}) + +describe('TextRetainer — head, stopWhenFull (early body stop)', () => { + it('keeps the prefix and asks to stop on the overflowing chunk', () => { + const r = new TextRetainer({ kind: 'head', maxBytes: 5, stop: 'stopWhenFull' }) + expect(r.push('abc')).toEqual({ kept: true, truncated: false, shouldStop: false }) + // 'de' fills the cap exactly (5 bytes) — still fully kept. + expect(r.push('de')).toEqual({ kept: true, truncated: false, shouldStop: false }) + // 'fgh' is wholly dropped: kept:false, and stopWhenFull → shouldStop. + expect(r.push('fgh')).toEqual({ kept: false, truncated: true, shouldStop: true }) + + const result = r.finish() + expect(result.text).toBe('abcde') + expect(result.truncated).toBe(true) + // Early stop: a lower bound, not an exact size. + expect(result.omittedBytes).toEqual({ kind: 'atLeast', count: 3 }) + }) + + it('flags a partially-dropped chunk as not fully kept', () => { + const r = new TextRetainer({ kind: 'head', maxBytes: 4, stop: 'stopWhenFull' }) + r.push('ab') + // 'cde' straddles the cap: 'c','d' fit, 'e' drops → kept:false, shouldStop. + expect(r.push('cde')).toEqual({ kept: false, truncated: true, shouldStop: true }) + expect(r.finish().text).toBe('abcd') + }) +}) + +describe('TextRetainer — head, readToEnd (exact omission)', () => { + it('keeps the prefix, drains the rest, and counts exactly', () => { + const r = new TextRetainer({ kind: 'head', maxBytes: 3, stop: 'readToEnd' }) + r.push('abc') + expect(r.push('defg')).toEqual({ kept: false, truncated: true, shouldStop: false }) + const result = r.finish() + expect(result.text).toBe('abc') + expect(result.omittedBytes).toEqual({ kind: 'exact', count: 4 }) + }) +}) + +describe('TextRetainer — tail (exact omission, reads to end)', () => { + it('keeps the final maxBytes and reports exact omission', () => { + const r = new TextRetainer({ kind: 'tail', maxBytes: 4 }) + // tail never asks to stop — it must read to the end to know the true suffix. + expect(r.push('hello')).toEqual({ kept: false, truncated: true, shouldStop: false }) + r.push('world') + const result = r.finish() + expect(result.text).toBe('orld') // last 4 bytes of 'helloworld' + expect(result.truncated).toBe(true) + expect(result.omittedBytes).toEqual({ kind: 'exact', count: 6 }) + }) + + it('keeps everything when the stream is under the cap', () => { + const r = new TextRetainer({ kind: 'tail', maxBytes: 100 }) + r.push('short') + const result = r.finish() + expect(result.text).toBe('short') + expect(result.truncated).toBe(false) + expect(result.omittedBytes).toEqual({ kind: 'none' }) + }) + + it('drops old chunks as they slide out of the tail window', () => { + const r = new TextRetainer({ kind: 'tail', maxBytes: 3 }) + for (const c of ['11', '22', '33', '44']) r.push(c) + // Only the final 3 bytes survive; earlier whole chunks are dropped. + expect(r.finish().text).toBe('344') + }) +}) + +describe('TextRetainer — headTail (prefix + suffix, omit the middle)', () => { + it('keeps a stable head and tail, omitting the middle exactly', () => { + const r = new TextRetainer({ kind: 'headTail', headBytes: 3, tailBytes: 3 }) + r.push('abcdefghij') // 10 bytes: head 'abc', tail 'hij', middle 'defg' omitted + const result = r.finish() + expect(result.text).toBe('abchij') + expect(result.truncated).toBe(true) + expect(result.omittedBytes).toEqual({ kind: 'exact', count: 4 }) + }) + + it('does not double-count when head+tail cover the whole stream', () => { + const r = new TextRetainer({ kind: 'headTail', headBytes: 3, tailBytes: 3 }) + r.push('abcdef') // exactly head(3) + tail(3), nothing omitted + const result = r.finish() + expect(result.text).toBe('abcdef') + expect(result.truncated).toBe(false) + expect(result.omittedBytes).toEqual({ kind: 'none' }) + }) +}) + +describe('TextRetainer — zero budgets', () => { + it('head maxBytes 0 keeps nothing and stops on first byte (stopWhenFull)', () => { + const r = new TextRetainer({ kind: 'head', maxBytes: 0, stop: 'stopWhenFull' }) + expect(r.push('x')).toEqual({ kept: false, truncated: true, shouldStop: true }) + const result = r.finish() + expect(result.text).toBe('') + expect(result.omittedBytes).toEqual({ kind: 'atLeast', count: 1 }) + }) + + it('an empty stream omits nothing', () => { + const r = new TextRetainer({ kind: 'headTail', headBytes: 2, tailBytes: 2 }) + const result = r.finish() + expect(result.text).toBe('') + expect(result.truncated).toBe(false) + expect(result.omittedBytes).toEqual({ kind: 'none' }) + }) + + it('rejects non-integer / negative byte budgets', () => { + expect(() => new TextRetainer({ kind: 'head', maxBytes: -1, stop: 'readToEnd' })) + .toThrow(/maxBytes must be a non-negative integer/) + expect(() => new TextRetainer({ kind: 'tail', maxBytes: 2.5 })) + .toThrow(/maxBytes must be a non-negative integer/) + expect(() => new TextRetainer({ kind: 'headTail', headBytes: -1, tailBytes: 2 })) + .toThrow(/headBytes must be a non-negative integer/) + expect(() => new TextRetainer({ kind: 'headTail', headBytes: 2, tailBytes: 1.1 })) + .toThrow(/tailBytes must be a non-negative integer/) + }) +}) + +describe('TextRetainer — UTF-8 boundary handling', () => { + it('trims a partial codepoint at the head cut instead of emitting U+FFFD', () => { + // '€' is 3 bytes (E2 82 AC). A 2-byte head cap keeps 'a' (61) + the first + // byte of '€' (E2); that partial lead byte must be trimmed, not decoded to + // a replacement char. + const r = new TextRetainer({ kind: 'head', maxBytes: 2, stop: 'readToEnd' }) + r.push('a€b') // bytes: 61 E2 82 AC 62 + const result = r.finish() + expect(result.text).toBe('a') // partial '€' dropped, no U+FFFD + expect(result.text).not.toContain('�') + // Omission counts BYTES not kept by retention: 5 total − 2 prefix = 3. + expect(result.omittedBytes).toEqual({ kind: 'exact', count: 3 }) + }) + + it('trims a leading partial codepoint at the tail cut', () => { + // Tail cap 2 over 'a€b' (5 bytes) keeps AC 62 — AC is a continuation byte + // (the middle of '€'); the leading continuation byte is dropped so the tail + // begins on a boundary. + const r = new TextRetainer({ kind: 'tail', maxBytes: 2 }) + r.push('a€b') + const result = r.finish() + expect(result.text).toBe('b') // partial '€' at the front dropped + expect(result.text).not.toContain('�') + }) + + it('preserves a whole multibyte codepoint that fits exactly', () => { + const r = new TextRetainer({ kind: 'head', maxBytes: 3, stop: 'readToEnd' }) + r.push('€x') // '€' is exactly 3 bytes + expect(r.finish().text).toBe('€') + }) + + it('does not reconstruct a codepoint across the omitted middle', () => { + // headBytes ends mid-'€' and tailBytes starts mid-another '€'; neither cut + // may glue a valid codepoint across the gap. + const r = new TextRetainer({ kind: 'headTail', headBytes: 2, tailBytes: 2 }) + r.push('€€€') // 9 bytes + const result = r.finish() + expect(result.text).not.toContain('�') + expect(result.truncated).toBe(true) + }) + + it('accepts a raw Uint8Array chunk', () => { + const r = new TextRetainer({ kind: 'head', maxBytes: 2, stop: 'readToEnd' }) + r.push(utf8('xy')) + r.push(utf8('z')) + expect(r.finish().text).toBe('xy') + }) + + it('trims a partial 2-byte codepoint at the head cut', () => { + // 'é' is 2 bytes (C3 A9). A 2-byte head cap over 'aé' keeps 'a' (61) + the + // lead byte of 'é' (C3) — an incomplete 2-byte sequence to trim. + const r = new TextRetainer({ kind: 'head', maxBytes: 2, stop: 'readToEnd' }) + r.push('aé') // bytes: 61 C3 A9 + const result = r.finish() + expect(result.text).toBe('a') + expect(result.text).not.toContain('�') + }) + + it('trims a partial 4-byte codepoint (emoji) at the head cut', () => { + // '😀' is 4 bytes (F0 9F 98 80). A 3-byte head cap keeps 'a' + the first two + // bytes of the emoji — an incomplete 4-byte sequence that must be trimmed. + const r = new TextRetainer({ kind: 'head', maxBytes: 3, stop: 'readToEnd' }) + r.push('a😀') // bytes: 61 F0 9F 98 80 + const result = r.finish() + expect(result.text).toBe('a') + expect(result.text).not.toContain('�') + }) + + it('keeps a whole 4-byte codepoint that fits exactly', () => { + const r = new TextRetainer({ kind: 'head', maxBytes: 4, stop: 'readToEnd' }) + r.push('😀x') + expect(r.finish().text).toBe('😀') + }) + + it('leaves a head cut ending on a stray continuation run untouched', () => { + // A cut whose trailing bytes are ALL continuation bytes with no lead in + // reach is not a trimmable incomplete sequence — the trimmer bails (no lead + // byte found) and leaves them for the non-fatal decoder to replace. + const r = new TextRetainer({ kind: 'head', maxBytes: 2, stop: 'readToEnd' }) + // 0x80 0x80 are bare continuation bytes; 'z' follows so the head keeps just + // the two continuation bytes and the cut lands right after them. + r.push(new Uint8Array([0x80, 0x80, 0x7a])) + const result = r.finish() + // The trimmer did not throw and did not eat the bytes as a partial sequence; + // only the trailing 'z' is omitted by the 2-byte cap. + expect(result.omittedBytes).toEqual({ kind: 'exact', count: 1 }) + }) + + it('leaves a head cut ending on an invalid lead byte untouched', () => { + // 0xF8 is not a valid UTF-8 lead byte (only 0x00–0xF7 lead). The trimmer + // recognizes it as "not a lead" (expected length 0) and leaves the byte in + // place rather than trimming a phantom partial sequence. + const r = new TextRetainer({ kind: 'head', maxBytes: 1, stop: 'readToEnd' }) + r.push(new Uint8Array([0xf8, 0x61])) // 0xF8 kept, 'a' dropped by the 1-byte cap + const result = r.finish() + expect(result.omittedBytes).toEqual({ kind: 'exact', count: 1 }) + }) +}) + +describe('describeOmitted — false precision safety', () => { + it('prints an exact count for exact omission', () => { + expect(describeOmitted({ kind: 'exact', count: 3 }, 'items')).toBe('Omitted 3 items.') + expect(describeOmitted({ kind: 'exact', count: 12 }, 'bytes')).toBe('Omitted 12 bytes.') + }) + + it('prints NO count for atLeast (early stop) and unknown', () => { + // The whole point of atLeast: never claim "omitted 1" when the true count is + // unknown. Both atLeast and unknown collapse to a countless clause. + expect(describeOmitted({ kind: 'atLeast', count: 1 }, 'items')).toBe('More items were omitted.') + expect(describeOmitted({ kind: 'unknown' }, 'lines')).toBe('More lines were omitted.') + }) + + it('returns empty string when nothing was omitted', () => { + expect(describeOmitted({ kind: 'none' }, 'chars')).toBe('') + }) +}) + +describe('formatRetentionNotice', () => { + const notice = (omitted: Omitted): RetentionNotice => ({ + scope: 'grep', + strategy: 'head', + unit: 'items', + limit: 100, + kept: 100, + omitted, + }) + + it('joins the standardized omission clause with the tool recovery guidance', () => { + const out = formatRetentionNotice( + notice({ kind: 'atLeast', count: 1 }), + ({ kept }) => `Results capped at ${kept}. Narrow the pattern, path, or include to see more.`, + ) + expect(out).toBe('More items were omitted. Results capped at 100. Narrow the pattern, path, or include to see more.') + }) + + it('omits the empty half when nothing was omitted', () => { + const out = formatRetentionNotice(notice({ kind: 'none' }), () => 'Recovery text.') + expect(out).toBe('Recovery text.') + }) + + it('omits the empty half when the tool supplies no recovery text', () => { + const out = formatRetentionNotice(notice({ kind: 'exact', count: 2 }), () => '') + expect(out).toBe('Omitted 2 items.') + }) + + it('passes the full notice to the recovery builder (limit as a head/tail pair)', () => { + const headTail: RetentionNotice = { + scope: 'bash stdout', + strategy: 'headTail', + unit: 'bytes', + limit: { head: 2_000, tail: 2_000 }, + kept: 4_000, + omitted: { kind: 'exact', count: 500 }, + } + const out = formatRetentionNotice(headTail, n => + typeof n.limit === 'object' ? `Kept ${n.limit.head}B head + ${n.limit.tail}B tail.` : '') + expect(out).toBe('Omitted 500 bytes. Kept 2000B head + 2000B tail.') + }) +}) diff --git a/packages/util/retention/tsconfig.json b/packages/util/retention/tsconfig.json new file mode 100644 index 0000000000..749cb0208e --- /dev/null +++ b/packages/util/retention/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "lib/types" + }, + "include": [ + "src" + ], + "references": [] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 54050bd0f4..3bf60ac60a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -963,6 +963,12 @@ importers: specifier: ^4.0.0-rc.6 version: 4.0.0-rc.6(@cordisjs/plugin-include@1.0.4)(@cordisjs/plugin-loader@1.0.0-rc.4) + packages/util/retention: + devDependencies: + cordis: + specifier: ^4.0.0-rc.6 + version: 4.0.0-rc.6(@cordisjs/plugin-include@1.0.4)(@cordisjs/plugin-loader@1.0.0-rc.4) + packages/util/timeout: devDependencies: cordis: diff --git a/tsconfig.build.json b/tsconfig.build.json index ebf8ffef14..8f1a869d91 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -12,6 +12,7 @@ { "path": "./vendor/logger-console" }, { "path": "./packages/util/brand" }, { "path": "./packages/util/timeout" }, + { "path": "./packages/util/retention" }, { "path": "./packages/llm/llm" }, { "path": "./packages/core/session" }, { "path": "./packages/session-persistence/session-persistence" }, diff --git a/tsconfig.json b/tsconfig.json index 49cce594dd..4a4cc1668f 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -23,6 +23,7 @@ { "path": "./vendor/logger-console" }, { "path": "./packages/util/brand" }, { "path": "./packages/util/timeout" }, + { "path": "./packages/util/retention" }, { "path": "./packages/llm/llm" }, { "path": "./packages/core/session" }, { "path": "./packages/session-persistence/session-persistence" }, From cbfdf9d0922736151bab3a406dd1a951ef7bc551 Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Tue, 7 Jul 2026 09:46:15 +0800 Subject: [PATCH 03/24] fix: preserve a codepoint spanning the head|tail split (codex round 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When headTail budgets cover the whole stream, omitted is 0 and the two retained halves are contiguous — the split is artificial and a multibyte codepoint can straddle it. finish() now decodes the contiguous buffer as one in that case; the per-side UTF-8 boundary trims and separate decoding apply only when a real middle gap exists. Without this, a headTail retainer could drop a character while reporting truncated:false. --- packages/util/retention/src/index.ts | 25 +++++++++--------- .../util/retention/tests/retention.spec.ts | 26 +++++++++++++++++++ 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/packages/util/retention/src/index.ts b/packages/util/retention/src/index.ts index bf754e2a8b..91f6412691 100644 --- a/packages/util/retention/src/index.ts +++ b/packages/util/retention/src/index.ts @@ -391,21 +391,22 @@ export class TextRetainer { const omitted = this.omittedAt(this.total) const truncated = omitted > 0 - // A cut exists at the prefix end only if content followed it (moved to the - // suffix or omitted); likewise the suffix start is a cut only if content - // preceded it. When the whole stream fits in one side, pass bytes through - // untrimmed so valid output is never altered. - let prefix = concat(this.prefixChunks) - if (suffixLen > 0 || omitted > 0) prefix = trimTrailingPartialUtf8(prefix) + const prefix = concat(this.prefixChunks) // exactly prefixLen bytes (prefixHeld === prefixLen) + const suffix = concat(this.suffixChunks).subarray(this.suffixHeld - suffixLen) - const suffixBuf = concat(this.suffixChunks) - let suffix = suffixBuf.subarray(this.suffixHeld - suffixLen) - if (prefixLen > 0 || omitted > 0) suffix = trimLeadingContinuationUtf8(suffix) + // With nothing omitted, prefix and suffix are ADJACENT slices of one stream + // (prefixLen + suffixLen === total), so the head|tail split is artificial: a + // codepoint may span it. Decode the contiguous whole as one buffer — trimming + // or decoding the halves separately here would corrupt a boundary-spanning + // codepoint though no content was actually dropped. Only a real omitted gap + // makes each side a true cut: trim each to a UTF-8 boundary and decode + // separately so a codepoint is never reconstructed across the gap. + const text = truncated + ? decoder.decode(trimTrailingPartialUtf8(prefix)) + decoder.decode(trimLeadingContinuationUtf8(suffix)) + : decoder.decode(concat([prefix, suffix])) return { - // Decode the two sides separately so a codepoint is never reconstructed - // across the omitted middle. - text: decoder.decode(prefix) + decoder.decode(suffix), + text, truncated, omittedBytes: truncated ? { kind: this.allowStop ? 'atLeast' : 'exact', count: omitted } diff --git a/packages/util/retention/tests/retention.spec.ts b/packages/util/retention/tests/retention.spec.ts index b3401cf50a..fd512508be 100644 --- a/packages/util/retention/tests/retention.spec.ts +++ b/packages/util/retention/tests/retention.spec.ts @@ -156,6 +156,32 @@ describe('TextRetainer — headTail (prefix + suffix, omit the middle)', () => { expect(result.truncated).toBe(false) expect(result.omittedBytes).toEqual({ kind: 'none' }) }) + + it('does not drop a codepoint that spans the head|tail split when nothing is omitted', () => { + // Regression: with head+tail covering the whole stream, the split is + // artificial — a multibyte codepoint may straddle it. 'éab' is C3 A9 61 62 + // (4 bytes); headBytes 1 + tailBytes 3 covers all 4 with omitted === 0, but + // the split falls INSIDE 'é'. The bytes are contiguous, so the full 'éab' + // must survive — not be trimmed to 'ab'. + const r = new TextRetainer({ kind: 'headTail', headBytes: 1, tailBytes: 3 }) + r.push('éab') + const result = r.finish() + expect(result.text).toBe('éab') + expect(result.truncated).toBe(false) + expect(result.omittedBytes).toEqual({ kind: 'none' }) + }) + + it('still trims boundary partials once a real middle is omitted', () => { + // With a genuine gap the two sides ARE true cuts: '€' (3 bytes) split across + // the omitted middle must not resurface as a replacement char on either side. + const r = new TextRetainer({ kind: 'headTail', headBytes: 2, tailBytes: 2 }) + r.push('a€€b') // 8 bytes; head 'a'+partial, tail partial+'b', middle omitted + const result = r.finish() + expect(result.truncated).toBe(true) + expect(result.text).not.toContain('�') + expect(result.text.startsWith('a')).toBe(true) + expect(result.text.endsWith('b')).toBe(true) + }) }) describe('TextRetainer — zero budgets', () => { From f4acb1cf05719859ac6b4f54e9354efdf0adb960 Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Tue, 7 Jul 2026 10:01:03 +0800 Subject: [PATCH 04/24] fix: report omitted bytes against retained text, not budget (codex round 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit finish() derived the exact omitted count from the pre-trim prefix/suffix budget, but the UTF-8 boundary trims drop additional partial-codepoint bytes, so an "Omitted N bytes" notice overstated what was kept (head maxBytes:2 over 'a€b' returned 'a' but claimed exact 3 when 4 bytes are absent). Derive the count from the bytes actually returned (total − keptPrefix − keptSuffix) so exact metadata matches the text and atLeast stays a valid lower bound. --- packages/util/retention/src/index.ts | 32 ++++++++++++------- .../util/retention/tests/retention.spec.ts | 21 ++++++++++-- 2 files changed, 40 insertions(+), 13 deletions(-) diff --git a/packages/util/retention/src/index.ts b/packages/util/retention/src/index.ts index 91f6412691..f0f40ba7af 100644 --- a/packages/util/retention/src/index.ts +++ b/packages/util/retention/src/index.ts @@ -388,23 +388,33 @@ export class TextRetainer { finish(): RetainedText { const prefixLen = Math.min(this.total, this.prefixCap) const suffixLen = Math.min(this.total - prefixLen, this.suffixCap) - const omitted = this.omittedAt(this.total) - const truncated = omitted > 0 const prefix = concat(this.prefixChunks) // exactly prefixLen bytes (prefixHeld === prefixLen) const suffix = concat(this.suffixChunks).subarray(this.suffixHeld - suffixLen) - // With nothing omitted, prefix and suffix are ADJACENT slices of one stream - // (prefixLen + suffixLen === total), so the head|tail split is artificial: a - // codepoint may span it. Decode the contiguous whole as one buffer — trimming - // or decoding the halves separately here would corrupt a boundary-spanning - // codepoint though no content was actually dropped. Only a real omitted gap - // makes each side a true cut: trim each to a UTF-8 boundary and decode - // separately so a codepoint is never reconstructed across the gap. - const text = truncated - ? decoder.decode(trimTrailingPartialUtf8(prefix)) + decoder.decode(trimLeadingContinuationUtf8(suffix)) + // With nothing omitted by budget, prefix and suffix are ADJACENT slices of + // one stream (prefixLen + suffixLen === total), so the head|tail split is + // artificial: a codepoint may span it. Decode the contiguous whole as one + // buffer — trimming or decoding the halves separately here would corrupt a + // boundary-spanning codepoint though no content was dropped. Only a real + // omitted gap makes each side a true cut: trim each to a UTF-8 boundary and + // decode separately so a codepoint is never reconstructed across the gap. + const budgetOmitted = this.omittedAt(this.total) + const [keptPrefix, keptSuffix] = budgetOmitted > 0 + ? [trimTrailingPartialUtf8(prefix), trimLeadingContinuationUtf8(suffix)] + : [prefix, suffix] + const text = budgetOmitted > 0 + ? decoder.decode(keptPrefix) + decoder.decode(keptSuffix) : decoder.decode(concat([prefix, suffix])) + // Report omission against the bytes ACTUALLY returned, not the pre-trim + // budget: a boundary trim drops partial-codepoint bytes too, so an exact + // count derived from the budget alone would overstate the retained text (and + // any "Omitted N bytes" notice built from it would be a lie). total_seen − + // retained stays a valid lower bound under `atLeast` (true total ≥ seen). + const omitted = this.total - keptPrefix.length - keptSuffix.length + const truncated = omitted > 0 + return { text, truncated, diff --git a/packages/util/retention/tests/retention.spec.ts b/packages/util/retention/tests/retention.spec.ts index fd512508be..ec424595cb 100644 --- a/packages/util/retention/tests/retention.spec.ts +++ b/packages/util/retention/tests/retention.spec.ts @@ -223,8 +223,10 @@ describe('TextRetainer — UTF-8 boundary handling', () => { const result = r.finish() expect(result.text).toBe('a') // partial '€' dropped, no U+FFFD expect(result.text).not.toContain('�') - // Omission counts BYTES not kept by retention: 5 total − 2 prefix = 3. - expect(result.omittedBytes).toEqual({ kind: 'exact', count: 3 }) + // Omission counts bytes ACTUALLY absent from the returned text, including + // the partial 'E2' the boundary trim dropped: 5 total − 1 retained = 4 + // (not the pre-trim budget of 3, which would overstate what was kept). + expect(result.omittedBytes).toEqual({ kind: 'exact', count: 4 }) }) it('trims a leading partial codepoint at the tail cut', () => { @@ -236,6 +238,21 @@ describe('TextRetainer — UTF-8 boundary handling', () => { const result = r.finish() expect(result.text).toBe('b') // partial '€' at the front dropped expect(result.text).not.toContain('�') + // Honest count: 5 total − 1 retained ('b') = 4, including the trimmed AC. + expect(result.omittedBytes).toEqual({ kind: 'exact', count: 4 }) + }) + + it('omitted count matches the bytes actually absent, across a headTail boundary trim', () => { + // Regression: the exact count must equal total − retained (post-trim), never + // the pre-trim budget. 'a€€b' is 8 bytes (61 E2828C… ×2 61? no: 61 E2 82 AC + // E2 82 AC 62). headBytes 2 keeps 'a'+partial-E2 → trims to 'a' (1 byte); + // tailBytes 2 keeps partial-AC+'b' → trims to 'b' (1 byte). Retained text is + // 2 bytes, so omitted must be 8 − 2 = 6 — not the budget's 8 − 2 − 2 = 4. + const r = new TextRetainer({ kind: 'headTail', headBytes: 2, tailBytes: 2 }) + r.push('a€€b') + const result = r.finish() + const retainedBytes = new TextEncoder().encode(result.text).length + expect(result.omittedBytes).toEqual({ kind: 'exact', count: 8 - retainedBytes }) }) it('preserves a whole multibyte codepoint that fits exactly', () => { From 4f2f34c6fd04f3787a5e74712f3fcbb6c97461b0 Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Tue, 7 Jul 2026 10:25:29 +0800 Subject: [PATCH 05/24] docs: BashRunResult timedOut/aborted are first-cause, not independent (codex round 3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The timeout convergence on this branch made timedOut/aborted mutually exclusive — one fused deadline reports the single cause that first cut the command short — as the timeout-library RFC documents deliberately. The seam type's JSDoc still described the old independent latches, so a consumer could code the wrong contract. State first-cause classification on both fields and cross-link the RFC. Docs-only; the code already matches. --- packages/bash/bash/src/types.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/bash/bash/src/types.ts b/packages/bash/bash/src/types.ts index 9acd5c7cb7..457ca87d9e 100644 --- a/packages/bash/bash/src/types.ts +++ b/packages/bash/bash/src/types.ts @@ -131,9 +131,19 @@ export interface BashRunResult { exitCode: number | null /** Terminating signal (e.g. 'SIGTERM'); null on normal exit. */ signal: NodeJS.Signals | null - /** True when the executor's own timeout killed the command. */ + /** + * True when the executor's own timeout was the FIRST cause to cut the command + * short. Mutually exclusive with {@link aborted}: one fused deadline drives + * both the timeout and the caller's cancellation, so a timeout and an abort + * racing before process close report the single first-abort cause, not both + * (see the [timeout-library RFC](../../../../docs/rfc/implemented/architecture/2026-07-06-timeout-deadline-library.md)). + */ timedOut: boolean - /** True when the caller's AbortSignal killed the command. */ + /** + * True when the caller's `AbortSignal` was the FIRST cause to kill the command + * (and it was not the executor's own timeout). Mutually exclusive with + * {@link timedOut} — see there for the first-cause classification. + */ aborted: boolean /** The effective timeout applied to this run (after defaulting/capping). */ timeoutMs: number From 463b72ce9631e7c034089cd508e3ccacbcb75e39 Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Wed, 8 Jul 2026 19:20:50 +0800 Subject: [PATCH 06/24] feat(spill): add tool-output spill seam, local backend, and policy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Oversized plain-text tool results now spill to a session-scoped file and return a bounded preview plus the spill path, so a verbose result stays readable via `read` without consuming the next model request in full. - dsh-spill: minimal SpillFiles seam (saveText → session-scoped SpillPath) - dsh-spill-local: private 0700 session dirs, traversal-safe names, exclusive owner-only writes - dsh-spill-policy: tools/post-execute transformer; no-op unless maxInlineBytes is set; skips read; best-effort on save failure (never turns a success into an isError) web_fetch is the showcase — no tool-specific spill code. The coding-agent example loads the stack so its keyless Loader smoke guards the namespace-plugin export shape. Snapshot gap for a transcript-visible web_fetch spill is recorded in the RFC's Consequences (ACP replay is keyless and cannot hit the web). --- docs/capability-seams.md | 8 + docs/cordis-catalog/services.md | 16 ++ docs/event-producer-consumer.md | 2 +- docs/module-graph.md | 17 ++ docs/rfc/INDEX.md | 1 + .../2026-07-08-tool-output-spill-files.md | 191 +++++++++++++++++ examples/coding-agent/composition.md | 6 + examples/coding-agent/cordis.yml | 13 ++ packages/README.md | 1 + packages/spill/README.md | 13 ++ packages/spill/spill-local/README.md | 19 ++ packages/spill/spill-local/package.json | 38 ++++ packages/spill/spill-local/src/index.ts | 61 ++++++ packages/spill/spill-local/src/store.ts | 102 +++++++++ .../spill-local/tests/spill-local.spec.ts | 138 ++++++++++++ packages/spill/spill-local/tsconfig.json | 14 ++ packages/spill/spill-policy/README.md | 31 +++ packages/spill/spill-policy/package.json | 44 ++++ packages/spill/spill-policy/src/index.ts | 149 +++++++++++++ packages/spill/spill-policy/src/types.ts | 26 +++ .../spill-policy/tests/spill-policy.spec.ts | 196 ++++++++++++++++++ packages/spill/spill-policy/tsconfig.json | 18 ++ packages/spill/spill/README.md | 27 +++ packages/spill/spill/package.json | 36 ++++ packages/spill/spill/src/index.ts | 60 ++++++ packages/spill/spill/src/types.ts | 68 ++++++ packages/spill/spill/tests/service.spec.ts | 56 +++++ packages/spill/spill/tsconfig.json | 15 ++ packages/web/tool-web/package.json | 2 + packages/web/tool-web/tests/spill.spec.ts | 93 +++++++++ pnpm-lock.yaml | 71 +++++++ scripts/gen-doc-graphs.ts | 10 + scripts/gen-module-graph.ts | 1 + tsconfig.base.json | 1 + tsconfig.build.json | 3 + tsconfig.json | 3 + 36 files changed, 1549 insertions(+), 1 deletion(-) create mode 100644 docs/rfc/implemented/architecture/2026-07-08-tool-output-spill-files.md create mode 100644 packages/spill/README.md create mode 100644 packages/spill/spill-local/README.md create mode 100644 packages/spill/spill-local/package.json create mode 100644 packages/spill/spill-local/src/index.ts create mode 100644 packages/spill/spill-local/src/store.ts create mode 100644 packages/spill/spill-local/tests/spill-local.spec.ts create mode 100644 packages/spill/spill-local/tsconfig.json create mode 100644 packages/spill/spill-policy/README.md create mode 100644 packages/spill/spill-policy/package.json create mode 100644 packages/spill/spill-policy/src/index.ts create mode 100644 packages/spill/spill-policy/src/types.ts create mode 100644 packages/spill/spill-policy/tests/spill-policy.spec.ts create mode 100644 packages/spill/spill-policy/tsconfig.json create mode 100644 packages/spill/spill/README.md create mode 100644 packages/spill/spill/package.json create mode 100644 packages/spill/spill/src/index.ts create mode 100644 packages/spill/spill/src/types.ts create mode 100644 packages/spill/spill/tests/service.spec.ts create mode 100644 packages/spill/spill/tsconfig.json create mode 100644 packages/web/tool-web/tests/spill.spec.ts diff --git a/docs/capability-seams.md b/docs/capability-seams.md index 414ee898d1..18af1f2ea8 100644 --- a/docs/capability-seams.md +++ b/docs/capability-seams.md @@ -60,6 +60,10 @@ flowchart LR pkg_web_search_perplexity["web-search-perplexity"] pkg_web_search_deepseek["web-search-deepseek"] pkg_web_fetch_local["web-fetch-local"] + pkg_spill["spill"] + svc_spillFiles["ctx.spillFiles
Spill storage seam"] + pkg_spill_local["spill-local"] + pkg_spill_policy["spill-policy"] pkg_agent --> svc_agents pkg_agent_loop --> svc_agentLoop pkg_bash --> svc_bash @@ -76,6 +80,8 @@ flowchart LR pkg_session_persistence --> svc_sessionPersistence pkg_session_persistence_jsonl --> svc_sessionPersistence pkg_session_persistence_sqlite --> svc_sessionPersistence + pkg_spill --> svc_spillFiles + pkg_spill_local --> svc_spillFiles pkg_subagent --> svc_subagents pkg_subagent_acp --> svc_subagents pkg_subagent_fork --> svc_subagents @@ -108,6 +114,7 @@ flowchart LR svc_sessions --> pkg_invariants svc_sessions --> pkg_session_persistence svc_sessions --> pkg_subagent_inprocess + svc_spillFiles --> pkg_spill_policy svc_subagents --> pkg_tool_subagent svc_systemPrompt --> pkg_agent_loop svc_systemPrompt --> pkg_tool_fs @@ -138,5 +145,6 @@ flowchart LR | `ctx.compact` | `seam` | [`compact`](../packages/compact/compact) | [`compact-basic`](../packages/compact/compact-basic) | [`compact-basic`](../packages/compact/compact-basic) | - | The basic backend currently consumes the pre-step event directly; a model-facing compact tool remains deferred. | | `ctx.subagents` | `seam` | [`subagent`](../packages/subagent/subagent) | [`subagent-spawn`](../packages/subagent/subagent-spawn), [`subagent-fork`](../packages/subagent/subagent-fork), [`subagent-acp`](../packages/subagent/subagent-acp), [`subagent-mock`](../packages/support/subagent-mock) | [`tool-subagent`](../packages/subagent/tool-subagent) | - | Providers implement transports; tool-subagent exposes one configured provider as a model-facing tool name. | | `ctx.web` | `seam` | [`web`](../packages/web/web) | [`web-search-exa`](../packages/web/web-search-exa), [`web-search-perplexity`](../packages/web/web-search-perplexity), [`web-search-deepseek`](../packages/web/web-search-deepseek), [`web-fetch-local`](../packages/web/web-fetch-local) | [`tool-web`](../packages/web/tool-web) | - | Search and fetch providers register into one ctx.web seam; tool-web owns the stable model-facing names. | +| `ctx.spillFiles` | `seam` | [`spill`](../packages/spill/spill) | [`spill-local`](../packages/spill/spill-local) | [`spill-policy`](../packages/spill/spill-policy) | - | The backend saves oversized tool text to a session-scoped path; spill-policy is the tools/post-execute consumer that decides when to spill. | Maintenance mode: hybrid: services are discovered from Cordis declarations; interface/implementation/consumer roles are classified in `scripts/gen-doc-graphs.ts` with a completeness guard. diff --git a/docs/cordis-catalog/services.md b/docs/cordis-catalog/services.md index c91f1578b2..b552df3417 100644 --- a/docs/cordis-catalog/services.md +++ b/docs/cordis-catalog/services.md @@ -165,6 +165,22 @@ list(): Session[] Source: [`packages/core/session/src/index.ts:327`](../../packages/core/session/src/index.ts) +## `ctx.spillFiles` — `SpillFiles` (abstract seam) + +Abstract spill storage service. Subclass, implement saveText, and load the subclass as a plugin — it registers as `ctx.spillFiles` (one implementation per context; loading a second throws, cordis' standard duplicate-service behavior). + +Semantics every implementation must honor: + +- saveText persists the FULL `content` verbatim and returns a path the local `read` tool can open, plus the exact byte length written. +- Storage is scoped by the request's SaveTextSpill.owner session; the backend chooses a private (not world-readable) location and a collision-free name derived from — never equal to — the caller's `suggestedName`. +- `saveText` REJECTS on a real storage failure (permissions, ENOSPC, backend unavailable); the caller decides how to degrade (the spill policy treats a rejection as best-effort and keeps the inline result). + +```ts cordis-catalog +abstract saveText(input: SaveTextSpill): Promise +``` + +Source: [`packages/spill/spill/src/index.ts:46`](../../packages/spill/spill/src/index.ts) + ## `ctx.subagents` — `SubagentService` The `subagents` service: a registry of named SubagentProviders and a capability-checked start surface. diff --git a/docs/event-producer-consumer.md b/docs/event-producer-consumer.md index 18c2b9faf0..60c042b26f 100644 --- a/docs/event-producer-consumer.md +++ b/docs/event-producer-consumer.md @@ -32,7 +32,7 @@ This matrix shows which packages dispatch each harness-owned event and which pac | `system-prompt/assemble` | `waterfall` | [`packages/core/system-prompt/src/index.ts:38`](../packages/core/system-prompt/src/index.ts) | [`system-prompt`](../packages/core/system-prompt) (`waterfall`) | - | | `system-prompt/change` | `emit` | [`packages/core/system-prompt/src/index.ts:44`](../packages/core/system-prompt/src/index.ts) | [`system-prompt`](../packages/core/system-prompt) (`emit`) | - | | `tools/change` | `emit` | [`packages/core/tools/src/index.ts:87`](../packages/core/tools/src/index.ts) | [`tools`](../packages/core/tools) (`emit`) | - | -| `tools/post-execute` | `waterfall` | [`packages/core/tools/src/index.ts:82`](../packages/core/tools/src/index.ts) | [`tools`](../packages/core/tools) (`waterfall`) | [`hooks-claude`](../packages/hooks/hooks-claude), [`hooks-codex`](../packages/hooks/hooks-codex) | +| `tools/post-execute` | `waterfall` | [`packages/core/tools/src/index.ts:82`](../packages/core/tools/src/index.ts) | [`tools`](../packages/core/tools) (`waterfall`) | [`hooks-claude`](../packages/hooks/hooks-claude), [`hooks-codex`](../packages/hooks/hooks-codex), [`spill-policy`](../packages/spill/spill-policy) | | `tools/pre-execute` | `waterfall` | [`packages/core/tools/src/index.ts:66`](../packages/core/tools/src/index.ts) | [`tools`](../packages/core/tools) (`waterfall`) | [`hooks-claude`](../packages/hooks/hooks-claude), [`hooks-codex`](../packages/hooks/hooks-codex) | Maintenance mode: hybrid generated: Cordis event declarations and most producer/listener edges are AST-scanned; dynamic dispatch sites are classified in `scripts/gen-doc-graphs.ts`. diff --git a/docs/module-graph.md b/docs/module-graph.md index d2537028f6..2bdd32329a 100644 --- a/docs/module-graph.md +++ b/docs/module-graph.md @@ -56,6 +56,11 @@ flowchart TD pkg_web_search_exa["web-search-exa"] pkg_web_search_perplexity["web-search-perplexity"] end + subgraph group_spill["packages/spill"] + pkg_spill["spill"] + pkg_spill_local["spill-local"] + pkg_spill_policy["spill-policy"] + end subgraph group_todo["packages/todo"] pkg_tool_todo["tool-todo"] end @@ -105,6 +110,9 @@ flowchart TD pkg_web_search_deepseek --> pkg_web pkg_web_search_exa --> pkg_web pkg_web_search_perplexity --> pkg_web + pkg_spill --> pkg_brand + pkg_spill --> pkg_llm + pkg_spill --> pkg_session pkg_hook_protocol --> pkg_bash pkg_hook_protocol --> pkg_session pkg_session_persistence --> pkg_session @@ -117,6 +125,7 @@ flowchart TD pkg_compact_basic --> pkg_compact pkg_compact_basic --> pkg_llm pkg_compact_basic --> pkg_session + pkg_spill_local --> pkg_spill pkg_session_persistence_jsonl --> pkg_session pkg_session_persistence_jsonl --> pkg_session_persistence pkg_session_persistence_sqlite --> pkg_session @@ -147,6 +156,11 @@ flowchart TD pkg_tool_web --> pkg_system_prompt pkg_tool_web --> pkg_tools pkg_tool_web --> pkg_web + pkg_spill_policy --> pkg_llm + pkg_spill_policy --> pkg_retention + pkg_spill_policy --> pkg_session + pkg_spill_policy --> pkg_spill + pkg_spill_policy --> pkg_tools pkg_tool_todo --> pkg_agent pkg_tool_todo --> pkg_session pkg_tool_todo --> pkg_tools @@ -229,11 +243,13 @@ flowchart TD | [`web-search-deepseek`](../packages/web/web-search-deepseek) | `web` | [`web`](../packages/web/web) | | [`web-search-exa`](../packages/web/web-search-exa) | `web` | [`web`](../packages/web/web) | | [`web-search-perplexity`](../packages/web/web-search-perplexity) | `web` | [`web`](../packages/web/web) | +| [`spill`](../packages/spill/spill) | `spill` | [`brand`](../packages/util/brand), [`llm`](../packages/llm/llm), [`session`](../packages/core/session) | | [`hook-protocol`](../packages/hooks/hook-protocol) | `hooks` | [`bash`](../packages/bash/bash), [`session`](../packages/core/session) | | [`session-persistence`](../packages/session-persistence/session-persistence) | `session-persistence` | [`session`](../packages/core/session) | | [`llm-replay`](../packages/support/llm-replay) | `support` | [`llm`](../packages/llm/llm), [`session`](../packages/core/session) | | [`tools`](../packages/core/tools) | `core` | [`agent`](../packages/core/agent), [`llm`](../packages/llm/llm), [`system-prompt`](../packages/core/system-prompt) | | [`compact-basic`](../packages/compact/compact-basic) | `compact` | [`agent`](../packages/core/agent), [`compact`](../packages/compact/compact), [`llm`](../packages/llm/llm), [`session`](../packages/core/session) | +| [`spill-local`](../packages/spill/spill-local) | `spill` | [`spill`](../packages/spill/spill) | | [`session-persistence-jsonl`](../packages/session-persistence/session-persistence-jsonl) | `session-persistence` | [`session`](../packages/core/session), [`session-persistence`](../packages/session-persistence/session-persistence) | | [`session-persistence-sqlite`](../packages/session-persistence/session-persistence-sqlite) | `session-persistence` | [`session`](../packages/core/session), [`session-persistence`](../packages/session-persistence/session-persistence) | | [`invariants`](../packages/support/invariants) | `support` | [`agent`](../packages/core/agent), [`llm`](../packages/llm/llm), [`session`](../packages/core/session) | @@ -242,6 +258,7 @@ flowchart TD | [`tool-fs`](../packages/fs/tool-fs) | `fs` | [`fs`](../packages/fs/fs), [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools) | | [`subagent`](../packages/subagent/subagent) | `subagent` | [`agent`](../packages/core/agent), [`llm`](../packages/llm/llm), [`tools`](../packages/core/tools) | | [`tool-web`](../packages/web/tool-web) | `web` | [`llm`](../packages/llm/llm), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`web`](../packages/web/web) | +| [`spill-policy`](../packages/spill/spill-policy) | `spill` | [`llm`](../packages/llm/llm), [`retention`](../packages/util/retention), [`session`](../packages/core/session), [`spill`](../packages/spill/spill), [`tools`](../packages/core/tools) | | [`tool-todo`](../packages/todo/tool-todo) | `todo` | [`agent`](../packages/core/agent), [`session`](../packages/core/session), [`tools`](../packages/core/tools) | | [`hooks-codex`](../packages/hooks/hooks-codex) | `hooks` | [`agent`](../packages/core/agent), [`hook-protocol`](../packages/hooks/hook-protocol), [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`tools`](../packages/core/tools) | | [`acp`](../packages/ui/acp) | `ui` | [`agent`](../packages/core/agent), [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-persistence`](../packages/session-persistence/session-persistence), [`tools`](../packages/core/tools) | diff --git a/docs/rfc/INDEX.md b/docs/rfc/INDEX.md index 1ce999d071..025ee16945 100644 --- a/docs/rfc/INDEX.md +++ b/docs/rfc/INDEX.md @@ -120,6 +120,7 @@ Generated by `pnpm run gen-rfc-index` from the RFC tree — never edit by hand; | [Subagent provider-lifecycle events — `subagent/provider-added` / `subagent/provider-removed`](implemented/architecture/2026-07-05-subagent-provider-lifecycle-events.md) | 2026-07-05 | | [A shared timeout/deadline primitive, with hard-kill left to each capability](implemented/architecture/2026-07-06-timeout-deadline-library.md) | 2026-07-06 | | [Tool result retention library](implemented/architecture/2026-07-06-tool-result-retention-library.md) | 2026-07-06 | +| [Tool output spill policy](implemented/architecture/2026-07-08-tool-output-spill-files.md) | 2026-07-08 | ### Process diff --git a/docs/rfc/implemented/architecture/2026-07-08-tool-output-spill-files.md b/docs/rfc/implemented/architecture/2026-07-08-tool-output-spill-files.md new file mode 100644 index 0000000000..a7d5aa4627 --- /dev/null +++ b/docs/rfc/implemented/architecture/2026-07-08-tool-output-spill-files.md @@ -0,0 +1,191 @@ +# RFC: Tool output spill policy + +Status: implemented + +## Problem + +Tool outputs need bounded model-facing previews, but some oversized results are still useful later. A fetched page body or a verbose tool response should not consume the next model request in full, but the model should be able to inspect the complete formatted result later with existing file-reading tools. + +Before this change the behavior was uneven. `dsh-bash-local` already writes complete stdout/stderr streams to private temp spill files when its in-memory tail overflows, but ordinary text tool results were returned inline unless the tool hand-rolled its own cap. The [tool result retention library](./2026-07-06-tool-result-retention-library.md) owns preview mechanics, but it does not own storage or an execution-pipeline policy that applies those mechanics to final tool results. + +The shape matches the timeout policy design: a tool author normally returns the text result, and a policy plugin enforces the deployment's default context budget. Tool-specific early spill remains possible later for outputs that do not survive to the final `ToolExecutionResult`; the first cut proves the default final-result path. + +## Decision + +A thin spill storage seam plus a default spill policy plugin, in a new `packages/spill/` group: + +| Package | Role | +|---|---| +| `@deepseek-ai/dsh-spill` | Interface: `ctx.spillFiles`, vocabulary types, no filesystem implementation. | +| `@deepseek-ai/dsh-spill-local` | Local backend: private, session-scoped file storage on the host filesystem. | +| `@deepseek-ai/dsh-spill-policy` | Tool-result policy plugin: wraps final text results after dispatch and replaces oversized results with a retained preview plus a spill-file path. | + +There is no dedicated model-facing consumer package. The consumer is the existing `ctx.tools` execution pipeline: `dsh-spill-policy` consumes final tool results through the `tools/post-execute` waterfall, and the model uses the existing `read` tool to inspect the returned path. + +### Spill seam + +The storage seam is minimal: save text and return a local path. + +```ts ignore-check +interface SpillFiles { + saveText(input: SaveTextSpill): Promise +} + +interface SpillSource { + toolName: string + callId: CallId + label: string +} + +interface SaveTextSpill { + owner: { sessionId: SessionId } + source: SpillSource + suggestedName: string + content: string +} + +type SpillPath = Branded<'SpillPath'> + +interface SpillRef { + path: SpillPath + bytes: number +} +``` + +`SpillPath` is a [branded](../../../../packages/util/brand) local filesystem path returned by the backend and intended for `read`. The brand records that the path came from the spill seam (a runtime artifact); it is rendered to the model as an ordinary path string in v1. `SpillOwner` scopes storage to a `SessionId` — spill is inherently session-scoped, so the seam imports `dsh-session`'s `SessionId` directly rather than minting a decoupled token like the bash executor's `OwnerToken`. + +`dsh-spill-local` owns only storage details: session-scoped directory selection, safe names, path-traversal protection, the write, and returning `{ path, bytes }`. It does not own retention policy, model-facing wording, tool-result replacement, search, or file inspection. Files land at `/session-/-`, where `root` is a configured path or a lazily-created private (0700) per-process temp dir, the session subdir is a short `sha256(sessionId)` prefix, and the leaf is a random hex prefix plus the caller's `suggestedName` sanitized to one path segment (mirrors the JSONL backend's `encodeSegment`). The write is `open(path, 'wx', 0o600)` — exclusive and owner-only, so a planted symlink cannot redirect it. + +The v1 local backend returns a real local `path` readable by the existing `read` tool. A future remote or virtual backend may replace this with a `spill://...` URI plus a read-only filesystem bridge; v1 keeps the interface path-shaped until that backend exists. + +### Spill policy + +`dsh-spill-policy` is a `tools/post-execute` result transformer with one configuration knob: + +```ts ignore-check +interface Config { + /** Omitted means no automatic spill policy. Present means apply to oversized plain text tool results. */ + maxInlineBytes?: number +} +``` + +When `maxInlineBytes` is omitted the plugin registers nothing (a true no-op). When set, it applies a default policy to final plain-text tool results: + +1. Let the tool run normally, delegating via `next()` so a downstream listener settles the result first. +2. Flatten the accepted final `ContentBlock[]` only when it is entirely plain text; a result with any non-text block is left untouched. +3. If its UTF-8 byte size is at or below `maxInlineBytes`, leave it unchanged. +4. If it is larger, call `ctx.spillFiles.saveText()` with the full final text. +5. Replace the model-facing result with a retained head/tail preview plus the spill path. + +The preview is an implementation default owned by the policy: a head/tail split of `maxInlineBytes` via the retention library's `TextRetainer`. Future config can expose preview sizing only after a second deployment needs it. + +The replacement text is intentionally generic because the policy only knows the final formatted tool result, not the tool's internal resource: + +```text + + +(Omitted N bytes. Full formatted result saved to: /.../session-.../....txt. Use read with offset/limit to inspect it.) +``` + +If `ctx.spillFiles.saveText()` fails (permissions, ENOSPC, backend unavailable), or the call has no session owner, or no backend is loaded, the plugin logs the reason and returns the original result unchanged. Spill failure never turns a successful tool call into an `isError` result or hides the inline result. + +The policy skips `read` to avoid a circular `read -> spill file -> read again` loop. Additional opt-out configuration is deferred until a real second tool needs it. + +## Showcase: web_fetch + +`web_fetch` is the first showcase because it returns a naturally large text result and needs no tool-specific spill code. The tool is ordinary: + +```ts ignore-check +ctx.tools.register(defineTool({ + name: 'web_fetch', + async execute(args, exec) { + const result = await ctx.web.fetch({ url: args.url }, exec.signal ? { signal: exec.signal } : undefined) + return [{ type: 'text', text: formatFetchOutput(result) }] + }, +})) +``` + +With `dsh-spill-policy` configured, a large formatted fetch result is automatically retained and spilled. A deployment demonstrates the behavior by setting the provider resource cap higher than the policy cap: + +```yaml +- id: web-fetch-local + name: '@deepseek-ai/dsh-web-fetch-local' + config: + maxBodyChars: 500000 + +- id: spill-local + name: '@deepseek-ai/dsh-spill-local' + +- id: spill-policy + name: '@deepseek-ai/dsh-spill-policy' + config: + maxInlineBytes: 50000 +``` + +This separation is important. `web-fetch-local` still owns resource caps (`maxResponseBytes`, `maxBodyChars`) to protect network, memory, and decoding work. `spill-policy` owns only the model-facing context cap after the result already exists. If the provider already returned `truncated: true`, the spill file contains the full formatted result the tool returned, not the full original webpage; the policy does not claim otherwise. + +## Relationship to retention and early spill + +Retention is separate from spill storage: + +- `@deepseek-ai/dsh-retention` owns preview mechanics (`TextRetainer`, `ItemRetainer`, omitted metadata, early-stop decisions). +- `@deepseek-ai/dsh-spill` owns saving final text to a session-scoped path. +- `@deepseek-ai/dsh-spill-policy` applies the default final-result policy in the tool pipeline, composing the two. + +The final-result policy cannot replace tool-owned early spill. Some useful content is not present in final `ToolExecutionResult.content`: + +- `bash` final output is already a tail plus a temp spill path; the complete stdout/stderr streams live in executor files. +- `subagent` final output is the child final answer, not the child rollout. +- Future `grep`/`glob` may early-stop and never collect full results. + +Those cases can consume `ctx.spillFiles` directly in later work. They are not part of the first showcase. + +## Non-goals + +- No new model-facing `artifact_read` or `artifact_search` tool in v1. +- No per-tool retention configuration in v1. +- No model-facing timeout/truncation arguments. +- No migration of `read` output into spill files. +- No replacement for provider/resource caps such as `web-fetch-local.maxBodyChars`. +- No bash temp-file normalization or subagent rollout capture in the first cut. + +## Deferred + +- `saveFile()` / `linkOrCopy` for existing executor spill files, needed for bash normalization. +- Tool-owned spill for subagent rollouts (`await run.result`, read in-process child session before `run.dispose()`, save JSONL). +- Per-tool opt-out or per-tool policy declarations if the built-in `read` skip is insufficient. +- A virtual `spill://` URI and read-only filesystem bridge. +- Remote storage backends for ACP or remote environments where a local path is not meaningful. +- Cleanup and retention policy for old spill files, likely tied to session cleanup. + +## Testing + +- `dsh-spill` unit tests pin the seam contract: registration as `ctx.spillFiles`, one-implementation-per-context, and disposal release. +- `dsh-spill-local` unit tests cover `saveText`, `encodeSegment` sanitization (separators/tilde/whole-segment dots/empty), the session-hash directory, owner-only permissions, distinct paths per save, the configured/private root, and a storage-failure rejection. +- `dsh-spill-policy` unit tests drive real tools through `ctx.tools.execute`: disabled-mode no-op, oversized-text replacement, small/non-text passthrough, `read` skip, best-effort fallback (save failure / no backend / no owner), and downstream-composition (bounding a replaced result, preserving `additionalContext`). +- `dsh-tool-web` integration drives `web_fetch` through `ctx.tools.execute` with the real `spill-local` backend + policy, proving the model-facing text changes only by the deliberate spill notice while the spill file holds the full formatted result. +- The `coding-agent` example loads `spill-local` + `spill-policy`, so its keyless Loader smoke exercises the real load path (the namespace-plugin export shape + `inject`). + +## Consequences + +The default policy only sees final formatted text. It cannot preserve provider-internal content that was already capped or runtime artifacts that were never part of the result. This is acceptable for the first cut because the showcase is final-result spill, not early spill; tool-owned early spill remains deferred work. + +Returning real paths from the local backend keeps v1 simple and matches proven agent-tool behavior, but it exposes implementation paths to the model and may not work for remote backends. The interface should be revisited when a virtual or remote spill backend exists. + +The v1 value proposition depends on the existing `read` tool being able to inspect the returned local path, even when the spill directory is outside the session cwd. That holds today because the filesystem policy records observations and write guards but does not confine reads to the workspace. A future workspace-confinement policy must either allow spill paths explicitly or provide a read-only spill bridge, or the spill notice would point at an unreadable path. + +**Snapshot gap.** No ACP snapshot scenario covers the transcript-visible `web_fetch` spill notice yet. The ACP snapshot harness replays keyless and cannot hit the live web, and a `web_fetch` spill requires a real over-cap HTTP body; a deterministic scenario would need a seeded loopback fetch target the replay tree does not currently wire (the examples do not load `tool-web` at all). The behavior is covered instead by the `dsh-tool-web` integration test against a loopback server. Closing the gap is follow-up work: wire `tool-web` + a seeded fetch target into the ACP example, then record a `web-fetch-spill` scenario. + +The policy can become too large if it starts owning tool-specific semantics. It stays narrow: plain-text final results only. Tool-owned early spill remains future work. + +## Alternatives considered + +**Require each tool to opt in with a retention declaration.** Rejected for v1: the goal is a default behavior similar to Claude Code's generic tool-result persistence. A single `maxInlineBytes` deployment knob is enough to prove the shape. + +**Make `tool-results` a broad tool-result platform.** Rejected: a broad package name invites retention policy, result replacement, preview wording, search, and early spill into one seam. The shared storage part is smaller: save text and return a path. + +**Use `ctx.fs.writeText` or the model-facing `write` tool.** Rejected: workspace filesystem writes carry project-file semantics, write/edit policy, observation state, and user-facing side effects. Spill files are runtime artifacts, not model-authored workspace edits. The existing `read` tool may inspect them later, but creation belongs to the runtime spill seam. + +**Let `web-fetch-local` fetch without caps and rely on spill-policy.** Rejected: spill-policy runs after the final tool result exists and cannot protect network, memory, or decoding resources. Provider resource caps stay mandatory. + +**Merge retention into spill.** Rejected: retention and spill have different responsibilities. `TextRetainer`/`ItemRetainer` decide what preview is kept and whether upstream may stop; spill storage only saves the final text the policy asks it to save. diff --git a/examples/coding-agent/composition.md b/examples/coding-agent/composition.md index 926cb56109..979737d05f 100644 --- a/examples/coding-agent/composition.md +++ b/examples/coding-agent/composition.md @@ -43,6 +43,10 @@ flowchart LR cfg --> plugin_coding_fs_policy plugin_coding_tool_fs["tool-fs
@deepseek-ai/dsh-tool-fs"] cfg --> plugin_coding_tool_fs + plugin_coding_spill_local["spill-local
@deepseek-ai/dsh-spill-local"] + cfg --> plugin_coding_spill_local + plugin_coding_spill_policy["spill-policy
@deepseek-ai/dsh-spill-policy"] + cfg --> plugin_coding_spill_policy ``` | Plugin id | Package / module | @@ -61,6 +65,8 @@ flowchart LR | `fs-local` | `@deepseek-ai/dsh-fs-local` | | `fs-policy` | `@deepseek-ai/dsh-fs-policy` | | `tool-fs` | `@deepseek-ai/dsh-tool-fs` | +| `spill-local` | `@deepseek-ai/dsh-spill-local` | +| `spill-policy` | `@deepseek-ai/dsh-spill-policy` | Source config: [`examples/coding-agent/cordis.yml`](cordis.yml). diff --git a/examples/coding-agent/cordis.yml b/examples/coding-agent/cordis.yml index 0439262e33..c740aa2907 100644 --- a/examples/coding-agent/cordis.yml +++ b/examples/coding-agent/cordis.yml @@ -120,3 +120,16 @@ - id: tool-fs name: '@deepseek-ai/dsh-tool-fs' + +# Tool-output spill stack: a local backend that saves oversized tool text under +# a private session-scoped dir, and the tools/post-execute policy that replaces +# an over-budget plain-text result with a preview + the spill path (the model +# reads the full result later). A leaf pair after the app (needs ctx.tools). The +# policy is a no-op until a tool returns more than maxInlineBytes of plain text. +- id: spill-local + name: '@deepseek-ai/dsh-spill-local' + +- id: spill-policy + name: '@deepseek-ai/dsh-spill-policy' + config: + maxInlineBytes: 50000 diff --git a/packages/README.md b/packages/README.md index 1f7cb9121b..79ddd5fcac 100644 --- a/packages/README.md +++ b/packages/README.md @@ -15,6 +15,7 @@ Packages are grouped by modular role at `packages///`. The group dir | [`compact/`](compact/README.md) | Compaction capability family: the abstract seam + a basic backend (tool deferred) | Product — stable surface | | [`subagent/`](subagent/README.md) | Subagent capability family: the provider-registry seam and the model-facing delegation tool | Product — stable surface | | [`web/`](web/README.md) | Web capability family: the abstract seam, search/fetch provider impls, and the model-facing web tools | Product — stable surface | +| [`spill/`](spill/README.md) | Spill capability family: the storage seam, a local impl, and the tool-result spill policy | Product — stable surface | | [`todo/`](todo/README.md) | Todo/planning family: the model-facing `todo_write` tool (whole-list task tracking on the session log) | Product — stable surface | | [`hooks/`](hooks/README.md) | Hook bridges + the shared Claude Code / Codex wire-protocol library | Product — stable surface | | [`session-persistence/`](session-persistence/README.md) | Persistence capability family: the seam + JSONL/SQLite backends | Product — stable surface | diff --git a/packages/spill/README.md b/packages/spill/README.md new file mode 100644 index 0000000000..35122275a3 --- /dev/null +++ b/packages/spill/README.md @@ -0,0 +1,13 @@ +# spill/ - spill storage capability family + +The tool-output spill capability seam: an abstract storage interface, a local filesystem implementation, and the tool-result policy that uses it. All **product** packages. + +| Package | Role | ctx key | +|---|---|---| +| `spill/` | Abstract spill storage seam (`saveText` — persist oversized tool text to a session-scoped path) | `ctx.spillFiles` | +| `spill-local/` | Local-filesystem backend: private, session-scoped files with traversal-safe names | (registers on `ctx.spillFiles`) | +| `spill-policy/` | `tools/post-execute` policy: replaces oversized plain-text results with a preview + spill path | (no service surface) | + +The interface lives at `spill/spill/`. The split mirrors bash/fs: the seam owns storage only, `spill-local` owns the filesystem mechanics, and `spill-policy` owns WHEN to spill and the model-facing notice. Preview mechanics stay in [`util/retention`](../util/README.md) — the policy composes the two without either owning the other's job. + +See the [tool output spill RFC](../../docs/rfc/implemented/architecture/2026-07-08-tool-output-spill-files.md) for the design rationale, including why final-result spill is separate from tool-owned early spill (bash streams, subagent rollouts) and why creation belongs to the runtime spill seam rather than the model-facing `write` tool. diff --git a/packages/spill/spill-local/README.md b/packages/spill/spill-local/README.md new file mode 100644 index 0000000000..1205b31eef --- /dev/null +++ b/packages/spill/spill-local/README.md @@ -0,0 +1,19 @@ +# @deepseek-ai/dsh-spill-local + +The **local-filesystem** implementation of the [`@deepseek-ai/dsh-spill`](../spill) storage seam. Registers as `ctx.spillFiles` and persists a tool's oversized text to a private, session-scoped file the model's `read` tool can open. + +## Storage layout + +Files land at `/session-/​-`: + +- **`root`** — the config `root` (resolved to absolute), or a lazily-created private (0700) per-process directory under the OS temp dir when omitted. A predictable, world-readable root would let other local users read spilled tool output or plant symlinks. +- **`session-`** — a short `sha256(sessionId)` prefix, so a session's spill files group together and a future cleanup can drop them per session. +- **`-`** — an unpredictable hex prefix (defeats symlink planting in a shared root) plus the caller's `suggestedName` sanitized to one safe path segment (traversal-proof; mirrors the JSONL persistence backend's `encodeSegment`). The write is exclusive + owner-only (`open(path, 'wx', 0o600)`): it fails on any pre-existing path, symlink or not, so a planted target cannot redirect it. + +## Config + +| Key | Default | Meaning | +|---|---|---| +| `root` | private 0700 temp dir | Root directory for spill files. Set to keep them under a known location. | + +`saveText` rejects on a real storage failure (permissions, ENOSPC); the spill policy treats a rejection as best-effort and keeps the inline result. See the seam README for the vocabulary and the [tool output spill RFC](../../../docs/rfc/implemented/architecture/2026-07-08-tool-output-spill-files.md) for the design. diff --git a/packages/spill/spill-local/package.json b/packages/spill/spill-local/package.json new file mode 100644 index 0000000000..a75c4bfc0b --- /dev/null +++ b/packages/spill/spill-local/package.json @@ -0,0 +1,38 @@ +{ + "name": "@deepseek-ai/dsh-spill-local", + "description": "Local-filesystem implementation of the DeepSeek Harness spill storage seam (private session-scoped files)", + "version": "0.0.1", + "private": true, + "type": "module", + "main": "lib/index.js", + "types": "lib/types/index.d.ts", + "exports": { + ".": { + "types": "./lib/types/index.d.ts", + "default": "./lib/index.js" + }, + "./src/*": "./src/*", + "./package.json": "./package.json" + }, + "files": [ + "lib/index.js", + "lib/types/**/*.d.ts", + "lib/types/**/*.d.ts.map", + "src" + ], + "license": "BSD-3-Clause", + "peerDependencies": { + "@deepseek-ai/dsh-spill": "^0.0.1", + "cordis": "^4.0.0-rc.6" + }, + "dependencies": { + "schemastery": "^3.18.0" + }, + "devDependencies": { + "@deepseek-ai/dsh-brand": "workspace:^", + "@deepseek-ai/dsh-llm": "workspace:^", + "@deepseek-ai/dsh-session": "workspace:^", + "@deepseek-ai/dsh-spill": "workspace:^", + "cordis": "^4.0.0-rc.6" + } +} diff --git a/packages/spill/spill-local/src/index.ts b/packages/spill/spill-local/src/index.ts new file mode 100644 index 0000000000..8aaf35e0a8 --- /dev/null +++ b/packages/spill/spill-local/src/index.ts @@ -0,0 +1,61 @@ +/** + * `LocalSpillFiles`: the host-filesystem implementation of the + * `@deepseek-ai/dsh-spill` storage seam. Persists a tool's oversized text to a + * private, session-scoped file (see `./store.ts` for the traversal-safe naming + * and exclusive owner-only write) and returns a path the local `read` tool can + * open. + * + * @module @deepseek-ai/dsh-spill-local + */ + +import { Context } from 'cordis' +import { resolve } from 'node:path' +import z from 'schemastery' +import { SpillFiles, SpillPath } from '@deepseek-ai/dsh-spill' +import type { SaveTextSpill, SpillRef } from '@deepseek-ai/dsh-spill' +import { privateRoot, saveTextFile } from './store.ts' + +export { encodeSegment, privateRoot, saveTextFile, sessionDir } from './store.ts' +export type { SavedText, SaveTextOptions } from './store.ts' + +/** Plugin config (all optional — `static Config` supplies the defaults). */ +export interface Config { + /** + * Root directory for spill files. Omitted uses a lazily-created private + * (0700) per-process directory under the OS temp dir — the safe default for + * a local deployment. Set it to keep spill files under a known location. + */ + root?: string +} + +/** + * Local-filesystem spill backend. Files land under `/session-/…` + * with unpredictable names, an exclusive owner-only (0600) write, and a private + * (0700) root — a spilled tool result must not be readable by other local users + * or redirectable via a planted symlink. + */ +export class LocalSpillFiles extends SpillFiles { + static Config: z = z.object({ + root: z.string(), + }) + + /** Resolved absolute spill root (config `root`, else the private default), fixed at construction. */ + readonly root: string + + constructor(ctx: Context, config: Config) { + super(ctx) + this.root = config.root !== undefined ? resolve(config.root) : privateRoot() + } + + async saveText(input: SaveTextSpill): Promise { + const saved = await saveTextFile({ + root: this.root, + sessionId: input.owner.sessionId, + suggestedName: input.suggestedName, + content: input.content, + }) + return { path: SpillPath(saved.path), bytes: saved.bytes } + } +} + +export default LocalSpillFiles diff --git a/packages/spill/spill-local/src/store.ts b/packages/spill/spill-local/src/store.ts new file mode 100644 index 0000000000..adf538e740 --- /dev/null +++ b/packages/spill/spill-local/src/store.ts @@ -0,0 +1,102 @@ +/** + * Cordis-free storage mechanics for the local spill backend: private + * session-scoped directory selection, safe-name derivation, path-traversal + * protection, and the exclusive owner-only write. Kept out of the service class + * (like `dsh-bash-local`'s `run.ts`) so the filesystem behavior is unit-testable + * without a `ctx` and without the OS temp dir. + * + * @module @deepseek-ai/dsh-spill-local/store + */ + +import { createHash, randomBytes } from 'node:crypto' +import { mkdtempSync } from 'node:fs' +import { mkdir, open } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' + +let defaultRoot: string | undefined + +/** + * The default spill root: a private (0700) per-process directory under the OS + * tmpdir, created lazily. Predictable world-readable paths would let other + * local users read spilled tool output or pre-create symlinks; `mkdtemp` gives + * an unpredictable suffix and 0700 semantics. + */ +export function privateRoot(): string { + defaultRoot ??= mkdtempSync(join(tmpdir(), 'dsh-spill-')) + return defaultRoot +} + +/** + * Encode an arbitrary string as one safe path segment, injectively over ALL JS + * (UTF-16) strings. A session id / suggested name is untrusted input, so this + * neutralizes `../`, absolute paths, NUL, and separators before any filesystem + * use. Each code unit is kept literal (`[A-Za-z0-9._-]`, minus `~`) or escaped + * as `~XXXX`; `~` is itself escaped, so the mapping is reversible and distinct + * inputs never collide. The whole-segment tokens `.`/`..` are escaped so they + * can never traverse. An empty string encodes to `~` (never an empty segment). + * (Mirrors the JSONL persistence backend's `encodeSegment`.) + */ +export function encodeSegment(raw: string): string { + if (raw.length === 0) return '~' + if (raw === '.') return '~002E' + if (raw === '..') return '~002E~002E' + let out = '' + for (let i = 0; i < raw.length; i++) { + const code = raw.charCodeAt(i) + const ch = String.fromCharCode(code) + if (ch !== '~' && /^[A-Za-z0-9._-]$/.test(ch)) { + out += ch + } else { + out += '~' + code.toString(16).toUpperCase().padStart(4, '0') + } + } + return out +} + +/** The session-scoped directory: `/session-`, a short stable hash. */ +export function sessionDir(root: string, sessionId: string): string { + const hash = createHash('sha256').update(sessionId).digest('hex').slice(0, 12) + return join(root, `session-${hash}`) +} + +/** Options for {@link saveTextFile} — the resolved root and the request fields the store needs. */ +export interface SaveTextOptions { + /** The spill root directory (configured or the lazy private default). */ + root: string + /** The owning session id (scopes the directory). */ + sessionId: string + /** Caller-suggested base name; sanitized to one safe segment before use. */ + suggestedName: string + /** The full text to persist. */ + content: string +} + +/** A written spill file. */ +export interface SavedText { + path: string + bytes: number +} + +/** + * Write `content` to a fresh file under the session-scoped directory and return + * its path + byte length. The filename is a random hex prefix plus the + * sanitized `suggestedName`, so it is unpredictable (defeats symlink planting in + * a shared root) AND stays readable. The open is exclusive + owner-only + * (`'wx', 0o600`): it fails on any existing path — symlink or not — so a + * pre-planted target cannot redirect the write. + */ +export async function saveTextFile(options: SaveTextOptions): Promise { + const dir = sessionDir(options.root, options.sessionId) + await mkdir(dir, { recursive: true, mode: 0o700 }) + const safeName = encodeSegment(options.suggestedName) + const path = join(dir, `${randomBytes(6).toString('hex')}-${safeName}`) + const bytes = Buffer.byteLength(options.content, 'utf8') + const handle = await open(path, 'wx', 0o600) + try { + await handle.writeFile(options.content) + } finally { + await handle.close() + } + return { path, bytes } +} diff --git a/packages/spill/spill-local/tests/spill-local.spec.ts b/packages/spill/spill-local/tests/spill-local.spec.ts new file mode 100644 index 0000000000..7357c1ede5 --- /dev/null +++ b/packages/spill/spill-local/tests/spill-local.spec.ts @@ -0,0 +1,138 @@ +/** + * Tests for the LOCAL spill backend: `saveText` writes a session-scoped file and + * returns its path + byte length, filename sanitization neutralizes traversal, + * the configured `root` is honored (and the private default when omitted), and a + * storage failure rejects. The Cordis-free `store.ts` helpers are exercised + * directly for the naming/encoding edge cases. + */ + +import { describe, expect, it, beforeEach, afterEach } from 'vitest' +import { Context } from 'cordis' +import { mkdtempSync, readFileSync, rmSync, statSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, isAbsolute, join } from 'node:path' +import { CallId } from '@deepseek-ai/dsh-llm' +import { SessionId } from '@deepseek-ai/dsh-session' +import type { SaveTextSpill } from '@deepseek-ai/dsh-spill' +import LocalSpillFiles, { encodeSegment, privateRoot, saveTextFile, sessionDir } from '@deepseek-ai/dsh-spill-local' + +let root: string + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'dsh-spill-test-')) +}) +afterEach(() => { + rmSync(root, { recursive: true, force: true }) +}) + +function request(overrides: Partial = {}): SaveTextSpill { + return { + owner: { sessionId: SessionId('sess-1') }, + source: { toolName: 'web_fetch', callId: CallId('call-1'), label: 'result' }, + suggestedName: 'web_fetch.txt', + content: 'the full body', + ...overrides, + } +} + +describe('encodeSegment', () => { + it('keeps the safe set literal', () => { + expect(encodeSegment('web_fetch.txt')).toBe('web_fetch.txt') + expect(encodeSegment('a-B_9.z')).toBe('a-B_9.z') + }) + + it('escapes separators and tilde (dots are literal except as whole-segment tokens)', () => { + // `.` is in the safe set, so `..` inside a longer string stays literal; the + // traversal defense is that separators escape, keeping the result ONE segment. + expect(encodeSegment('../etc/passwd')).toBe('..~002Fetc~002Fpasswd') + expect(encodeSegment('a/b')).toBe('a~002Fb') + expect(encodeSegment('~')).toBe('~007E') + }) + + it('escapes the whole-segment dot tokens', () => { + expect(encodeSegment('.')).toBe('~002E') + expect(encodeSegment('..')).toBe('~002E~002E') + }) + + it('encodes the empty string to a non-empty segment', () => { + expect(encodeSegment('')).toBe('~') + }) +}) + +describe('sessionDir', () => { + it('is a stable per-session hash under the root', () => { + const dir = sessionDir('/spill', 'sess-1') + expect(dir).toBe(sessionDir('/spill', 'sess-1')) + expect(dir).toMatch(/\/spill\/session-[0-9a-f]{12}$/) + expect(sessionDir('/spill', 'sess-2')).not.toBe(dir) + }) +}) + +describe('saveTextFile', () => { + it('writes the content under the session dir and reports bytes', async () => { + const saved = await saveTextFile({ root, sessionId: 'sess-1', suggestedName: 'r.txt', content: 'héllo' }) + expect(readFileSync(saved.path, 'utf8')).toBe('héllo') + expect(saved.bytes).toBe(Buffer.byteLength('héllo', 'utf8')) + expect(dirname(saved.path)).toBe(sessionDir(root, 'sess-1')) + expect(saved.path).toMatch(/\/[0-9a-f]{12}-r\.txt$/) + }) + + it('sanitizes a traversal-shaped suggested name into one segment', async () => { + const saved = await saveTextFile({ root, sessionId: 'sess-1', suggestedName: '../../evil', content: 'x' }) + // The separators escaped, so the whole name is one leaf under the session dir. + expect(dirname(saved.path)).toBe(sessionDir(root, 'sess-1')) + expect(saved.path.includes('/..')).toBe(false) + }) + + it('creates the session dir with owner-only permissions', async () => { + const saved = await saveTextFile({ root, sessionId: 'sess-1', suggestedName: 'r.txt', content: 'x' }) + // 0o700 dir, 0o600 file (masked by umask, but the owner bits must hold). + expect(statSync(dirname(saved.path)).mode & 0o700).toBe(0o700) + expect(statSync(saved.path).mode & 0o600).toBe(0o600) + }) + + it('gives distinct paths to two saves of the same name', async () => { + const a = await saveTextFile({ root, sessionId: 'sess-1', suggestedName: 'r.txt', content: 'a' }) + const b = await saveTextFile({ root, sessionId: 'sess-1', suggestedName: 'r.txt', content: 'b' }) + expect(a.path).not.toBe(b.path) + }) +}) + +describe('privateRoot', () => { + it('is a stable absolute directory under the temp dir', () => { + const first = privateRoot() + expect(isAbsolute(first)).toBe(true) + expect(privateRoot()).toBe(first) + }) +}) + +describe('LocalSpillFiles service', () => { + it('registers as ctx.spillFiles and saves under the configured root', async () => { + const ctx = new Context() + await ctx.plugin(LocalSpillFiles, { root }) + const ref = await ctx.spillFiles.saveText(request()) + expect(dirname(ref.path)).toBe(sessionDir(root, 'sess-1')) + expect(readFileSync(ref.path, 'utf8')).toBe('the full body') + expect(ref.bytes).toBe(Buffer.byteLength('the full body', 'utf8')) + }) + + it('resolves a relative configured root to absolute', async () => { + const ctx = new Context() + await ctx.plugin(LocalSpillFiles, { root: '.' }) + expect(isAbsolute((ctx.spillFiles as LocalSpillFiles).root)).toBe(true) + }) + + it('falls back to the private root when none is configured', async () => { + const ctx = new Context() + await ctx.plugin(LocalSpillFiles, {}) + expect((ctx.spillFiles as LocalSpillFiles).root).toBe(privateRoot()) + }) + + it('rejects when the root is not writable (missing parent, exclusive open)', async () => { + const ctx = new Context() + // A file (not a dir) as the root makes mkdir under it fail — a real storage error. + const filePath = (await saveTextFile({ root, sessionId: 's', suggestedName: 'f', content: 'x' })).path + await ctx.plugin(LocalSpillFiles, { root: filePath }) + await expect(ctx.spillFiles.saveText(request())).rejects.toThrow() + }) +}) diff --git a/packages/spill/spill-local/tsconfig.json b/packages/spill/spill-local/tsconfig.json new file mode 100644 index 0000000000..8e818212f5 --- /dev/null +++ b/packages/spill/spill-local/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "lib/types" + }, + "include": ["src"], + "references": [ + { "path": "../../../vendor/cosmokit" }, + { "path": "../../../vendor/cordis" }, + { "path": "../../../vendor/schemastery" }, + { "path": "../spill" } + ] +} diff --git a/packages/spill/spill-policy/README.md b/packages/spill/spill-policy/README.md new file mode 100644 index 0000000000..42c3334bf7 --- /dev/null +++ b/packages/spill/spill-policy/README.md @@ -0,0 +1,31 @@ +# @deepseek-ai/dsh-spill-policy + +The **tool-result spill policy**: a `tools/post-execute` transformer that keeps oversized plain-text tool results out of the model's context. When a final result exceeds `maxInlineBytes`, it saves the FULL text to a session-scoped spill file via [`ctx.spillFiles`](../spill) and replaces the model-facing result with a bounded head/tail preview plus the spill path — the model reads the complete result later with the existing `read` tool. + +This plugin registers **no service** and owns no storage or preview mechanics: preview is [`@deepseek-ai/dsh-retention`](../../util/retention) (`TextRetainer`), storage is `ctx.spillFiles`. It only decides WHEN to spill and composes the notice. + +## Config + +| Key | Default | Meaning | +|---|---|---| +| `maxInlineBytes` | *(omitted)* | Model-facing context cap for a plain-text result, in UTF-8 bytes. **Omitted disables the policy entirely** (the plugin registers nothing). When set, a larger result is spilled and replaced with a preview derived from the same budget (head/tail split). | + +## Behavior + +1. Let the tool run (delegates via `next()`, so it bounds whatever a downstream hook accepted). +2. Skip `read` (avoids a `read → spill file → read again` loop) and any non-`accept` decision (a `block`'s corrective feedback passes through). +3. Flatten the accepted content only when it is **plain text** (all `text` blocks); a result with any non-text block is left untouched. +4. If its UTF-8 size is `≤ maxInlineBytes`, leave it unchanged. +5. Otherwise save the full text and replace the result with a preview + this notice: + + ```text + + + (Omitted N bytes. Full formatted result saved to: /…/session-…/…-web_fetch.txt. Use read with offset/limit to inspect it.) + ``` + +**Best-effort:** no session owner, no `ctx.spillFiles` backend, or a `saveText` rejection ⇒ the policy logs a warning and returns the original result. A spill failure never turns a successful call into an `isError` or hides the inline result. + +## Scope + +The policy sees only the FINAL formatted tool result — not a tool's internal resource. If a provider already truncated (e.g. `web-fetch-local.maxBodyChars`), the spill file holds the full formatted result the tool returned, not the full original source. Provider/resource caps stay mandatory and separate. Tool-owned early spill (bash streams, subagent rollouts) is future work — see the [tool output spill RFC](../../../docs/rfc/implemented/architecture/2026-07-08-tool-output-spill-files.md). diff --git a/packages/spill/spill-policy/package.json b/packages/spill/spill-policy/package.json new file mode 100644 index 0000000000..9c28ea5382 --- /dev/null +++ b/packages/spill/spill-policy/package.json @@ -0,0 +1,44 @@ +{ + "name": "@deepseek-ai/dsh-spill-policy", + "description": "Tool-result spill policy for the DeepSeek Harness — replaces oversized plain-text tool results with a retained preview plus a spill-file path (no service surface)", + "version": "0.0.1", + "private": true, + "type": "module", + "main": "lib/index.js", + "types": "lib/types/index.d.ts", + "exports": { + ".": { + "types": "./lib/types/index.d.ts", + "default": "./lib/index.js" + }, + "./src/*": "./src/*", + "./package.json": "./package.json" + }, + "files": [ + "lib/index.js", + "lib/types/**/*.d.ts", + "lib/types/**/*.d.ts.map", + "src" + ], + "license": "BSD-3-Clause", + "peerDependencies": { + "@deepseek-ai/dsh-llm": "^0.0.1", + "@deepseek-ai/dsh-retention": "^0.0.1", + "@deepseek-ai/dsh-session": "^0.0.1", + "@deepseek-ai/dsh-spill": "^0.0.1", + "@deepseek-ai/dsh-tools": "^0.0.1", + "cordis": "^4.0.0-rc.6" + }, + "dependencies": { + "schemastery": "^3.18.0" + }, + "devDependencies": { + "@deepseek-ai/dsh-agent": "workspace:^", + "@deepseek-ai/dsh-llm": "workspace:^", + "@deepseek-ai/dsh-retention": "workspace:^", + "@deepseek-ai/dsh-session": "workspace:^", + "@deepseek-ai/dsh-spill": "workspace:^", + "@deepseek-ai/dsh-tools": "workspace:^", + "cordis": "^4.0.0-rc.6" + } +} diff --git a/packages/spill/spill-policy/src/index.ts b/packages/spill/spill-policy/src/index.ts new file mode 100644 index 0000000000..f4672a4751 --- /dev/null +++ b/packages/spill/spill-policy/src/index.ts @@ -0,0 +1,149 @@ +/** + * The spill-policy PLUGIN: a `tools/post-execute` result transformer that keeps + * oversized plain-text tool results out of the model's context. When a final + * result's UTF-8 size exceeds `maxInlineBytes`, it saves the FULL text to a + * session-scoped spill file (`ctx.spillFiles`) and replaces the model-facing + * result with a bounded head/tail preview plus the spill path — the model reads + * the complete result later with the existing `read` tool. + * + * It registers NO service and owns NO storage or preview mechanics: preview is + * `@deepseek-ai/dsh-retention` (`TextRetainer`), storage is `ctx.spillFiles`. + * The policy only decides WHEN to spill and composes the notice. + * + * ## Deliberately narrow + * + * - Omitted `maxInlineBytes` ⇒ the plugin registers nothing (a true no-op). + * - Plain-text results only: a result carrying any non-text block is left + * untouched (the policy knows only the final formatted text, not tool + * internals). + * - `read` is skipped to avoid a `read → spill file → read again` loop. + * - Best-effort: no session owner, no `ctx.spillFiles` backend, or a save + * failure ⇒ log and return the original result. A spill failure must NEVER + * turn a successful tool call into an `isError` or hide the inline result. + * + * It COMPOSES with other post-execute listeners: it delegates via `next()` and + * bounds the resulting `accept` content, so a hook that replaced the content + * still has its replacement bounded, and a `block` decision passes through + * unchanged. + * + * @module @deepseek-ai/dsh-spill-policy + */ + +import type { Context } from 'cordis' +import z from 'schemastery' +import type { ContentBlock } from '@deepseek-ai/dsh-llm' +import { TextRetainer, describeOmitted } from '@deepseek-ai/dsh-retention' +import type { Omitted } from '@deepseek-ai/dsh-retention' +import type { SaveTextSpill } from '@deepseek-ai/dsh-spill' +import type { SessionId } from '@deepseek-ai/dsh-session' +import type { PostToolDecision, ToolExecution } from '@deepseek-ai/dsh-tools' +import type { SpillPolicyExec } from './types.ts' + +export type { SpillPolicyExec } from './types.ts' + +/** Plugin config. */ +export interface Config { + /** + * The model-facing context cap for a plain-text tool result, in UTF-8 bytes. + * Omitted disables the policy entirely (no-op). When set, a result larger than + * this is spilled and replaced with a preview derived from this same budget. + */ + maxInlineBytes?: number +} + +/** Cordis plugin name used by loader diagnostics. */ +export const name = 'spill-policy' + +/** Require the tool registry (its `tools/post-execute` waterfall is the seam we transform). */ +export const inject = ['tools'] + +export const Config: z = z.object({ + maxInlineBytes: z.number(), +}) + +/** All-text content flattened to one UTF-8 string, or `undefined` if any block is non-text. */ +function flattenPlainText(content: ContentBlock[]): string | undefined { + let text = '' + for (const block of content) { + if (block.type !== 'text') return undefined + text += block.text + } + return text +} + +/** The owning session id, or `undefined` for a call with no agent (a direct/test call). */ +function ownerSessionId(exec: ToolExecution): SessionId | undefined { + return (exec as SpillPolicyExec).agent?.session.header.id +} + +/** Build the bounded head/tail preview for `text`, splitting `maxInlineBytes` across the two ends. */ +function preview(text: string, maxInlineBytes: number): { text: string; omitted: Omitted } { + const headBytes = Math.ceil(maxInlineBytes / 2) + const tailBytes = Math.floor(maxInlineBytes / 2) + const retainer = new TextRetainer({ kind: 'headTail', headBytes, tailBytes }) + retainer.push(text) + const kept = retainer.finish() + return { text: kept.text, omitted: kept.omittedBytes } +} + +/** + * Compose the replacement text: the bounded preview, a blank line, then the + * spill notice. The omission clause comes from the retention library + * (`describeOmitted`); the recovery sentence names the concrete spill path. + */ +function replacementText(previewText: string, omitted: Omitted, spillPath: string): string { + const omission = describeOmitted(omitted, 'bytes') + const notice = `(${omission} Full formatted result saved to: ${spillPath}. Use read with offset/limit to inspect it.)` + return `${previewText}\n\n${notice}` +} + +export function apply(ctx: Context, config: Config): void { + const maxInlineBytes = config.maxInlineBytes + // Omitted ⇒ no automatic spill policy: register nothing at all. + if (maxInlineBytes === undefined) return + + ctx.on('tools/post-execute', async (exec, result, next): Promise => { + // Delegate first so a downstream listener (e.g. a hook) settles the result; + // we bound whatever it accepted. A block passes through — spill only shapes + // accepted plain-text results, never corrective feedback. + const decision = await next() + // Skip `read` to avoid a read → spill file → read again loop. + if (decision.kind !== 'accept' || exec.name === 'read') return decision + + const content = decision.content ?? result.content + const text = flattenPlainText(content) + if (text === undefined) return decision + if (Buffer.byteLength(text, 'utf8') <= maxInlineBytes) return decision + + const sessionId = ownerSessionId(exec) + if (sessionId === undefined) { + ctx.logger.warn(`spill-policy: no session owner for ${exec.name} result; keeping the inline result`) + return decision + } + const spillFiles = ctx.get('spillFiles') + if (!spillFiles) { + ctx.logger.warn('spill-policy: no ctx.spillFiles backend loaded; keeping the inline result') + return decision + } + + const save: SaveTextSpill = { + owner: { sessionId }, + source: { toolName: exec.name, callId: exec.callId, label: 'result' }, + suggestedName: `${exec.name}.txt`, + content: text, + } + let path: string + try { + ({ path } = await spillFiles.saveText(save)) + } catch (error: unknown) { + // Best-effort: a storage failure (permissions, ENOSPC, backend down) must + // never fail the call or hide the result — keep the original inline. + ctx.logger.warn(`spill-policy: saveText failed for ${exec.name}: ${String(error)}; keeping the inline result`) + return decision + } + + const { text: previewText, omitted } = preview(text, maxInlineBytes) + const replaced: ContentBlock[] = [{ type: 'text', text: replacementText(previewText, omitted, path) }] + return { kind: 'accept', content: replaced, ...decision.additionalContext ? { additionalContext: decision.additionalContext } : {} } + }) +} diff --git a/packages/spill/spill-policy/src/types.ts b/packages/spill/spill-policy/src/types.ts new file mode 100644 index 0000000000..032d0af550 --- /dev/null +++ b/packages/spill/spill-policy/src/types.ts @@ -0,0 +1,26 @@ +/** + * Vocabulary for the spill-policy plugin: the minimal structural view of a tool + * execution the policy needs to derive the owning session for a spill file. + * + * `@deepseek-ai/dsh-tools`' `ToolExecution` satisfies this shape, so the policy + * reads `exec` straight through without importing `dsh-tools` or `dsh-agent`. + * Only the session HEADER id is read — the same identity every other subsystem + * keys off (see `dsh-tool-bash`'s owner derivation). + * + * @module @deepseek-ai/dsh-spill-policy/types + */ + +import type { SessionId } from '@deepseek-ai/dsh-session' + +/** Minimal structural view of a tool execution: the owning session's header id, when present. */ +export interface SpillPolicyExec { + /** The agent on whose behalf the call runs, when there is one. */ + agent?: { + session: { + header: { + /** The canonical session identity — the spill owner. */ + id: SessionId + } + } + } +} diff --git a/packages/spill/spill-policy/tests/spill-policy.spec.ts b/packages/spill/spill-policy/tests/spill-policy.spec.ts new file mode 100644 index 0000000000..1a2592cb53 --- /dev/null +++ b/packages/spill/spill-policy/tests/spill-policy.spec.ts @@ -0,0 +1,196 @@ +/** + * Tests for the spill-policy PLUGIN. It registers no service, only the + * `tools/post-execute` transformer. We drive real tools through + * `ctx.tools.execute(...)` and assert: disabled mode is a true no-op, an + * oversized plain-text result is spilled and replaced with a preview + path, + * a small result and a non-text result pass through, `read` is skipped, and a + * `saveText` failure / missing backend / missing owner all preserve the original + * result without an `isError`. + */ + +import { describe, expect, it, vi } from 'vitest' +import { Context } from 'cordis' +import { CallId } from '@deepseek-ai/dsh-llm' +import type { ContentBlock } from '@deepseek-ai/dsh-llm' +import { SessionId } from '@deepseek-ai/dsh-session' +import SystemPrompt from '@deepseek-ai/dsh-system-prompt' +import ToolRegistry, { defineTool } from '@deepseek-ai/dsh-tools' +import type { ToolExecution } from '@deepseek-ai/dsh-tools' +import { SpillFiles, SpillPath } from '@deepseek-ai/dsh-spill' +import type { SaveTextSpill, SpillRef } from '@deepseek-ai/dsh-spill' +import * as SpillPolicy from '@deepseek-ai/dsh-spill-policy' + +/** A stub spill backend recording its saves; `fail` exercises the best-effort fallback. */ +class StubSpill extends SpillFiles { + saves: SaveTextSpill[] = [] + fail = false + + async saveText(input: SaveTextSpill): Promise { + if (this.fail) throw new Error('disk full') + this.saves.push(input) + return { path: SpillPath(`/spill/${input.suggestedName}`), bytes: Buffer.byteLength(input.content, 'utf8') } + } +} + +/** A tool returning `text` verbatim (name configurable so we can register `read`). */ +function textTool(name: string, text: string) { + return defineTool({ + name, + description: name, + parameters: {}, + async execute(): Promise { return [{ type: 'text', text }] }, + }) +} + +/** A minimal exec carrying a session header id (the spill owner). */ +function exec(name: string, session = 's1'): ToolExecution { + // Only agent.session.header.id is read by the policy; a structural stub suffices. + const agent = { session: { header: { id: SessionId(session) } } } + return { callId: CallId(`call-${name}`), name, arguments: {}, agent } as unknown as ToolExecution +} + +/** + * Build a context with tools + the policy, and optionally a spill backend. + * Returns the context and the backend handle (undefined when `withSpill` false). + */ +async function setup(config: SpillPolicy.Config, withSpill = true): Promise<{ ctx: Context; spill?: StubSpill }> { + const ctx = new Context() + await ctx.plugin(SystemPrompt) + await ctx.plugin(ToolRegistry) + let spill: StubSpill | undefined + if (withSpill) { + await ctx.plugin(StubSpill) + spill = ctx.spillFiles as StubSpill + } + await ctx.plugin(SpillPolicy, config) + return { ctx, ...spill ? { spill } : {} } +} + +/** Flatten a result's text blocks. */ +function textOf(content: ContentBlock[]): string { + return content.filter((b): b is Extract => b.type === 'text').map(b => b.text).join('') +} + +describe('disabled mode', () => { + it('registers no post-execute listener when maxInlineBytes is omitted', async () => { + const { ctx, spill } = await setup({}) + ctx.tools.register(textTool('big', 'x'.repeat(1000))) + const result = await ctx.tools.execute(exec('big')) + expect(textOf(result.content)).toBe('x'.repeat(1000)) + expect(result.isError).toBe(false) + expect(spill?.saves).toHaveLength(0) + }) +}) + +describe('oversized plain-text replacement', () => { + it('spills the full text and replaces the result with a preview + path', async () => { + const { ctx, spill } = await setup({ maxInlineBytes: 20 }) + const body = 'HEAD'.repeat(20) + 'TAIL'.repeat(20) // 160 bytes > 20 + ctx.tools.register(textTool('big', body)) + const result = await ctx.tools.execute(exec('big')) + + expect(result.isError).toBe(false) + expect(spill?.saves).toHaveLength(1) + expect(spill?.saves[0]?.content).toBe(body) + expect(spill?.saves[0]?.source.toolName).toBe('big') + expect(spill?.saves[0]?.suggestedName).toBe('big.txt') + expect(spill?.saves[0]?.owner.sessionId).toBe('s1') + + const text = textOf(result.content) + expect(text).not.toBe(body) + expect(text.startsWith('HEAD')).toBe(true) + expect(text).toContain('Full formatted result saved to: /spill/big.txt') + expect(text).toContain('Use read with offset/limit') + expect(text).toContain('Omitted') + }) + + it('leaves a small plain-text result unchanged', async () => { + const { ctx, spill } = await setup({ maxInlineBytes: 1000 }) + ctx.tools.register(textTool('small', 'tiny')) + const result = await ctx.tools.execute(exec('small')) + expect(textOf(result.content)).toBe('tiny') + expect(spill?.saves).toHaveLength(0) + }) + + it('leaves a result with a non-text block unchanged', async () => { + const { ctx, spill } = await setup({ maxInlineBytes: 5 }) + ctx.tools.register(defineTool({ + name: 'mixed', + description: 'mixed', + parameters: {}, + async execute(): Promise { + return [{ type: 'text', text: 'x'.repeat(100) }, { type: 'reasoning', text: 'why' }] + }, + })) + const result = await ctx.tools.execute(exec('mixed')) + expect(spill?.saves).toHaveLength(0) + expect(result.content).toHaveLength(2) + }) +}) + +describe('read skip', () => { + it('never spills the read tool result (avoids a read → spill → read loop)', async () => { + const { ctx, spill } = await setup({ maxInlineBytes: 10 }) + ctx.tools.register(textTool('read', 'x'.repeat(1000))) + const result = await ctx.tools.execute(exec('read')) + expect(textOf(result.content)).toBe('x'.repeat(1000)) + expect(spill?.saves).toHaveLength(0) + }) +}) + +describe('best-effort fallback', () => { + it('keeps the original result when saveText fails', async () => { + const { ctx, spill } = await setup({ maxInlineBytes: 10 }) + spill!.fail = true + const warn = vi.spyOn(ctx.logger, 'warn').mockImplementation(() => {}) + ctx.tools.register(textTool('big', 'x'.repeat(1000))) + const result = await ctx.tools.execute(exec('big')) + expect(textOf(result.content)).toBe('x'.repeat(1000)) + expect(result.isError).toBe(false) + expect(warn).toHaveBeenCalled() + }) + + it('keeps the original result when no spill backend is loaded', async () => { + const { ctx } = await setup({ maxInlineBytes: 10 }, false) + const warn = vi.spyOn(ctx.logger, 'warn').mockImplementation(() => {}) + ctx.tools.register(textTool('big', 'x'.repeat(1000))) + const result = await ctx.tools.execute(exec('big')) + expect(textOf(result.content)).toBe('x'.repeat(1000)) + expect(warn).toHaveBeenCalled() + }) + + it('keeps the original result when the call has no session owner', async () => { + const { ctx, spill } = await setup({ maxInlineBytes: 10 }) + const warn = vi.spyOn(ctx.logger, 'warn').mockImplementation(() => {}) + ctx.tools.register(textTool('big', 'x'.repeat(1000))) + const result = await ctx.tools.execute({ callId: CallId('c'), name: 'big', arguments: {} }) + expect(textOf(result.content)).toBe('x'.repeat(1000)) + expect(spill?.saves).toHaveLength(0) + expect(warn).toHaveBeenCalled() + }) +}) + +describe('composition', () => { + it('bounds content a downstream post-execute listener replaced', async () => { + const { ctx, spill } = await setup({ maxInlineBytes: 10 }) + // A later-registered listener replaces the (small) tool result with a big one; + // the policy delegated via next(), so it bounds the replacement. + ctx.on('tools/post-execute', async (_e, _r, _next) => + ({ kind: 'accept', content: [{ type: 'text', text: 'z'.repeat(500) }] })) + ctx.tools.register(textTool('small', 'tiny')) + const result = await ctx.tools.execute(exec('small')) + expect(spill?.saves[0]?.content).toBe('z'.repeat(500)) + expect(textOf(result.content)).toContain('Full formatted result saved to') + }) + + it('preserves a downstream accept decision additionalContext when spilling', async () => { + const { ctx } = await setup({ maxInlineBytes: 10 }) + const context = { content: [{ type: 'text' as const, text: 'note' }], source: { kind: 'plugin' as const, plugin: 'test' } } + ctx.on('tools/post-execute', async (_e, _r, _next) => + ({ kind: 'accept', additionalContext: context })) + ctx.tools.register(textTool('big', 'x'.repeat(1000))) + const result = await ctx.tools.execute(exec('big')) + expect(textOf(result.content)).toContain('Full formatted result saved to') + expect(result.additionalContext).toEqual(context) + }) +}) diff --git a/packages/spill/spill-policy/tsconfig.json b/packages/spill/spill-policy/tsconfig.json new file mode 100644 index 0000000000..6a81ab2f3c --- /dev/null +++ b/packages/spill/spill-policy/tsconfig.json @@ -0,0 +1,18 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "lib/types" + }, + "include": ["src"], + "references": [ + { "path": "../../../vendor/cosmokit" }, + { "path": "../../../vendor/cordis" }, + { "path": "../../../vendor/schemastery" }, + { "path": "../../util/retention" }, + { "path": "../../llm/llm" }, + { "path": "../../core/session" }, + { "path": "../spill" }, + { "path": "../../core/tools" } + ] +} diff --git a/packages/spill/spill/README.md b/packages/spill/spill/README.md new file mode 100644 index 0000000000..50f115573b --- /dev/null +++ b/packages/spill/spill/README.md @@ -0,0 +1,27 @@ +# @deepseek-ai/dsh-spill + +The **spill storage seam**: an abstract `SpillFiles` service (`ctx.spillFiles`) defining WHAT a spill backend does — persist a tool's oversized text to a session-scoped path the model can later `read` — without saying HOW. + +This package is one third of the spill capability, split so each concern evolves (and swaps) independently: + +| Package | Role | +|---|---| +| `@deepseek-ai/dsh-spill` (this) | the interface: abstract service + vocabulary types | +| `@deepseek-ai/dsh-spill-local` | an implementation: private session-scoped files on the host filesystem | +| `@deepseek-ai/dsh-spill-policy` | the tool-result policy that spills oversized final results | + +The split mirrors the bash/fs seams. A future remote or virtual backend (e.g. a `spill://…` URI plus a read-only bridge for ACP or remote environments) implements this interface without touching the policy plugin. + +## Service API (`ctx.spillFiles`) + +| Member | Semantics | +|---|---| +| `saveText(input)` | Persist `input.content` verbatim to a session-scoped file; resolves with a `SpillRef` (path readable by the local `read` tool + exact bytes written). **Rejects on a real storage failure** (permissions, ENOSPC, backend unavailable) — the caller decides how to degrade. | + +Storage is scoped by the request's `owner` session; the backend chooses a private (not world-readable) location and a collision-free name derived from — never equal to — the caller's `suggestedName`. The seam owns storage only: NO retention policy (that is [`@deepseek-ai/dsh-retention`](../../util/retention)), NO tool-result replacement (that is `@deepseek-ai/dsh-spill-policy`), NO file inspection (the model uses the existing `read` tool on the returned path). + +## Vocabulary + +`SaveTextSpill` (owner, source, suggestedName, content) is the request; `SpillRef` (path, bytes) is the result. `SpillPath` is [branded](../../util/brand) and rendered to the model as an ordinary path string in v1 — the brand records provenance (a runtime artifact, not a workspace file) so a future virtual backend can swap the path shape without a consumer change. `SpillOwner` scopes storage to a `SessionId`; unlike the bash executor's decoupled `OwnerToken`, spill is inherently session-scoped, so the seam imports `dsh-session`'s `SessionId` directly. `SpillSource` (toolName, callId, label) is descriptive provenance for the filename and future cleanup, not access control. See `src/types.ts` for the full contracts. + +See the [tool output spill RFC](../../../docs/rfc/implemented/architecture/2026-07-08-tool-output-spill-files.md) for the design rationale, including why creation belongs to the runtime spill seam rather than the model-facing `write` tool. diff --git a/packages/spill/spill/package.json b/packages/spill/spill/package.json new file mode 100644 index 0000000000..167c67183c --- /dev/null +++ b/packages/spill/spill/package.json @@ -0,0 +1,36 @@ +{ + "name": "@deepseek-ai/dsh-spill", + "description": "Abstract spill storage seam (ctx.spillFiles) for the DeepSeek Harness — save oversized tool text to a session-scoped path", + "version": "0.0.1", + "private": true, + "type": "module", + "main": "lib/index.js", + "types": "lib/types/index.d.ts", + "exports": { + ".": { + "types": "./lib/types/index.d.ts", + "default": "./lib/index.js" + }, + "./src/*": "./src/*", + "./package.json": "./package.json" + }, + "files": [ + "lib/index.js", + "lib/types/**/*.d.ts", + "lib/types/**/*.d.ts.map", + "src" + ], + "license": "BSD-3-Clause", + "peerDependencies": { + "@deepseek-ai/dsh-brand": "^0.0.1", + "@deepseek-ai/dsh-llm": "^0.0.1", + "@deepseek-ai/dsh-session": "^0.0.1", + "cordis": "^4.0.0-rc.6" + }, + "devDependencies": { + "@deepseek-ai/dsh-brand": "workspace:^", + "@deepseek-ai/dsh-llm": "workspace:^", + "@deepseek-ai/dsh-session": "workspace:^", + "cordis": "^4.0.0-rc.6" + } +} diff --git a/packages/spill/spill/src/index.ts b/packages/spill/spill/src/index.ts new file mode 100644 index 0000000000..4c8fa37030 --- /dev/null +++ b/packages/spill/spill/src/index.ts @@ -0,0 +1,60 @@ +/** + * The spill storage seam (`ctx.spillFiles`): an abstract service defining WHAT a + * spill backend does — persist a tool's oversized text to a session-scoped path + * the model can later `read` — without saying HOW. Implementations subclass + * {@link SpillFiles} and register as the `spillFiles` service; + * `@deepseek-ai/dsh-spill-local` (host filesystem) is the first. + * + * The seam is deliberately minimal: `saveText` and nothing else. It owns NO + * retention policy (that is `@deepseek-ai/dsh-retention`), NO tool-result + * replacement (that is `@deepseek-ai/dsh-spill-policy`), and NO file inspection + * (the model uses the existing `read` tool on the returned path). A future + * remote/virtual backend may return a `spill://…` URI plus a read-only bridge; + * v1 keeps the path filesystem-shaped until such a backend exists. + * + * @module @deepseek-ai/dsh-spill + */ + +import { Context, Service } from 'cordis' +import type { SaveTextSpill, SpillRef } from './types.ts' + +export { SpillPath } from './types.ts' +export type { SaveTextSpill, SpillOwner, SpillRef, SpillSource } from './types.ts' + +declare module 'cordis' { + interface Context { + spillFiles: SpillFiles + } +} + +/** + * Abstract spill storage service. Subclass, implement {@link saveText}, and load + * the subclass as a plugin — it registers as `ctx.spillFiles` (one + * implementation per context; loading a second throws, cordis' standard + * duplicate-service behavior). + * + * Semantics every implementation must honor: + * - {@link saveText} persists the FULL `content` verbatim and returns a path + * the local `read` tool can open, plus the exact byte length written. + * - Storage is scoped by the request's {@link SaveTextSpill.owner} session; the + * backend chooses a private (not world-readable) location and a collision-free + * name derived from — never equal to — the caller's `suggestedName`. + * - `saveText` REJECTS on a real storage failure (permissions, ENOSPC, backend + * unavailable); the caller decides how to degrade (the spill policy treats a + * rejection as best-effort and keeps the inline result). + */ +export abstract class SpillFiles extends Service { + constructor(ctx: Context) { + super(ctx, 'spillFiles') + } + + /** + * Persist `input.content` to a session-scoped spill file. + * @param input - the owner, provenance, suggested name, and full text to save. + * @returns the saved file's {@link SpillRef} (path + bytes written); rejects on + * a storage failure. + */ + abstract saveText(input: SaveTextSpill): Promise +} + +export default SpillFiles diff --git a/packages/spill/spill/src/types.ts b/packages/spill/spill/src/types.ts new file mode 100644 index 0000000000..8dfd4c1d1d --- /dev/null +++ b/packages/spill/spill/src/types.ts @@ -0,0 +1,68 @@ +/** + * Vocabulary for the spill storage seam. Types only — the abstract service + * lives in `./index.ts`, implementations in sibling packages + * (`@deepseek-ai/dsh-spill-local` first). + * + * @module @deepseek-ai/dsh-spill/types + */ + +import type { Branded } from '@deepseek-ai/dsh-brand' +import type { CallId } from '@deepseek-ai/dsh-llm' +import type { SessionId } from '@deepseek-ai/dsh-session' + +/** + * A local filesystem path produced by the spill seam, intended for the model's + * `read` tool. The brand records that the path came from {@link SpillFiles.saveText} + * (a runtime artifact, not a workspace file); it is still rendered to the model + * as an ordinary path string in v1. A future remote/virtual backend may replace + * this with a `spill://…` URI, so consumers treat it as opaque. + */ +export type SpillPath = Branded<'SpillPath'> + +/** Brand a string as a {@link SpillPath}. */ +export function SpillPath(path: string): SpillPath { + return path as SpillPath +} + +/** + * Who a spilled file belongs to: the session whose tool call produced it. The + * backend scopes storage per session (its directory layout, its cleanup unit), + * so the owner is the session id, not a decoupled token — spill is inherently + * session-scoped, unlike the bash executor's cross-session `OwnerToken`. + */ +export interface SpillOwner { + sessionId: SessionId +} + +/** + * Provenance of one spilled artifact — recorded by the backend for a readable + * filename and future cleanup/inspection. Not interpreted for access control + * (the {@link SpillOwner} scopes storage); purely descriptive. + */ +export interface SpillSource { + /** The tool whose result was spilled (e.g. `web_fetch`). */ + toolName: string + /** The model-issued call id the result belongs to. */ + callId: CallId + /** A short human label for the artifact (e.g. `result`). */ + label: string +} + +/** One request to persist text to a spill file. */ +export interface SaveTextSpill { + owner: SpillOwner + source: SpillSource + /** + * A caller-suggested base name (e.g. `web_fetch.txt`). The backend sanitizes + * it to a single safe path segment before use — it is a hint, never a path. + */ + suggestedName: string + /** The full text to persist (UTF-8). */ + content: string +} + +/** A saved spill file: its path plus the byte length written. */ +export interface SpillRef { + path: SpillPath + bytes: number +} diff --git a/packages/spill/spill/tests/service.spec.ts b/packages/spill/spill/tests/service.spec.ts new file mode 100644 index 0000000000..271725442b --- /dev/null +++ b/packages/spill/spill/tests/service.spec.ts @@ -0,0 +1,56 @@ +/** + * Tests for the spill seam INTERFACE: a minimal concrete subclass registers as + * `ctx.spillFiles`, a second load throws (duplicate service), and disposal + * releases the service. The storage behavior is the implementation's concern + * (`@deepseek-ai/dsh-spill-local`); here we only pin the seam contract. + */ + +import { describe, expect, it } from 'vitest' +import { Context } from 'cordis' +import { CallId } from '@deepseek-ai/dsh-llm' +import { SessionId } from '@deepseek-ai/dsh-session' +import { SpillFiles, SpillPath } from '@deepseek-ai/dsh-spill' +import type { SaveTextSpill, SpillRef } from '@deepseek-ai/dsh-spill' + +/** Minimal concrete backend: records the last request, returns a fixed ref. */ +class StubSpill extends SpillFiles { + last: SaveTextSpill | undefined + + async saveText(input: SaveTextSpill): Promise { + this.last = input + return { path: SpillPath(`/stub/${input.suggestedName}`), bytes: Buffer.byteLength(input.content, 'utf8') } + } +} + +function request(content: string): SaveTextSpill { + return { + owner: { sessionId: SessionId('s1') }, + source: { toolName: 'web_fetch', callId: CallId('c1'), label: 'result' }, + suggestedName: 'web_fetch.txt', + content, + } +} + +describe('spill seam', () => { + it('registers as ctx.spillFiles and saves text', async () => { + const ctx = new Context() + await ctx.plugin(StubSpill) + const ref = await ctx.spillFiles.saveText(request('hello')) + expect(ref).toEqual({ path: '/stub/web_fetch.txt', bytes: 5 }) + expect((ctx.spillFiles as StubSpill).last?.content).toBe('hello') + }) + + it('rejects a second implementation (one per context)', async () => { + const ctx = new Context() + await ctx.plugin(StubSpill) + await expect(ctx.plugin(StubSpill)).rejects.toThrow() + }) + + it('releases the service on disposal', async () => { + const ctx = new Context() + const fiber = await ctx.plugin(StubSpill) + expect(ctx.spillFiles).toBeInstanceOf(StubSpill) + await fiber.dispose() + expect((ctx as Context & { spillFiles?: unknown }).spillFiles).toBeUndefined() + }) +}) diff --git a/packages/spill/spill/tsconfig.json b/packages/spill/spill/tsconfig.json new file mode 100644 index 0000000000..0c2fd5c57f --- /dev/null +++ b/packages/spill/spill/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "lib/types" + }, + "include": ["src"], + "references": [ + { "path": "../../../vendor/cosmokit" }, + { "path": "../../../vendor/cordis" }, + { "path": "../../util/brand" }, + { "path": "../../llm/llm" }, + { "path": "../../core/session" } + ] +} diff --git a/packages/web/tool-web/package.json b/packages/web/tool-web/package.json index 8c22afa9a8..c777ae869b 100644 --- a/packages/web/tool-web/package.json +++ b/packages/web/tool-web/package.json @@ -35,6 +35,8 @@ "@deepseek-ai/dsh-agent": "workspace:^", "@deepseek-ai/dsh-llm": "workspace:^", "@deepseek-ai/dsh-session": "workspace:^", + "@deepseek-ai/dsh-spill-local": "workspace:^", + "@deepseek-ai/dsh-spill-policy": "workspace:^", "@deepseek-ai/dsh-system-prompt": "workspace:^", "@deepseek-ai/dsh-tools": "workspace:^", "@deepseek-ai/dsh-web": "workspace:^", diff --git a/packages/web/tool-web/tests/spill.spec.ts b/packages/web/tool-web/tests/spill.spec.ts new file mode 100644 index 0000000000..6e2ce6d11d --- /dev/null +++ b/packages/web/tool-web/tests/spill.spec.ts @@ -0,0 +1,93 @@ +/** + * Showcase integration: the real `web_fetch` tool + the real spill stack + * (`dsh-spill-local` backend + `dsh-spill-policy`), exercised through + * `ctx.tools.execute()`. Proves the RFC's default path — a large formatted fetch + * result is automatically retained and spilled with NO tool-specific spill code, + * and the model-facing text changes ONLY by the deliberate spill notice (the + * full formatted result lands in the spill file). + */ + +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http' +import { AddressInfo } from 'node:net' +import { mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Context } from 'cordis' +import { CallId } from '@deepseek-ai/dsh-llm' +import { SessionId } from '@deepseek-ai/dsh-session' +import SystemPrompt from '@deepseek-ai/dsh-system-prompt' +import ToolRegistry from '@deepseek-ai/dsh-tools' +import type { ToolExecution } from '@deepseek-ai/dsh-tools' +import WebService from '@deepseek-ai/dsh-web' +import * as WebFetchLocal from '@deepseek-ai/dsh-web-fetch-local' +import LocalSpillFiles from '@deepseek-ai/dsh-spill-local' +import * as SpillPolicy from '@deepseek-ai/dsh-spill-policy' +import * as ToolWeb from '@deepseek-ai/dsh-tool-web' + +type Handler = (req: IncomingMessage, res: ServerResponse) => void + +let server: Server +let base: string +let handler: Handler +let spillRoot: string +let ctx: Context + +const BODY = 'X'.repeat(4000) // formatted result is well over the 200-byte policy cap + +beforeEach(async () => { + handler = (_req, res) => { res.writeHead(200, { 'content-type': 'text/plain' }); res.end(BODY) } + server = createServer((req, res) => { handler(req, res) }) + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)) + base = `http://127.0.0.1:${(server.address() as AddressInfo).port}` + spillRoot = mkdtempSync(join(tmpdir(), 'dsh-spill-web-')) + + ctx = new Context() + await ctx.plugin(SystemPrompt) + await ctx.plugin(ToolRegistry) + await ctx.plugin(WebService, { fetchProvider: WebFetchLocal.LOCAL_FETCH_PROVIDER_ID }) + // Provider cap generous so the tool returns a large formatted result; the + // policy cap is what triggers the spill (the RFC's separation of concerns). + await ctx.plugin(WebFetchLocal, { maxBodyChars: 500_000 }) + await ctx.plugin(LocalSpillFiles, { root: spillRoot }) + await ctx.plugin(SpillPolicy, { maxInlineBytes: 200 }) + await ctx.plugin(ToolWeb) +}) + +afterEach(async () => { + await new Promise(resolve => server.close(() => { resolve() })) + rmSync(spillRoot, { recursive: true, force: true }) +}) + +/** A web_fetch call carrying a session owner (so the policy can scope the spill). */ +function fetchCall(): Promise<{ isError: boolean; content: { type: string; text?: string }[] }> { + const agent = { session: { header: { id: SessionId('web-sess') } } } + const exec = { callId: CallId('call-1'), name: 'web_fetch', arguments: { url: base }, agent } as unknown as ToolExecution + return ctx.tools.execute(exec) +} + +describe('web_fetch spill showcase', () => { + it('spills a large formatted result and returns a preview + spill path', async () => { + const out = await fetchCall() + expect(out.isError).toBe(false) + const text = out.content.map(b => b.text).join('') + + // Model-facing text is a preview + notice, NOT the full body. + expect(text.length).toBeLessThan(BODY.length) + expect(text).toContain(`Fetched ${base}`) // the head of the formatted result survives + expect(text).toContain('Full formatted result saved to:') + expect(text).toContain('Use read with offset/limit') + + // The spill file holds the FULL formatted result the tool returned. + const match = /saved to: (\S+?)\. Use read/.exec(text) + expect(match).not.toBeNull() + const spillPath = match![1]! + const saved = readFileSync(spillPath, 'utf8') + // The provider cap was generous, so the tool did not truncate: the spill file + // holds the full formatted result (header + the complete body), far larger + // than the model-facing preview. + expect(saved).toContain('(HTTP 200)') + expect(saved).toContain(BODY) + expect(saved.length).toBeGreaterThan(text.length) + }) +}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3bf60ac60a..21632275ae 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -555,6 +555,71 @@ importers: specifier: ^4.0.0-rc.6 version: 4.0.0-rc.6(@cordisjs/plugin-include@1.0.4)(@cordisjs/plugin-loader@1.0.0-rc.4) + packages/spill/spill: + devDependencies: + '@deepseek-ai/dsh-brand': + specifier: workspace:^ + version: link:../../util/brand + '@deepseek-ai/dsh-llm': + specifier: workspace:^ + version: link:../../llm/llm + '@deepseek-ai/dsh-session': + specifier: workspace:^ + version: link:../../core/session + cordis: + specifier: ^4.0.0-rc.6 + version: 4.0.0-rc.6(@cordisjs/plugin-include@1.0.4)(@cordisjs/plugin-loader@1.0.0-rc.4) + + packages/spill/spill-local: + dependencies: + schemastery: + specifier: ^3.18.0 + version: 3.18.0 + devDependencies: + '@deepseek-ai/dsh-brand': + specifier: workspace:^ + version: link:../../util/brand + '@deepseek-ai/dsh-llm': + specifier: workspace:^ + version: link:../../llm/llm + '@deepseek-ai/dsh-session': + specifier: workspace:^ + version: link:../../core/session + '@deepseek-ai/dsh-spill': + specifier: workspace:^ + version: link:../spill + cordis: + specifier: ^4.0.0-rc.6 + version: 4.0.0-rc.6(@cordisjs/plugin-include@1.0.4)(@cordisjs/plugin-loader@1.0.0-rc.4) + + packages/spill/spill-policy: + dependencies: + schemastery: + specifier: ^3.18.0 + version: 3.18.0 + devDependencies: + '@deepseek-ai/dsh-agent': + specifier: workspace:^ + version: link:../../core/agent + '@deepseek-ai/dsh-llm': + specifier: workspace:^ + version: link:../../llm/llm + '@deepseek-ai/dsh-retention': + specifier: workspace:^ + version: link:../../util/retention + '@deepseek-ai/dsh-session': + specifier: workspace:^ + version: link:../../core/session + '@deepseek-ai/dsh-spill': + specifier: workspace:^ + version: link:../spill + '@deepseek-ai/dsh-tools': + specifier: workspace:^ + version: link:../../core/tools + cordis: + specifier: ^4.0.0-rc.6 + version: 4.0.0-rc.6(@cordisjs/plugin-include@1.0.4)(@cordisjs/plugin-loader@1.0.0-rc.4) + packages/subagent/subagent: devDependencies: '@deepseek-ai/dsh-agent': @@ -990,6 +1055,12 @@ importers: '@deepseek-ai/dsh-session': specifier: workspace:^ version: link:../../core/session + '@deepseek-ai/dsh-spill-local': + specifier: workspace:^ + version: link:../../spill/spill-local + '@deepseek-ai/dsh-spill-policy': + specifier: workspace:^ + version: link:../../spill/spill-policy '@deepseek-ai/dsh-system-prompt': specifier: workspace:^ version: link:../../core/system-prompt diff --git a/scripts/gen-doc-graphs.ts b/scripts/gen-doc-graphs.ts index 4f90b79e8e..c8f6204bd5 100644 --- a/scripts/gen-doc-graphs.ts +++ b/scripts/gen-doc-graphs.ts @@ -74,6 +74,7 @@ const GROUP_ORDER = [ 'compact', 'subagent', 'web', + 'spill', 'todo', 'hooks', 'session-persistence', @@ -186,6 +187,15 @@ const SERVICE_ROLES: ServiceRole[] = [ consumers: ['tool-web'], note: 'Search and fetch providers register into one ctx.web seam; tool-web owns the stable model-facing names.', }, + { + key: 'spillFiles', + pkg: 'spill', + title: 'Spill storage seam', + mode: 'seam', + implementations: ['spill-local'], + consumers: ['spill-policy'], + note: 'The backend saves oversized tool text to a session-scoped path; spill-policy is the tools/post-execute consumer that decides when to spill.', + }, ] const DYNAMIC_EVENT_DISPATCHERS: Array<{ event: string; pkg: string; method: string }> = [ diff --git a/scripts/gen-module-graph.ts b/scripts/gen-module-graph.ts index b84701c819..20c421268e 100644 --- a/scripts/gen-module-graph.ts +++ b/scripts/gen-module-graph.ts @@ -45,6 +45,7 @@ const GROUP_ORDER = [ 'compact', 'subagent', 'web', + 'spill', 'todo', 'hooks', 'session-persistence', diff --git a/tsconfig.base.json b/tsconfig.base.json index 40e4dbe728..a7a28f6b47 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -47,6 +47,7 @@ "./packages/compact/*/src", "./packages/subagent/*/src", "./packages/web/*/src", + "./packages/spill/*/src", "./packages/todo/*/src", "./packages/hooks/*/src", "./packages/session-persistence/*/src", diff --git a/tsconfig.build.json b/tsconfig.build.json index 8f1a869d91..562d1004cd 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -40,6 +40,9 @@ { "path": "./packages/web/web-search-deepseek" }, { "path": "./packages/web/web-fetch-local" }, { "path": "./packages/web/tool-web" }, + { "path": "./packages/spill/spill" }, + { "path": "./packages/spill/spill-local" }, + { "path": "./packages/spill/spill-policy" }, { "path": "./packages/support/invariants" }, { "path": "./packages/ui/acp" }, { "path": "./packages/ui/acp-agent" }, diff --git a/tsconfig.json b/tsconfig.json index 4a4cc1668f..6c817a4929 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -51,6 +51,9 @@ { "path": "./packages/web/web-search-deepseek" }, { "path": "./packages/web/web-fetch-local" }, { "path": "./packages/web/tool-web" }, + { "path": "./packages/spill/spill" }, + { "path": "./packages/spill/spill-local" }, + { "path": "./packages/spill/spill-policy" }, { "path": "./packages/support/invariants" }, { "path": "./packages/ui/acp" }, { "path": "./packages/ui/acp-agent" }, From d0c2f0916dfd14f9299f25e9c9669e760c378a60 Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Wed, 8 Jul 2026 22:54:26 +0800 Subject: [PATCH 07/24] fix: address codex review round 1 - spill-policy validates maxInlineBytes as a non-negative integer at LOAD, so a bad config fails the deployment instead of letting a negative value reach TextRetainer and turn every oversized-result call into an isError. - Document the spill seam vocabulary in docs/core-data-structures/spill.md (SaveTextSpill/SpillOwner/SpillSource/SpillRef/SpillPath, verbatim + type-equiv gated) and index it from core.md, matching the other capability seams. --- docs/core-data-structures/core.md | 1 + docs/core-data-structures/spill.md | 55 +++++++++++++++++++ packages/spill/spill-policy/README.md | 2 +- packages/spill/spill-policy/src/index.ts | 6 ++ .../spill-policy/tests/spill-policy.spec.ts | 10 ++++ scripts/type-equiv.manifest.json | 8 ++- 6 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 docs/core-data-structures/spill.md diff --git a/docs/core-data-structures/core.md b/docs/core-data-structures/core.md index 61f980ff73..5ec00acf45 100644 --- a/docs/core-data-structures/core.md +++ b/docs/core-data-structures/core.md @@ -24,6 +24,7 @@ Everything else is documented on a **sub-page**, not here. The rule that draws t | [compaction.md](compaction.md) | the compaction seam: the `compact/*` session events, `CompactionResult`, the `CompactService` interface | | [subagent.md](subagent.md) | the subagent seam: the named-provider registry, `SubagentStartRequest`/`Result`/`Run`, the start-time-vs-runtime capability split | | [web.md](web.md) | the web access seam: `WebSearchRequest`/`Result`, `WebFetchRequest`/`Result`, `WebFetchBody`, provider/capability status, `WebError` | +| [spill.md](spill.md) | the spill storage seam: `SaveTextSpill`, `SpillOwner`/`SpillSource`, `SpillRef`, the branded `SpillPath` | > Type definitions on this page are pasted **verbatim** from source and drift-checked by `pnpm run verify-type-equiv` (see [development.md](../development.md#documenting-types-verbatim-ts-type-equiv)). Inline JSDoc is omitted for readability; follow the source link for the full contracts. diff --git a/docs/core-data-structures/spill.md b/docs/core-data-structures/spill.md new file mode 100644 index 0000000000..7586d70633 --- /dev/null +++ b/docs/core-data-structures/spill.md @@ -0,0 +1,55 @@ +# Spill Storage + +The spill storage seam — a [capability seam](../rfc/implemented/architecture/2026-07-08-tool-output-spill-files.md) that persists a tool's oversized text to a session-scoped path the model can later `read`, split across packages: interface ([dsh-spill](../../packages/spill/spill), `ctx.spillFiles`), implementation ([dsh-spill-local](../../packages/spill/spill-local), private session-scoped files on the host filesystem), and consumer ([dsh-spill-policy](../../packages/spill/spill-policy), the `tools/post-execute` policy). Spill is **one optional capability**, not part of the agent-loop spine — so its vocabulary lives here, not in [core.md](core.md). Preview mechanics stay in [dsh-retention](../../packages/util/retention); this seam only saves the final text the policy hands it. + +Source: [`packages/spill/spill/src/types.ts`](../../packages/spill/spill/src/types.ts) + +## The save request + +`saveText` is the whole seam: persist `content` verbatim, return a readable path plus the exact byte count. The request carries WHO the artifact belongs to (`owner`), WHERE it came from (`source`, descriptive provenance for the filename and future cleanup — not access control), and a `suggestedName` the backend sanitizes to one safe path segment before use (it is a hint, never a path). + +```ts type-equiv +interface SaveTextSpill { + owner: SpillOwner + source: SpillSource + suggestedName: string + content: string +} +``` + +```ts type-equiv +interface SpillOwner { + sessionId: SessionId +} +``` + +`SpillOwner` scopes storage to a `SessionId` — spill is inherently session-scoped (its directory layout and future cleanup unit are per session), so the seam imports `dsh-session`'s `SessionId` directly rather than minting a decoupled token like the bash executor's cross-session `OwnerToken` ([bash.md](bash.md)). + +```ts type-equiv +interface SpillSource { + toolName: string + callId: CallId + label: string +} +``` + +## The result + +```ts type-equiv +interface SpillRef { + path: SpillPath + bytes: number +} +``` + +`SpillPath` is a [branded](core.md#branded-ids) local filesystem path returned by the backend and intended for the model's `read` tool. The brand records that the path came from the spill seam (a runtime artifact, not a workspace file the model authored); it is still rendered to the model as an ordinary path string in v1. A future remote or virtual backend may replace it with a `spill://…` URI plus a read-only filesystem bridge, so consumers treat it as opaque. + +```ts type-equiv +type SpillPath = Branded<'SpillPath'> +``` + +## The service + +`SpillFiles` (`ctx.spillFiles`, defined in [`packages/spill/spill/src/index.ts`](../../packages/spill/spill/src/index.ts)) is a one-method abstract service: `saveText(input) → Promise`. It persists the FULL `content`, chooses a private (not world-readable) location and a collision-free name derived from — never equal to — `suggestedName`, and REJECTS on a real storage failure (permissions, ENOSPC, backend unavailable). The seam owns storage only: no retention policy, no tool-result replacement, no file inspection. + +The local backend ([dsh-spill-local](../../packages/spill/spill-local)) writes under `/session-/-` — a configured or lazily-created private (0700) root, a `sha256(sessionId)` session subdir, and an exclusive owner-only (`open(path, 'wx', 0o600)`) write so a planted symlink cannot redirect it. The policy consumer ([dsh-spill-policy](../../packages/spill/spill-policy)) replaces an over-`maxInlineBytes` plain-text final result with a retention-library head/tail preview plus the spill path, best-effort: a save failure keeps the original inline result rather than turning a successful call into an `isError`. diff --git a/packages/spill/spill-policy/README.md b/packages/spill/spill-policy/README.md index 42c3334bf7..24e720c5c8 100644 --- a/packages/spill/spill-policy/README.md +++ b/packages/spill/spill-policy/README.md @@ -8,7 +8,7 @@ This plugin registers **no service** and owns no storage or preview mechanics: p | Key | Default | Meaning | |---|---|---| -| `maxInlineBytes` | *(omitted)* | Model-facing context cap for a plain-text result, in UTF-8 bytes. **Omitted disables the policy entirely** (the plugin registers nothing). When set, a larger result is spilled and replaced with a preview derived from the same budget (head/tail split). | +| `maxInlineBytes` | *(omitted)* | Model-facing context cap for a plain-text result, in UTF-8 bytes (a non-negative integer; validated at load). **Omitted disables the policy entirely** (the plugin registers nothing). When set, a larger result is spilled and replaced with a preview derived from the same budget (head/tail split). | ## Behavior diff --git a/packages/spill/spill-policy/src/index.ts b/packages/spill/spill-policy/src/index.ts index f4672a4751..57c651d70c 100644 --- a/packages/spill/spill-policy/src/index.ts +++ b/packages/spill/spill-policy/src/index.ts @@ -101,6 +101,12 @@ export function apply(ctx: Context, config: Config): void { const maxInlineBytes = config.maxInlineBytes // Omitted ⇒ no automatic spill policy: register nothing at all. if (maxInlineBytes === undefined) return + // Validate at LOAD, not per call: a negative/fractional cap would reach + // TextRetainer's assertBudget and throw, turning every oversized-result call + // into an isError. A bad config must fail the deployment, not the tool. + if (!Number.isInteger(maxInlineBytes) || maxInlineBytes < 0) { + throw new Error(`spill-policy: maxInlineBytes must be a non-negative integer (got ${maxInlineBytes})`) + } ctx.on('tools/post-execute', async (exec, result, next): Promise => { // Delegate first so a downstream listener (e.g. a hook) settles the result; diff --git a/packages/spill/spill-policy/tests/spill-policy.spec.ts b/packages/spill/spill-policy/tests/spill-policy.spec.ts index 1a2592cb53..61da0abe76 100644 --- a/packages/spill/spill-policy/tests/spill-policy.spec.ts +++ b/packages/spill/spill-policy/tests/spill-policy.spec.ts @@ -82,6 +82,16 @@ describe('disabled mode', () => { }) }) +describe('config validation', () => { + it('rejects a negative maxInlineBytes at load', async () => { + await expect(setup({ maxInlineBytes: -1 })).rejects.toThrow(/non-negative integer/) + }) + + it('rejects a fractional maxInlineBytes at load', async () => { + await expect(setup({ maxInlineBytes: 1.5 })).rejects.toThrow(/non-negative integer/) + }) +}) + describe('oversized plain-text replacement', () => { it('spills the full text and replaces the result with a preview + path', async () => { const { ctx, spill } = await setup({ maxInlineBytes: 20 }) diff --git a/scripts/type-equiv.manifest.json b/scripts/type-equiv.manifest.json index b1c3163782..8dcf5c4715 100644 --- a/scripts/type-equiv.manifest.json +++ b/scripts/type-equiv.manifest.json @@ -77,6 +77,12 @@ { "doc": "docs/core-data-structures/web.md", "symbol": "WebFetchRequest", "source": "packages/web/web/src/types.ts" }, { "doc": "docs/core-data-structures/web.md", "symbol": "WebFetchResult", "source": "packages/web/web/src/types.ts" }, { "doc": "docs/core-data-structures/web.md", "symbol": "WebFetchBody", "source": "packages/web/web/src/types.ts" }, - { "doc": "docs/core-data-structures/web.md", "symbol": "WebProviderStatus", "source": "packages/web/web/src/types.ts" } + { "doc": "docs/core-data-structures/web.md", "symbol": "WebProviderStatus", "source": "packages/web/web/src/types.ts" }, + + { "doc": "docs/core-data-structures/spill.md", "symbol": "SaveTextSpill", "source": "packages/spill/spill/src/types.ts" }, + { "doc": "docs/core-data-structures/spill.md", "symbol": "SpillOwner", "source": "packages/spill/spill/src/types.ts" }, + { "doc": "docs/core-data-structures/spill.md", "symbol": "SpillSource", "source": "packages/spill/spill/src/types.ts" }, + { "doc": "docs/core-data-structures/spill.md", "symbol": "SpillRef", "source": "packages/spill/spill/src/types.ts" }, + { "doc": "docs/core-data-structures/spill.md", "symbol": "SpillPath", "source": "packages/spill/spill/src/types.ts" } ] } From 326b199f255c61647c190286960dd974488a70ee Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Thu, 9 Jul 2026 09:51:35 +0800 Subject: [PATCH 08/24] fix: address codex review round 2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - spill-policy reserves the spill notice's byte cost inside maxInlineBytes, so the replacement (preview + notice) never exceeds the documented model-facing cap. When the notice alone fills the budget the preview is empty; when even a notice-only replacement is not smaller than the original, the inline result is kept (spilling would only add bytes). - retention TextRetainer trims an oversized single suffix chunk to the last suffixCap bytes on push, so tail/headTail retention stays bounded by suffixCap instead of retaining and re-copying the whole chunk in finish() — this is the spill preview path, which pushes the whole result as one chunk. --- packages/spill/spill-policy/README.md | 4 +- packages/spill/spill-policy/src/index.ts | 46 +++++++++++++------ .../spill-policy/tests/spill-policy.spec.ts | 22 +++++++-- packages/util/retention/src/index.ts | 13 ++++++ packages/web/tool-web/tests/spill.spec.ts | 8 ++-- 5 files changed, 71 insertions(+), 22 deletions(-) diff --git a/packages/spill/spill-policy/README.md b/packages/spill/spill-policy/README.md index 24e720c5c8..dcc2fffb30 100644 --- a/packages/spill/spill-policy/README.md +++ b/packages/spill/spill-policy/README.md @@ -16,7 +16,7 @@ This plugin registers **no service** and owns no storage or preview mechanics: p 2. Skip `read` (avoids a `read → spill file → read again` loop) and any non-`accept` decision (a `block`'s corrective feedback passes through). 3. Flatten the accepted content only when it is **plain text** (all `text` blocks); a result with any non-text block is left untouched. 4. If its UTF-8 size is `≤ maxInlineBytes`, leave it unchanged. -5. Otherwise save the full text and replace the result with a preview + this notice: +5. Otherwise save the full text and replace the result with a preview + this notice, sized so the whole replacement (preview + blank line + notice) stays within `maxInlineBytes` — the notice's byte cost is reserved out of the budget, so the preview shrinks to fit and the model-facing result never exceeds the cap: ```text @@ -24,6 +24,8 @@ This plugin registers **no service** and owns no storage or preview mechanics: p (Omitted N bytes. Full formatted result saved to: /…/session-…/…-web_fetch.txt. Use read with offset/limit to inspect it.) ``` + When the notice alone fills the budget (a tiny cap or a long path) the preview is empty and only the notice is returned. If even that notice-only replacement is not smaller than the original result, the policy keeps the inline result — spilling would only add bytes. + **Best-effort:** no session owner, no `ctx.spillFiles` backend, or a `saveText` rejection ⇒ the policy logs a warning and returns the original result. A spill failure never turns a successful call into an `isError` or hides the inline result. ## Scope diff --git a/packages/spill/spill-policy/src/index.ts b/packages/spill/spill-policy/src/index.ts index 57c651d70c..9fa55fcdf4 100644 --- a/packages/spill/spill-policy/src/index.ts +++ b/packages/spill/spill-policy/src/index.ts @@ -76,25 +76,20 @@ function ownerSessionId(exec: ToolExecution): SessionId | undefined { return (exec as SpillPolicyExec).agent?.session.header.id } -/** Build the bounded head/tail preview for `text`, splitting `maxInlineBytes` across the two ends. */ -function preview(text: string, maxInlineBytes: number): { text: string; omitted: Omitted } { - const headBytes = Math.ceil(maxInlineBytes / 2) - const tailBytes = Math.floor(maxInlineBytes / 2) +/** Build the bounded head/tail preview for `text`, splitting `budget` bytes across the two ends. */ +function preview(text: string, budget: number): { text: string; omitted: Omitted } { + const headBytes = Math.ceil(budget / 2) + const tailBytes = Math.floor(budget / 2) const retainer = new TextRetainer({ kind: 'headTail', headBytes, tailBytes }) retainer.push(text) const kept = retainer.finish() return { text: kept.text, omitted: kept.omittedBytes } } -/** - * Compose the replacement text: the bounded preview, a blank line, then the - * spill notice. The omission clause comes from the retention library - * (`describeOmitted`); the recovery sentence names the concrete spill path. - */ -function replacementText(previewText: string, omitted: Omitted, spillPath: string): string { +/** The spill-notice line for a given omission + path (no preview, no leading blank line). */ +function spillNotice(omitted: Omitted, spillPath: string): string { const omission = describeOmitted(omitted, 'bytes') - const notice = `(${omission} Full formatted result saved to: ${spillPath}. Use read with offset/limit to inspect it.)` - return `${previewText}\n\n${notice}` + return `(${omission} Full formatted result saved to: ${spillPath}. Use read with offset/limit to inspect it.)` } export function apply(ctx: Context, config: Config): void { @@ -119,7 +114,8 @@ export function apply(ctx: Context, config: Config): void { const content = decision.content ?? result.content const text = flattenPlainText(content) if (text === undefined) return decision - if (Buffer.byteLength(text, 'utf8') <= maxInlineBytes) return decision + const totalBytes = Buffer.byteLength(text, 'utf8') + if (totalBytes <= maxInlineBytes) return decision const sessionId = ownerSessionId(exec) if (sessionId === undefined) { @@ -148,8 +144,28 @@ export function apply(ctx: Context, config: Config): void { return decision } - const { text: previewText, omitted } = preview(text, maxInlineBytes) - const replaced: ContentBlock[] = [{ type: 'text', text: replacementText(previewText, omitted, path) }] + // Reserve the notice's byte cost INSIDE maxInlineBytes so the replacement + // (preview + blank line + notice) never exceeds the documented cap — a naive + // preview that spent the whole budget then appended the notice could be + // larger than the cap, and for a marginally-over result even larger than the + // original. The reservation uses a notice priced at the worst-case omission + // count (the full byte total): its digit count bounds the real count's, so + // the reserved size is a safe upper bound and the final notice is never + // longer than what we reserved. `\n\n` is the 2-byte join. + const reserve = Buffer.byteLength(spillNotice({ kind: 'exact', count: totalBytes }, path), 'utf8') + 2 + const previewBudget = Math.max(0, maxInlineBytes - reserve) + const { text: previewText, omitted } = preview(text, previewBudget) + const notice = spillNotice(omitted, path) + const replacedText = previewText.length > 0 ? `${previewText}\n\n${notice}` : notice + // Guard against a pathological tiny cap + long path where even the + // notice-only replacement is not smaller than the original: spilling then + // gains nothing and would only add bytes, so keep the inline result. (The + // spill file already written is a harmless orphan; cleanup is deferred.) + if (Buffer.byteLength(replacedText, 'utf8') >= totalBytes) { + ctx.logger.warn(`spill-policy: spill notice for ${exec.name} is not smaller than the result; keeping the inline result`) + return decision + } + const replaced: ContentBlock[] = [{ type: 'text', text: replacedText }] return { kind: 'accept', content: replaced, ...decision.additionalContext ? { additionalContext: decision.additionalContext } : {} } }) } diff --git a/packages/spill/spill-policy/tests/spill-policy.spec.ts b/packages/spill/spill-policy/tests/spill-policy.spec.ts index 61da0abe76..dd6235a9a3 100644 --- a/packages/spill/spill-policy/tests/spill-policy.spec.ts +++ b/packages/spill/spill-policy/tests/spill-policy.spec.ts @@ -93,9 +93,9 @@ describe('config validation', () => { }) describe('oversized plain-text replacement', () => { - it('spills the full text and replaces the result with a preview + path', async () => { - const { ctx, spill } = await setup({ maxInlineBytes: 20 }) - const body = 'HEAD'.repeat(20) + 'TAIL'.repeat(20) // 160 bytes > 20 + it('spills the full text and replaces the result with a preview + path within the cap', async () => { + const { ctx, spill } = await setup({ maxInlineBytes: 200 }) + const body = 'HEAD'.repeat(200) + 'TAIL'.repeat(200) // 1600 bytes > 200 ctx.tools.register(textTool('big', body)) const result = await ctx.tools.execute(exec('big')) @@ -112,6 +112,22 @@ describe('oversized plain-text replacement', () => { expect(text).toContain('Full formatted result saved to: /spill/big.txt') expect(text).toContain('Use read with offset/limit') expect(text).toContain('Omitted') + // The replacement (preview + blank line + notice) stays within the cap and + // is smaller than the original — the whole point of spilling. + expect(Buffer.byteLength(text, 'utf8')).toBeLessThanOrEqual(200) + expect(Buffer.byteLength(text, 'utf8')).toBeLessThan(body.length) + }) + + it('keeps the inline result when even the notice-only replacement is not smaller', async () => { + // A body just over a tiny cap: the notice alone is larger than the result, + // so spilling would only add bytes — the policy keeps the inline result. + const { ctx } = await setup({ maxInlineBytes: 4 }) + const warn = vi.spyOn(ctx.logger, 'warn').mockImplementation(() => {}) + const body = 'xxxxx' // 5 bytes > 4, but far shorter than the notice + ctx.tools.register(textTool('big', body)) + const result = await ctx.tools.execute(exec('big')) + expect(textOf(result.content)).toBe(body) + expect(warn).toHaveBeenCalled() }) it('leaves a small plain-text result unchanged', async () => { diff --git a/packages/util/retention/src/index.ts b/packages/util/retention/src/index.ts index f0f40ba7af..8c3b924a16 100644 --- a/packages/util/retention/src/index.ts +++ b/packages/util/retention/src/index.ts @@ -355,6 +355,19 @@ export class TextRetainer { this.suffixHeld -= head.length head = this.suffixChunks[0] } + // The head chunk can still hold leading bytes beyond the last `suffixCap` + // — a single chunk LARGER than the window is retained whole by the loop + // above (dropping the only chunk would leave < cap). Trim those leading + // bytes so the accumulator (and finish()'s concat) stays bounded by + // `suffixCap` instead of allocating/copying the full chunk again; + // finish() only ever reads the last `suffixLen ≤ suffixCap` bytes, so this + // drops nothing it would return. (head.length > excess by the loop + // invariant `suffixHeld - head.length < suffixCap`, so the slice is non-empty.) + if (head !== undefined && this.suffixHeld > this.suffixCap) { + const excess = this.suffixHeld - this.suffixCap + this.suffixChunks[0] = head.subarray(excess) + this.suffixHeld -= excess + } } // Dropped = bytes that no side can keep. Compute cumulative omission the diff --git a/packages/web/tool-web/tests/spill.spec.ts b/packages/web/tool-web/tests/spill.spec.ts index 6e2ce6d11d..e44cbb7b45 100644 --- a/packages/web/tool-web/tests/spill.spec.ts +++ b/packages/web/tool-web/tests/spill.spec.ts @@ -33,7 +33,8 @@ let handler: Handler let spillRoot: string let ctx: Context -const BODY = 'X'.repeat(4000) // formatted result is well over the 200-byte policy cap +const BODY = 'X'.repeat(4000) // formatted result is well over the policy cap +const MAX_INLINE_BYTES = 1000 // leaves room for a head/tail preview beside the notice beforeEach(async () => { handler = (_req, res) => { res.writeHead(200, { 'content-type': 'text/plain' }); res.end(BODY) } @@ -50,7 +51,7 @@ beforeEach(async () => { // policy cap is what triggers the spill (the RFC's separation of concerns). await ctx.plugin(WebFetchLocal, { maxBodyChars: 500_000 }) await ctx.plugin(LocalSpillFiles, { root: spillRoot }) - await ctx.plugin(SpillPolicy, { maxInlineBytes: 200 }) + await ctx.plugin(SpillPolicy, { maxInlineBytes: MAX_INLINE_BYTES }) await ctx.plugin(ToolWeb) }) @@ -72,8 +73,9 @@ describe('web_fetch spill showcase', () => { expect(out.isError).toBe(false) const text = out.content.map(b => b.text).join('') - // Model-facing text is a preview + notice, NOT the full body. + // Model-facing text is a preview + notice within the cap, NOT the full body. expect(text.length).toBeLessThan(BODY.length) + expect(Buffer.byteLength(text, 'utf8')).toBeLessThanOrEqual(MAX_INLINE_BYTES) expect(text).toContain(`Fetched ${base}`) // the head of the formatted result survives expect(text).toContain('Full formatted result saved to:') expect(text).toContain('Use read with offset/limit') From c9310d2a19c7f8129f8014fc927329c35a35906c Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Thu, 9 Jul 2026 10:07:07 +0800 Subject: [PATCH 09/24] fix: address codex review round 3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - spill-policy enforces the true cap invariant: it never emits a replacement larger than maxInlineBytes. When the notice alone exceeds the cap (tiny cap or long spill root) there is no within-cap replacement, so the inline result is kept — the previous guard only compared against the original size and could still return content over the cap for a large original. A within-cap replacement is always smaller than the original, so this subsumes the earlier check. - Add the HMR-disposal test the conventions require for a new registration: dispose the plugin fiber and assert oversized results stop being transformed and nothing more is spilled (no leaked tools/post-execute listener on reload). --- packages/spill/spill-policy/README.md | 2 +- packages/spill/spill-policy/src/index.ts | 15 +++--- .../spill-policy/tests/spill-policy.spec.ts | 51 ++++++++++++++++--- 3 files changed, 53 insertions(+), 15 deletions(-) diff --git a/packages/spill/spill-policy/README.md b/packages/spill/spill-policy/README.md index dcc2fffb30..fe128de9a0 100644 --- a/packages/spill/spill-policy/README.md +++ b/packages/spill/spill-policy/README.md @@ -24,7 +24,7 @@ This plugin registers **no service** and owns no storage or preview mechanics: p (Omitted N bytes. Full formatted result saved to: /…/session-…/…-web_fetch.txt. Use read with offset/limit to inspect it.) ``` - When the notice alone fills the budget (a tiny cap or a long path) the preview is empty and only the notice is returned. If even that notice-only replacement is not smaller than the original result, the policy keeps the inline result — spilling would only add bytes. + When the notice alone fills the budget (a tiny cap or a long path) the preview is empty and only the notice is returned. If even that notice-only replacement would exceed `maxInlineBytes`, the policy keeps the inline result — it never emits a replacement over the cap (and a within-cap replacement is always smaller than the original, so this also means spilling never adds bytes). **Best-effort:** no session owner, no `ctx.spillFiles` backend, or a `saveText` rejection ⇒ the policy logs a warning and returns the original result. A spill failure never turns a successful call into an `isError` or hides the inline result. diff --git a/packages/spill/spill-policy/src/index.ts b/packages/spill/spill-policy/src/index.ts index 9fa55fcdf4..0472bd9a8a 100644 --- a/packages/spill/spill-policy/src/index.ts +++ b/packages/spill/spill-policy/src/index.ts @@ -157,12 +157,15 @@ export function apply(ctx: Context, config: Config): void { const { text: previewText, omitted } = preview(text, previewBudget) const notice = spillNotice(omitted, path) const replacedText = previewText.length > 0 ? `${previewText}\n\n${notice}` : notice - // Guard against a pathological tiny cap + long path where even the - // notice-only replacement is not smaller than the original: spilling then - // gains nothing and would only add bytes, so keep the inline result. (The - // spill file already written is a harmless orphan; cleanup is deferred.) - if (Buffer.byteLength(replacedText, 'utf8') >= totalBytes) { - ctx.logger.warn(`spill-policy: spill notice for ${exec.name} is not smaller than the result; keeping the inline result`) + // Invariant: the policy NEVER emits a replacement larger than the cap. When + // the notice alone exceeds maxInlineBytes (a tiny cap or a long spill root), + // there is no within-cap replacement, so keep the inline result — spilling + // would break the advertised context cap. (A within-cap replacement is + // always smaller than the original, which is > cap by the entry condition, + // so this one check subsumes "not smaller than the original" too. The spill + // file already written is a harmless orphan; cleanup is deferred.) + if (Buffer.byteLength(replacedText, 'utf8') > maxInlineBytes) { + ctx.logger.warn(`spill-policy: spill notice for ${exec.name} exceeds maxInlineBytes; keeping the inline result`) return decision } const replaced: ContentBlock[] = [{ type: 'text', text: replacedText }] diff --git a/packages/spill/spill-policy/tests/spill-policy.spec.ts b/packages/spill/spill-policy/tests/spill-policy.spec.ts index dd6235a9a3..b137a73144 100644 --- a/packages/spill/spill-policy/tests/spill-policy.spec.ts +++ b/packages/spill/spill-policy/tests/spill-policy.spec.ts @@ -53,7 +53,7 @@ function exec(name: string, session = 's1'): ToolExecution { * Build a context with tools + the policy, and optionally a spill backend. * Returns the context and the backend handle (undefined when `withSpill` false). */ -async function setup(config: SpillPolicy.Config, withSpill = true): Promise<{ ctx: Context; spill?: StubSpill }> { +async function setup(config: SpillPolicy.Config, withSpill = true): Promise<{ ctx: Context; spill?: StubSpill; fiber: Awaited> }> { const ctx = new Context() await ctx.plugin(SystemPrompt) await ctx.plugin(ToolRegistry) @@ -62,8 +62,8 @@ async function setup(config: SpillPolicy.Config, withSpill = true): Promise<{ ct await ctx.plugin(StubSpill) spill = ctx.spillFiles as StubSpill } - await ctx.plugin(SpillPolicy, config) - return { ctx, ...spill ? { spill } : {} } + const fiber = await ctx.plugin(SpillPolicy, config) + return { ctx, fiber, ...spill ? { spill } : {} } } /** Flatten a result's text blocks. */ @@ -118,9 +118,9 @@ describe('oversized plain-text replacement', () => { expect(Buffer.byteLength(text, 'utf8')).toBeLessThan(body.length) }) - it('keeps the inline result when even the notice-only replacement is not smaller', async () => { - // A body just over a tiny cap: the notice alone is larger than the result, - // so spilling would only add bytes — the policy keeps the inline result. + it('keeps the inline result when the notice-only replacement would exceed the cap', async () => { + // A body just over a tiny cap: the notice alone is larger than the cap, so + // there is no within-cap replacement — the policy keeps the inline result. const { ctx } = await setup({ maxInlineBytes: 4 }) const warn = vi.spyOn(ctx.logger, 'warn').mockImplementation(() => {}) const body = 'xxxxx' // 5 bytes > 4, but far shorter than the notice @@ -198,7 +198,7 @@ describe('best-effort fallback', () => { describe('composition', () => { it('bounds content a downstream post-execute listener replaced', async () => { - const { ctx, spill } = await setup({ maxInlineBytes: 10 }) + const { ctx, spill } = await setup({ maxInlineBytes: 200 }) // A later-registered listener replaces the (small) tool result with a big one; // the policy delegated via next(), so it bounds the replacement. ctx.on('tools/post-execute', async (_e, _r, _next) => @@ -210,7 +210,7 @@ describe('composition', () => { }) it('preserves a downstream accept decision additionalContext when spilling', async () => { - const { ctx } = await setup({ maxInlineBytes: 10 }) + const { ctx } = await setup({ maxInlineBytes: 200 }) const context = { content: [{ type: 'text' as const, text: 'note' }], source: { kind: 'plugin' as const, plugin: 'test' } } ctx.on('tools/post-execute', async (_e, _r, _next) => ({ kind: 'accept', additionalContext: context })) @@ -220,3 +220,38 @@ describe('composition', () => { expect(result.additionalContext).toEqual(context) }) }) + +describe('cap invariant', () => { + it('keeps the inline result when the notice alone exceeds the cap, even for a large original', async () => { + // A large body (so it is well over the cap) but a cap smaller than the + // notice itself: there is no within-cap replacement, so the policy must keep + // the inline result rather than emit content over maxInlineBytes. + const { ctx } = await setup({ maxInlineBytes: 8 }) + const warn = vi.spyOn(ctx.logger, 'warn').mockImplementation(() => {}) + const body = 'x'.repeat(5000) + ctx.tools.register(textTool('big', body)) + const result = await ctx.tools.execute(exec('big')) + expect(textOf(result.content)).toBe(body) + expect(warn).toHaveBeenCalled() + }) +}) + +describe('disposal (HMR safety)', () => { + it('stops transforming oversized results after the plugin fiber is disposed', async () => { + const { ctx, spill, fiber } = await setup({ maxInlineBytes: 200 }) + const body = 'HEAD'.repeat(200) + 'TAIL'.repeat(200) + ctx.tools.register(textTool('big', body)) + + // Live: the listener spills and replaces. + const before = await ctx.tools.execute(exec('big')) + expect(textOf(before.content)).toContain('Full formatted result saved to') + expect(spill?.saves).toHaveLength(1) + + // After disposal the listener is gone — the result passes through untouched + // and nothing more is spilled (no leaked registration across reload). + await fiber.dispose() + const after = await ctx.tools.execute(exec('big')) + expect(textOf(after.content)).toBe(body) + expect(spill?.saves).toHaveLength(1) + }) +}) From fef4313685fcd384b7c5eed7a4628b087e95dd14 Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Thu, 9 Jul 2026 11:04:34 +0800 Subject: [PATCH 10/24] test(spill-policy): guard loader export shape --- .../spill/spill-policy/tests/spill-policy.spec.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/packages/spill/spill-policy/tests/spill-policy.spec.ts b/packages/spill/spill-policy/tests/spill-policy.spec.ts index b137a73144..b0678c27c2 100644 --- a/packages/spill/spill-policy/tests/spill-policy.spec.ts +++ b/packages/spill/spill-policy/tests/spill-policy.spec.ts @@ -10,6 +10,7 @@ import { describe, expect, it, vi } from 'vitest' import { Context } from 'cordis' +import Loader from '@cordisjs/plugin-loader' import { CallId } from '@deepseek-ai/dsh-llm' import type { ContentBlock } from '@deepseek-ai/dsh-llm' import { SessionId } from '@deepseek-ai/dsh-session' @@ -82,6 +83,20 @@ describe('disabled mode', () => { }) }) +describe('loader export shape', () => { + it('has no default export and keeps name/inject/Config through unwrapExports', () => { + expect('default' in SpillPolicy).toBe(false) + + const loader = Object.create(Loader.prototype) as Loader + const unwrapped = loader.unwrapExports(SpillPolicy) as Record + expect(unwrapped).toBe(SpillPolicy) + expect(unwrapped.name).toBe('spill-policy') + expect(unwrapped.inject).toEqual(['tools']) + expect(unwrapped.Config).toBeDefined() + expect(typeof unwrapped.apply).toBe('function') + }) +}) + describe('config validation', () => { it('rejects a negative maxInlineBytes at load', async () => { await expect(setup({ maxInlineBytes: -1 })).rejects.toThrow(/non-negative integer/) From 571c6025d56f827c0aeed55e0c9b8a9fa9f974ff Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Thu, 9 Jul 2026 13:01:14 +0800 Subject: [PATCH 11/24] fix: address master merge gate failures --- docs/config-catalog.md | 36 +++++++++++++++++++++++++ packages/README.md | 12 ++++----- packages/spill/spill-local/src/store.ts | 16 ++++++++++- packages/spill/spill/src/types.ts | 7 ++++- packages/util/timeout/README.md | 17 ++++++++---- 5 files changed, 75 insertions(+), 13 deletions(-) diff --git a/docs/config-catalog.md b/docs/config-catalog.md index 2429fe3163..c4a982e12a 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -474,6 +474,40 @@ export type JournalMode = 'wal' | 'delete' | 'truncate' | 'persist' Source: [`packages/session-persistence/session-persistence-sqlite/src/index.ts:50`](../packages/session-persistence/session-persistence-sqlite/src/index.ts) +## `@deepseek-ai/dsh-spill-local` + +```ts config-catalog +/** Plugin config (all optional — `static Config` supplies the defaults). */ +export interface Config { + /** + * Root directory for spill files. Omitted uses a lazily-created private + * (0700) per-process directory under the OS temp dir — the safe default for + * a local deployment. Set it to keep spill files under a known location. + */ + root?: string +} +``` + +Source: [`packages/spill/spill-local/src/index.ts:22`](../packages/spill/spill-local/src/index.ts) + +## `@deepseek-ai/dsh-spill-policy` + +Requires: `tools` + +```ts config-catalog +/** Plugin config. */ +export interface Config { + /** + * The model-facing context cap for a plain-text tool result, in UTF-8 bytes. + * Omitted disables the policy entirely (no-op). When set, a result larger than + * this is spilled and replaced with a preview derived from this same budget. + */ + maxInlineBytes?: number +} +``` + +Source: [`packages/spill/spill-policy/src/index.ts:45`](../packages/spill/spill-policy/src/index.ts) + ## `@deepseek-ai/dsh-stdio-agent` ```ts config-catalog @@ -879,6 +913,7 @@ Abstract service classes — a deployment loads a concrete implementation packag - `@deepseek-ai/dsh-compact` — abstract `CompactService` ([`packages/compact/compact/src/index.ts`](../packages/compact/compact/src/index.ts)) - `@deepseek-ai/dsh-fs` — abstract `FileSystem` ([`packages/fs/fs/src/index.ts`](../packages/fs/fs/src/index.ts)) - `@deepseek-ai/dsh-session-persistence` — abstract `SessionPersistence` ([`packages/session-persistence/session-persistence/src/index.ts`](../packages/session-persistence/session-persistence/src/index.ts)) +- `@deepseek-ai/dsh-spill` — abstract `SpillFiles` ([`packages/spill/spill/src/index.ts`](../packages/spill/spill/src/index.ts)) ## Library packages (no plugin entry) @@ -888,5 +923,6 @@ Imported as libraries by other packages; a `cordis.yml` cannot load them. - `@deepseek-ai/dsh-app-boot` ([`packages/ui/app-boot/src/index.ts`](../packages/ui/app-boot/src/index.ts)) - `@deepseek-ai/dsh-brand` ([`packages/util/brand/src/index.ts`](../packages/util/brand/src/index.ts)) - `@deepseek-ai/dsh-hook-protocol` ([`packages/hooks/hook-protocol/src/index.ts`](../packages/hooks/hook-protocol/src/index.ts)) +- `@deepseek-ai/dsh-retention` ([`packages/util/retention/src/index.ts`](../packages/util/retention/src/index.ts)) - `@deepseek-ai/dsh-subagent-inprocess` ([`packages/subagent/subagent-inprocess/src/index.ts`](../packages/subagent/subagent-inprocess/src/index.ts)) - `@deepseek-ai/dsh-timeout` ([`packages/util/timeout/src/index.ts`](../packages/util/timeout/src/index.ts)) diff --git a/packages/README.md b/packages/README.md index 98d3b22719..7060760311 100644 --- a/packages/README.md +++ b/packages/README.md @@ -1,10 +1,10 @@ # Packages -Harness packages, all under the `@deepseek-ai/dsh-*` scope. Each package is a Cordis plugin (microkernel-style): it exports either a default `Service` subclass or a functional plugin, declares its ctx key/events through declaration merging, and exposes extension points through `ctx.effect()`, `ctx.on()`, and `ctx.waterfall()`. Authoring conventions: [AGENTS.md](AGENTS.md) (subtree) and the root [AGENTS.md](../AGENTS.md) § Conventions. +Harness packages live under the `@deepseek-ai/dsh-*` scope. Each is a Cordis plugin: it exports a `Service` subclass or functional plugin, declares ctx keys/events through declaration merging, and extends through `ctx.effect()`, `ctx.on()`, and `ctx.waterfall()`. Authoring conventions: [AGENTS.md](AGENTS.md) and root [AGENTS.md](../AGENTS.md) § Conventions. ## Hierarchy -Packages are grouped by modular role at `packages///`. The group directory is a pure container (no `package.json` of its own); the package name stays `@deepseek-ai/dsh-` regardless of group. **Each group README is the canonical per-package map** — package roles, ctx keys, and the product-vs-support split live there, next to the code. +Packages are grouped by role at `packages///`. The group directory is a pure container; package names stay `@deepseek-ai/dsh-`. Group READMEs are the canonical maps for package roles, ctx keys, and product-vs-support split. | Group | Role | Release expectation | |---|---|---| @@ -17,7 +17,7 @@ Packages are grouped by modular role at `packages///`. The group dir | [`subagent/`](subagent/README.md) | Subagent capability family: the provider-registry seam and the model-facing delegation tool | Product — stable surface | | [`web/`](web/README.md) | Web capability family: the abstract seam, search/fetch provider impls, and the model-facing web tools | Product — stable surface | | [`spill/`](spill/README.md) | Spill capability family: the storage seam, a local impl, and the tool-result spill policy | Product — stable surface | -| [`todo/`](todo/README.md) | Todo/planning family: the model-facing `todo_write` tool (whole-list task tracking on the session log) | Product — stable surface | +| [`todo/`](todo/README.md) | Todo/planning family: the model-facing `todo_write` tool | Product — stable surface | | [`timeout/`](timeout/README.md) | Tool-call timeout policy: the `tools/execute` deadline enforcer | Product — stable surface | | [`guard/`](guard/README.md) | Loop-hygiene guards: advisory repeat-call reminders | Product — stable surface | | [`hooks/`](hooks/README.md) | Hook bridges + the shared Claude Code / Codex wire-protocol library | Product — stable surface | @@ -26,12 +26,12 @@ Packages are grouped by modular role at `packages///`. The group dir | [`support/`](support/README.md) | Dev/test/example infrastructure (invariants, replay adapter, subagent mock) | Support — lower compatibility expectations | | [`util/`](util/README.md) | Low-level zero-dependency primitives shared across groups (branding, timeout, retention) | Support — small, stable, harness-dep-free | -The split is the point: a package's group says whether it is part of the product API or support/test/example infrastructure, so release and removal decisions do not treat every package as an equal public contract. New packages join an existing group; adding a new top-level group is a deliberate act (extend the group READMEs and this table). +The split marks product API versus support/test/example infrastructure, so release and removal decisions do not treat every package as equally public. New packages join an existing group; a new top-level group updates the group READMEs and this table. ## Dependencies -The inter-package dependency graph is generated: [docs/module-graph.md](../docs/module-graph.md) (`pnpm run gen-module-graph`, freshness-gated in CI). +The dependency graph is generated: [docs/module-graph.md](../docs/module-graph.md) (`pnpm run gen-module-graph`, freshness-gated in CI). -The rule it must obey: **extension plugins depend on interfaces, never on the concrete loop.** `dsh-agent-loop` is swappable — UI/hook/tool plugins keep working against the `dsh-agent` vocabulary if the loop is replaced. The sanctioned exception is a **composition/bundle** package like `dsh-agent-core`, whose whole job is to assemble the concrete spine: it depends on `dsh-agent-loop` (and the other concrete spine plugins) on purpose. The rule constrains plugins that EXTEND the system, not the bundle that COMPOSES it. A swappable capability splits into interface / implementation / consumer packages (the bash trio is the template — see [capability seams](../docs/rfc/implemented/architecture/2026-06-13-capability-seams.md)). +The rule it must obey: **extension plugins depend on interfaces, never on the concrete loop.** `dsh-agent-loop` is swappable, so UI/hook/tool plugins keep working against `dsh-agent` if the loop changes. The exception is a composition bundle like `dsh-agent-core`: it depends on `dsh-agent-loop` because it assembles the concrete spine. Swappable capabilities split into interface / implementation / consumer packages (the bash trio is the template — see [capability seams](../docs/rfc/implemented/architecture/2026-06-13-capability-seams.md)). Each package has its own `README.md` with purpose, service API, events, extension points, and deliberate non-goals (TODOs). diff --git a/packages/spill/spill-local/src/store.ts b/packages/spill/spill-local/src/store.ts index adf538e740..44e4ee7129 100644 --- a/packages/spill/spill-local/src/store.ts +++ b/packages/spill/spill-local/src/store.ts @@ -21,6 +21,8 @@ let defaultRoot: string | undefined * tmpdir, created lazily. Predictable world-readable paths would let other * local users read spilled tool output or pre-create symlinks; `mkdtemp` gives * an unpredictable suffix and 0700 semantics. + * + * @returns The lazily-created private spill root. */ export function privateRoot(): string { defaultRoot ??= mkdtempSync(join(tmpdir(), 'dsh-spill-')) @@ -36,6 +38,9 @@ export function privateRoot(): string { * inputs never collide. The whole-segment tokens `.`/`..` are escaped so they * can never traverse. An empty string encodes to `~` (never an empty segment). * (Mirrors the JSONL persistence backend's `encodeSegment`.) + * + * @param raw The untrusted string to encode as one safe path segment. + * @returns An injective, filesystem-safe single path segment. */ export function encodeSegment(raw: string): string { if (raw.length === 0) return '~' @@ -54,7 +59,13 @@ export function encodeSegment(raw: string): string { return out } -/** The session-scoped directory: `/session-`, a short stable hash. */ +/** + * The session-scoped directory: `/session-`, a short stable hash. + * + * @param root The spill root directory. + * @param sessionId The owning session id to hash into a stable directory name. + * @returns The absolute session-scoped spill directory path. + */ export function sessionDir(root: string, sessionId: string): string { const hash = createHash('sha256').update(sessionId).digest('hex').slice(0, 12) return join(root, `session-${hash}`) @@ -85,6 +96,9 @@ export interface SavedText { * a shared root) AND stays readable. The open is exclusive + owner-only * (`'wx', 0o600`): it fails on any existing path — symlink or not — so a * pre-planted target cannot redirect the write. + * + * @param options The resolved root and request fields required to save the file. + * @returns The written file path and UTF-8 byte length. */ export async function saveTextFile(options: SaveTextOptions): Promise { const dir = sessionDir(options.root, options.sessionId) diff --git a/packages/spill/spill/src/types.ts b/packages/spill/spill/src/types.ts index 8dfd4c1d1d..28be96c738 100644 --- a/packages/spill/spill/src/types.ts +++ b/packages/spill/spill/src/types.ts @@ -19,7 +19,12 @@ import type { SessionId } from '@deepseek-ai/dsh-session' */ export type SpillPath = Branded<'SpillPath'> -/** Brand a string as a {@link SpillPath}. */ +/** + * Brand a string as a {@link SpillPath}. + * + * @param path The backend-produced path string to brand. + * @returns The branded spill path. + */ export function SpillPath(path: string): SpillPath { return path as SpillPath } diff --git a/packages/util/timeout/README.md b/packages/util/timeout/README.md index db2b06ba53..a15a552847 100644 --- a/packages/util/timeout/README.md +++ b/packages/util/timeout/README.md @@ -25,12 +25,19 @@ import { clampTimeout, deadline, timeoutOf, TimeoutReason } from '@deepseek-ai/d ## Usage shape -```ts ignore-check +```ts +import { deadline, timeoutOf } from '@deepseek-ai/dsh-timeout' + +declare function runWork(options: { signal: AbortSignal }): Promise + // Scope-lifetime consumer (foreground bash, one fetch): `using` disposes the timer. -using d = deadline(upstream, timeoutMs, 'BASH_TIMEOUT') -const outcome = await runWork({ signal: d.signal }) // work listens on d.signal and terminates itself -const timedOut = timeoutOf(d.signal, 'BASH_TIMEOUT') !== undefined // classify the first abort, scoped to OUR code -const aborted = d.signal.aborted && !timedOut // mutually exclusive: timeout won, or cancel did +export async function runWithDeadline(upstream: AbortSignal | undefined, timeoutMs: number): Promise { + using d = deadline(upstream, timeoutMs, 'BASH_TIMEOUT') + const outcome = await runWork({ signal: d.signal }) // work listens on d.signal and terminates itself + const timedOut = timeoutOf(d.signal, 'BASH_TIMEOUT') !== undefined // classify the first abort, scoped to OUR code + const aborted = d.signal.aborted && !timedOut // mutually exclusive: timeout won, or cancel did + return { outcome, timedOut, aborted } +} ``` The signal only *notifies* — the caller MUST attach its own termination (`d.signal.addEventListener('abort', kill)`, or hand `d.signal` to `fetch`). Racing a promise against a timer would resolve the tool-call while the child process or socket leaks on; handing out a signal forces a real termination path to exist. From 0d5b15b5b46a1c6ade75c47e6dba6d7fc6239cad Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Thu, 9 Jul 2026 13:44:38 +0800 Subject: [PATCH 12/24] test: cover spill plugins in built stdio consumer --- .../ui/stdio-agent/tests/built-bin.e2e.ts | 47 +++++++++++++++++-- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/packages/ui/stdio-agent/tests/built-bin.e2e.ts b/packages/ui/stdio-agent/tests/built-bin.e2e.ts index 7b84fad65b..befa509713 100644 --- a/packages/ui/stdio-agent/tests/built-bin.e2e.ts +++ b/packages/ui/stdio-agent/tests/built-bin.e2e.ts @@ -49,6 +49,14 @@ async function pkgName(absDir: string): Promise { return json.name } +async function installWorkspacePackageCopy(absDir: string, target: string): Promise { + await mkdir(dirname(target), { recursive: true }) + await cp(absDir, target, { + recursive: true, + filter: source => !source.split('/').includes('node_modules'), + }) +} + /** * Build a temp consumer dir: `node_modules` with the workspace + vendor packages * symlinked in, a `src/` carrying the example mock backend, and a `cordis.yml` @@ -59,15 +67,24 @@ async function pkgName(absDir: string): Promise { * design, so it exercises that the fail-loud entry-load guard does NOT mistake a * valid disabled entry for a failed import. */ -async function makeConsumer(welcome: string, disabledBrokenEntry = false): Promise { +async function makeConsumer( + welcome: string, + disabledBrokenEntry = false, + extraDshPackages: string[] = [], + extraEntries: string[] = [], +): Promise { const dir = await mkdtemp(join(tmpdir(), 'stdio-built-bin-')) const nm = join(dir, 'node_modules') - for (const rel of dshPackages) { + for (const rel of [...dshPackages, ...extraDshPackages]) { const abs = join(repoRoot, 'packages', rel) const name = await pkgName(abs) const target = join(nm, name) - await mkdir(dirname(target), { recursive: true }) - await symlink(abs, target) + if (extraDshPackages.includes(rel)) { + await installWorkspacePackageCopy(abs, target) + } else { + await mkdir(dirname(target), { recursive: true }) + await symlink(abs, target) + } } for (const v of vendorPackages) { const abs = join(repoRoot, 'vendor', v) @@ -94,6 +111,7 @@ async function makeConsumer(welcome: string, disabledBrokenEntry = false): Promi ' model: mock-echo', ' systemPrompt: \'demo\'', ` welcome: '${welcome}'`, + ...extraEntries, ...disabledBrokenEntry ? ['- id: off', ' name: \'./src/does-not-exist.ts\'', ' disabled: true'] : [], @@ -164,6 +182,27 @@ describe.skipIf(!existsSync(stdioBin))('dsh-stdio-agent BUILT bin (node lib/bin. expect(code).toBe(0) }, 30_000) + it('boots when optional spill plugins are loaded from a built consumer install', async () => { + consumer = await makeConsumer( + 'SPILL-OK ready.', + false, + ['spill/spill', 'spill/spill-local', 'spill/spill-policy', 'util/retention'], + [ + '- id: spill-local', + ' name: \'@deepseek-ai/dsh-spill-local\'', + '- id: spill-policy', + ' name: \'@deepseek-ai/dsh-spill-policy\'', + ' config:', + ' maxInlineBytes: 50000', + ], + ) + const { stdout, code, stderr } = await runBuiltBin(consumer, './cordis.yml', '') + expect(stderr).not.toContain('failed to load') + expect(stderr).not.toContain('Cannot find package') + expect(stdout).toContain('SPILL-OK ready.') + expect(code).toBe(0) + }, 30_000) + it('fails LOUD (non-zero exit + stderr) on a config whose directory does not exist', async () => { // A consumer who typos the config path must get a clear failure, not silent // success. This dir does not exist, so the include PLUGIN itself fails to From a4a9900be1c6f597a92a5955fe2768ca40314756 Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Thu, 9 Jul 2026 13:53:51 +0800 Subject: [PATCH 13/24] simplify retention omitted metadata --- ...026-07-06-tool-result-retention-library.md | 49 +++----- .../2026-07-08-tool-output-spill-files.md | 6 +- packages/util/README.md | 2 +- packages/util/retention/README.md | 49 ++++---- packages/util/retention/package.json | 2 +- packages/util/retention/src/index.ts | 104 ++++------------ .../util/retention/tests/retention.spec.ts | 111 ++++++++---------- 7 files changed, 119 insertions(+), 204 deletions(-) diff --git a/docs/rfc/implemented/architecture/2026-07-06-tool-result-retention-library.md b/docs/rfc/implemented/architecture/2026-07-06-tool-result-retention-library.md index 7ea954b0a2..344b50753a 100644 --- a/docs/rfc/implemented/architecture/2026-07-06-tool-result-retention-library.md +++ b/docs/rfc/implemented/architecture/2026-07-06-tool-result-retention-library.md @@ -4,9 +4,9 @@ Status: implemented ## Problem -Several model-facing tools already bound the amount of context they return, but each one owns a different local mechanism and vocabulary: bash keeps a tail plus spill files, web search caps source lists, web fetch caps body content, and `glob` / `grep` discovery needs `cap + 1` early stop while reading ripgrep output. A single post-hoc `truncate(text)` helper cannot cover those cases: by the time `grep` or `glob` has collected every result, the expensive traversal has already happened and the process may have emitted more output than the harness intended to buffer. +Several model-facing tools already bound the amount of context they return, but each one owns a different local mechanism and vocabulary: bash keeps a tail plus spill files, web search caps source lists, web fetch caps body content, and `glob` / `grep` discovery needs an inline first page while keeping exact omission metadata for the full result set. A single `truncate(text)` helper cannot cover those cases: item tools need item counts and grouping outside the primitive, while text tools need byte budgets and UTF-8-safe head/tail cuts. -The shared abstraction the tools need is **retention**, not generic collection. A caller feeds items or text chunks into a bounded object, receives a per-push decision about whether the upstream can stop, and later receives the retained content plus exact or partial omission metadata. Tool-specific code still owns business semantics: file grouping, line numbering, exit codes, provider error states, and model-facing prose. The common library owns only the mechanical question "what did we keep, what did we omit, and may the caller stop reading now?" +The shared abstraction the tools need is **retention**, not generic collection. A caller feeds items or text chunks into a bounded object and later receives the retained content plus exact omission metadata. Tool-specific code still owns business semantics: file grouping, line numbering, exit codes, provider error states, spill files, and model-facing prose. The common library owns only the mechanical question "what did we keep, and what did we omit?" ## Decision @@ -17,38 +17,27 @@ The library has two independent retainers: - `ItemRetainer` handles ordered logical units such as paths, grep matches, or search sources. It supports `head` retention only in v1. - `TextRetainer` handles byte-oriented text streams such as bash stdout/stderr or web response bodies. It supports `head`, `tail`, and `headTail` retention while preserving UTF-8 boundaries at `finish()`. -Both retainers return a `PushDecision` after each `push()`. `shouldStop` is the critical control-flow field: `glob` / `grep` use it to kill ripgrep once the probe item proves truncation, while bash ignores it because tail/head-tail retention must read to process exit to know the true suffix and to avoid pipe backpressure. +Both retainers return a small `PushDecision` after each `push()` so callers can tell whether that unit/chunk was fully retained and whether the accumulated result is now truncated. Omission counts are exact because callers keep feeding every observed item/chunk. ```ts ignore-check /** * How much content the retainer omitted. * - * `atLeast` is the early-stop shape: `glob` / `grep` see the first item past the cap, - * stop the upstream process, and know only that at least one item was omitted. + * `unknown` is reserved for callers that omit without a count; the retainers + * themselves return `none` or `exact`. */ type Omitted = | { kind: 'none' } | { kind: 'exact'; count: number } - | { kind: 'atLeast'; count: number } | { kind: 'unknown' } -/** - * The caller receives this after each `push()`. - * - * `shouldStop` is advisory, not automatic: the tool owns how to stop its upstream - * source, such as aborting an HTTP body, breaking a file scan, or killing ripgrep. - */ interface PushDecision { kept: boolean truncated: boolean - shouldStop: boolean } /** * Final result for ordered logical units. - * - * `seen` means units observed by the retainer, not necessarily total units in the - * upstream source; with early stop, total is intentionally unknown. */ interface RetainedItems { items: T[] @@ -73,25 +62,21 @@ interface RetainedText { ### Strategies -The strategy names are caller-facing and avoid implementation phrases such as "overflow". `stopWhenFull` means the retainer should ask the caller to stop once keeping more would exceed the budget. `readToEnd` means the retainer must keep accepting input even after the retained output is full, usually to preserve a true tail, count exact omission, or drain an upstream process. +Item retention supports a head window. Text retention supports head, tail, and headTail byte windows. ```ts ignore-check -type StopMode = 'stopWhenFull' | 'readToEnd' - type ItemRetentionStrategy = | { /** Keep the first `maxItems` units. Use for `glob`, `grep`, and web sources. */ kind: 'head' maxItems: number - stop: StopMode } type TextRetentionStrategy = | { - /** Keep the first `maxBytes` bytes. May stop an upstream body early. */ + /** Keep the first `maxBytes` bytes. */ kind: 'head' maxBytes: number - stop: StopMode } | { /** Keep the final `maxBytes` bytes. Requires reading to the end. */ @@ -112,15 +97,15 @@ type TextRetentionStrategy = `FsGlobEntry` and `FlatGrepMatch` below are the intended discovery-tool item shapes, not existing retention-library exports. `FsGlobEntry` is one backend-derived path, and `FlatGrepMatch` is one ungrouped grep match before the backend groups retained matches by file. -`glob` uses `ItemRetainer` with `{ kind: 'head', maxItems: globMaxResults, stop: 'stopWhenFull' }` inside the backend or executor that is consuming traversal output. The `(maxItems + 1)`th valid path is the probe item: it is not retained, it sets `truncated: true`, and `shouldStop: true` tells the caller to stop ripgrep, cancel a remote stream, or stop whatever upstream is producing candidates. `omitted` is `{ kind: 'atLeast', count: 1 }` because the traversal stopped before the full count was known. Path mapping, skipped candidates, and `incomplete` stay outside the retainer. +`glob` uses `ItemRetainer` with `{ kind: 'head', maxItems: globMaxResults }` after collecting the full sorted path list. The tool keeps the retained first page inline and may save the full list through the spill seam. Path mapping, skipped candidates, and `incomplete` stay outside the retainer. -`grep` uses `ItemRetainer` with `{ kind: 'head', maxItems: grepMaxMatches, stop: 'stopWhenFull' }` before grouping. The backend parses a ripgrep match record, maps the path, applies per-line preview truncation, then pushes a flat match. After `finish()`, the backend groups retained matches by file and sorts the returned subset. Grouping is not part of the retainer because the cap is total matches, not files; per-match preview truncation and `incomplete` are also separate from result-level retention. +`grep` uses `ItemRetainer` with `{ kind: 'head', maxItems: grepMaxMatches }` before grouping. The executor parses ripgrep output, maps paths, applies per-line preview truncation, and pushes flat matches. After `finish()`, the tool groups retained matches by file and can save the full match list through the spill seam when the inline result is capped. Grouping is not part of the retainer because the cap is total matches, not files; per-match preview truncation and `incomplete` are also separate from result-level retention. -`bash` uses `TextRetainer` with `tail` or `headTail` and reads to process completion. It does not stop when full: stopping the read would lose the real tail and can create pipe backpressure. The bash executor still owns spill files, exit status, signal, timeout, and background-task behavior; the retention helper only replaces ad hoc in-memory head/tail accounting where that behavior is desired. Long-running task ownership remains orthogonal to the [generic long-running tool runtime](../../proposed/architecture/2026-06-20-generic-long-running-tool-runtime.md) proposal. +`bash` can use `TextRetainer` with `tail` or `headTail` and reads to process completion. The bash executor still owns spill files, exit status, signal, timeout, and background-task behavior; the retention helper only replaces ad hoc in-memory head/tail accounting where that behavior is desired. Long-running task ownership remains orthogonal to the [generic long-running tool runtime](../../proposed/architecture/2026-06-20-generic-long-running-tool-runtime.md) proposal. -`web_fetch` can use `TextRetainer` with `head` when the provider exposes a stream, or keep provider-owned body caps when the provider must read and decode internally. Either way, the fetch result's `truncated` remains a provider/tool fact, and the library only supplies retained text and omission metadata. +`web_fetch` can use `TextRetainer` with `head` or `headTail`, or keep provider-owned body caps when the provider must read and decode internally. Either way, the fetch result's `truncated` remains a provider/tool fact, and the library only supplies retained text and omission metadata. -`web_search` can use `ItemRetainer` with `head`. Current providers often return an array, so this is post-hoc but still standardizes notices; a streaming provider can use the same strategy with `stopWhenFull`. +`web_search` can use `ItemRetainer` with `head`. Current providers often return an array, so this is post-hoc but still standardizes notices. ### Notices @@ -149,19 +134,19 @@ The formatter hook is deliberately small: a tool turns a `RetentionNotice` into ## Consequences -**What shipped.** `@deepseek-ai/dsh-retention` exports `ItemRetainer`, `TextRetainer`, the result types (`RetainedItems`, `RetainedText`), the strategy types (`ItemRetentionStrategy`, `TextRetentionStrategy`, `StopMode`), `Omitted`, `PushDecision`, `RetentionNotice`, and the neutral notice helpers `describeOmitted` / `formatRetentionNotice` — with no dependency on Cordis or any tool package. Unit tests cover item-head early stop with a probe item, item-head read-to-end with exact omission counts, text-head early stop, text-tail retention with exact omission counts, head-tail byte retention, zero budgets, UTF-8 boundary handling (2-, 3-, and 4-byte codepoints and invalid lead bytes at each cut), and the difference between `{ kind: 'atLeast', count: 1 }` and exact omission. +**What shipped.** `@deepseek-ai/dsh-retention` exports `ItemRetainer`, `TextRetainer`, the result types (`RetainedItems`, `RetainedText`), the strategy types (`ItemRetentionStrategy`, `TextRetentionStrategy`), `Omitted`, `PushDecision`, `RetentionNotice`, and the neutral notice helpers `describeOmitted` / `formatRetentionNotice` — with no dependency on Cordis or any tool package. Unit tests cover item-head retention with exact omission counts, text-head retention, text-tail retention, head-tail byte retention, zero budgets, UTF-8 boundary handling (2-, 3-, and 4-byte codepoints and invalid lead bytes at each cut), and unknown omission wording. -**What is documented but not yet migrated.** `glob`, `grep`, `bash`, `web_fetch`, and `web_search` have their mappings documented in the [package README](../../../../packages/util/retention/README.md) — each stating whether it may stop upstream early — but no tool has been migrated onto the library in this change; migration is deliberately separate follow-up work. `glob` / `grep` do not yet exist as tools, so the `shouldStop` early-stop path has no in-repo caller until they land. `read` is documented as intentionally out of scope: its `read-render` line-window contract (`offset`/`limit`, `totalLines`, offset-range errors, per-line preview truncation, a byte cap over the selected window) is not generic retention, and one `Omitted` count cannot represent both sides of a line window. +**What is documented but not yet migrated.** `glob`, `grep`, `bash`, `web_fetch`, and `web_search` have their mappings documented in the [package README](../../../../packages/util/retention/README.md), but not every tool has been migrated onto the library in this change; migration is deliberately separate follow-up work. `read` is documented as intentionally out of scope: its `read-render` line-window contract (`offset`/`limit`, `totalLines`, offset-range errors, per-line preview truncation, a byte cap over the selected window) is not generic retention, and one `Omitted` count cannot represent both sides of a line window. **Boundaries the library holds.** `truncated` means the retainer omitted otherwise-available content because of a budget; it never means the upstream was incomplete. Tool-specific states — `incomplete`, permission failures, provider partial failures, binary skips, bash spill-path recovery, invalid UTF-8 — stay in tool-domain fields, outside the retainer. When a future change migrates a tool, that package's README and tests must prove the model-facing result text is unchanged except for deliberate notice wording. -**Tradeoffs accepted.** The v1 surface deliberately supports only item `head` retention and text `head` / `tail` / `headTail`; windows, grouped budgets, and sort-aware caps wait until a second consumer proves the need (the generic-collector alternative is why). Text retention counts bytes for process/body safety, leaving character- and line-level preview budgets as separate tool-owned concerns. `glob` / `grep` cannot report an exact omitted count once they stop the upstream at the first overflow item, so `Omitted.atLeast` exists and `describeOmitted` prints no number for it — formatters never claim "omitted 1" when the true count may be far larger. +**Tradeoffs accepted.** The v1 surface deliberately supports only item `head` retention and text `head` / `tail` / `headTail`; windows, grouped budgets, sort-aware caps, and upstream-stop control wait until a second consumer proves the need. Text retention counts bytes for process/body safety, leaving character- and line-level preview budgets as separate tool-owned concerns. ## Alternatives considered -**Post-hoc `truncate(text)` only.** Rejected: it matches Codex's history/tool-output truncation use case but fails the `glob` / `grep` resource model. The tool must stop ripgrep once the probe result proves truncation; collecting all output and trimming afterward defeats the point and can exceed the command runner's in-memory output cap. +**Post-hoc `truncate(text)` only.** Rejected: it matches Codex's history/tool-output truncation use case but loses item counts, grouping boundaries, UTF-8-safe byte windows, and exact omission metadata. -**One generic `Collector` with pluggable callbacks.** Rejected for v1: it hides the two important resource modes. Logical item retention can ask the caller to stop after a probe item; text tail/head-tail retention usually must read to the end. Separate `ItemRetainer` and `TextRetainer` names make that difference explicit while keeping the API small. +**One generic `Collector` with pluggable callbacks.** Rejected for v1: it hides the two important resource modes. Logical item retention counts items; text retention counts bytes and preserves UTF-8 boundaries. Separate `ItemRetainer` and `TextRetainer` names make that difference explicit while keeping the API small. **Put `read` windowing behind `ItemRetainer`.** Rejected for v1: `read` is the only current window consumer, and its semantics are file pagination rather than generic retention. A single `Omitted` count cannot represent both sides of a line window, and `read` also carries `totalLines`, offset-range errors, per-line preview truncation, and a byte cap over selected output. Keeping `read-render` tool-owned avoids growing the shared library around one special case. diff --git a/docs/rfc/implemented/architecture/2026-07-08-tool-output-spill-files.md b/docs/rfc/implemented/architecture/2026-07-08-tool-output-spill-files.md index a7d5aa4627..f261ebf84e 100644 --- a/docs/rfc/implemented/architecture/2026-07-08-tool-output-spill-files.md +++ b/docs/rfc/implemented/architecture/2026-07-08-tool-output-spill-files.md @@ -128,7 +128,7 @@ This separation is important. `web-fetch-local` still owns resource caps (`maxRe Retention is separate from spill storage: -- `@deepseek-ai/dsh-retention` owns preview mechanics (`TextRetainer`, `ItemRetainer`, omitted metadata, early-stop decisions). +- `@deepseek-ai/dsh-retention` owns preview mechanics (`TextRetainer`, `ItemRetainer`, and omitted metadata). - `@deepseek-ai/dsh-spill` owns saving final text to a session-scoped path. - `@deepseek-ai/dsh-spill-policy` applies the default final-result policy in the tool pipeline, composing the two. @@ -136,7 +136,7 @@ The final-result policy cannot replace tool-owned early spill. Some useful conte - `bash` final output is already a tail plus a temp spill path; the complete stdout/stderr streams live in executor files. - `subagent` final output is the child final answer, not the child rollout. -- Future `grep`/`glob` may early-stop and never collect full results. +- Future tools may produce runtime artifacts that are never represented by their final `ToolExecutionResult.content`. Those cases can consume `ctx.spillFiles` directly in later work. They are not part of the first showcase. @@ -188,4 +188,4 @@ The policy can become too large if it starts owning tool-specific semantics. It **Let `web-fetch-local` fetch without caps and rely on spill-policy.** Rejected: spill-policy runs after the final tool result exists and cannot protect network, memory, or decoding resources. Provider resource caps stay mandatory. -**Merge retention into spill.** Rejected: retention and spill have different responsibilities. `TextRetainer`/`ItemRetainer` decide what preview is kept and whether upstream may stop; spill storage only saves the final text the policy asks it to save. +**Merge retention into spill.** Rejected: retention and spill have different responsibilities. `TextRetainer`/`ItemRetainer` decide what preview is kept and what was omitted; spill storage only saves the final text the policy asks it to save. diff --git a/packages/util/README.md b/packages/util/README.md index 6477523861..dcfb019207 100644 --- a/packages/util/README.md +++ b/packages/util/README.md @@ -12,4 +12,4 @@ Zero-dependency primitives shared across the other groups. A package lands here `dsh-timeout` follows the same shape for the timeout family: `dsh-bash` and `dsh-web-fetch-local` each fuse a caller's cancellation with a deadline and later classify "timed out" vs "cancelled" by depending on `dsh-timeout` alone. It deliberately owns only the timing/classification half — the *termination* (SIGKILL a process group, tear down a fetch socket) stays in each capability, because no shared layer can own every capability's kill (see [the timeout-library RFC](../../docs/rfc/implemented/architecture/2026-07-06-timeout-deadline-library.md)). -`dsh-retention` is the same split for bounded tool output: a tool (`glob`/`grep`/`bash`/`web_fetch`/`web_search`) feeds items or text into a retainer and gets back "what we kept, what we omitted, may you stop reading" — while grouping, exit codes, provider errors, and recovery prose stay tool-owned. It deliberately owns only the retention mechanic; `truncated` is a budget fact, never an "incomplete inspection" state (see [the retention-library RFC](../../docs/rfc/implemented/architecture/2026-07-06-tool-result-retention-library.md)). +`dsh-retention` is the same split for bounded tool output: a tool (`glob`/`grep`/`bash`/`web_fetch`/`web_search`) feeds items or text into a retainer and gets back what it kept and exactly what it omitted — while grouping, exit codes, provider errors, and recovery prose stay tool-owned. It deliberately owns only the retention mechanic; `truncated` is a budget fact, never an "incomplete inspection" state (see [the retention-library RFC](../../docs/rfc/implemented/architecture/2026-07-06-tool-result-retention-library.md)). diff --git a/packages/util/retention/README.md b/packages/util/retention/README.md index 50bac13828..7256bd3596 100644 --- a/packages/util/retention/README.md +++ b/packages/util/retention/README.md @@ -1,8 +1,8 @@ # dsh-retention -A dependency-light **retention** library: bounded model-facing output for tools that must cap how much context they return. A caller feeds items or text chunks into a bounded object, gets a per-push decision about whether the upstream may stop, and later gets the retained content plus exact or partial omission metadata. +A dependency-light **retention** library: bounded model-facing output for tools that must cap how much context they return. A caller feeds items or text chunks into a bounded object, then gets the retained content plus exact omission metadata. -The library owns **only** the mechanical question *"what did we keep, what did we omit, and may the caller stop reading now?"*. Tool-specific code keeps its business semantics: file grouping, line numbering, exit codes, provider error states, per-line preview truncation, spill files, and the model-facing prose. This is the boundary the [RFC](../../../docs/rfc/implemented/architecture/2026-07-06-tool-result-retention-library.md) draws. +The library owns **only** the mechanical question *"what did we keep, and what did we omit?"*. Tool-specific code keeps its business semantics: file grouping, line numbering, exit codes, provider error states, per-line preview truncation, spill files, and the model-facing prose. This is the boundary the [RFC](../../../docs/rfc/implemented/architecture/2026-07-06-tool-result-retention-library.md) draws. It is a **library, not a service or plugin**: no `ctx`, registers nothing, emits no events. The only state is per-retainer (one accumulation), never cross-call. Tool packages import it directly. @@ -15,7 +15,7 @@ import { } from '@deepseek-ai/dsh-retention' import type { Omitted, PushDecision, RetainedItems, RetainedText, - ItemRetentionStrategy, TextRetentionStrategy, StopMode, RetentionNotice, + ItemRetentionStrategy, TextRetentionStrategy, RetentionNotice, } from '@deepseek-ai/dsh-retention' ``` @@ -23,19 +23,17 @@ import type { |---|---| | `ItemRetainer` | Bounds ordered logical units (paths, grep matches, sources). `head` only in v1. `push()` → `PushDecision`; `finish()` → `RetainedItems`. | | `TextRetainer` | Bounds a byte-oriented text stream. `head` / `tail` / `headTail`, UTF-8 boundaries preserved at `finish()`. `push()` → `PushDecision`; `finish()` → `RetainedText`. | -| `describeOmitted(omitted, unit)` | Standardized, false-precision-safe omission clause (`exact` prints a count; `atLeast`/`unknown` do not). | +| `describeOmitted(omitted, unit)` | Standardized omission clause (`exact` prints a count; `unknown` does not). | | `formatRetentionNotice(notice, recovery)` | Joins the standardized omission clause with the tool's own recovery guidance. | -| `Omitted` | `none` / `exact` / `atLeast` / `unknown` — how much was omitted, and whether the count is a lower bound. | -| `PushDecision` | `{ kept, truncated, shouldStop }` — the per-push control-flow result. | +| `Omitted` | `none` / `exact` / `unknown` — how much was omitted. | +| `PushDecision` | `{ kept, truncated }` — the per-push retention result. | -## The two resource modes +## Resource Modes -The two retainers are separate names, not one generic collector, because they differ in **resource model** — and that difference is the whole point of the `shouldStop` field. +The two retainers are separate names, not one generic collector, because they differ in **resource model**. -- **`ItemRetainer` can stop the upstream early.** With `stop: 'stopWhenFull'`, the first over-cap unit is a *probe*: it is not retained, sets `truncated`, and returns `shouldStop: true`. A discovery tool uses that to kill ripgrep / cancel a stream the moment truncation is proven, instead of collecting everything and trimming afterward. Because it stopped before the true total was known, `omitted` is `{ kind: 'atLeast', count: 1 }` — a lower bound, never a false-precise exact count. -- **`TextRetainer` tail/headTail must read to the end.** A true tail is unknowable until the stream closes, and draining avoids pipe backpressure on a child process, so `tail` and `headTail` never set `shouldStop` and report an `exact` omitted byte count. Only `head` + `stopWhenFull` can stop a text stream early. - -`shouldStop` is **advisory**: the retainer cannot reach the upstream. The tool owns the actual stop — abort the HTTP body, break the scan, kill the process group. +- **`ItemRetainer` bounds ordered logical units.** A search tool can collect a full result set for spill-file recovery while retaining only the first `maxItems` for the model-facing preview. The omission count is exact because the caller keeps feeding every observed item. +- **`TextRetainer` bounds byte-oriented text.** `head`, `tail`, and `headTail` preserve UTF-8 boundaries at `finish()`; `headTail` is the shape `dsh-spill-policy` uses to build a bounded preview around a spill-file notice. ## `truncated` is a budget fact, never "incomplete" @@ -47,32 +45,33 @@ Text caps and `omittedBytes` count **bytes**, for process/body safety (a child's ## Tool mappings -Every current retention consumer maps to the library below; each row states whether it may stop its upstream early. A broad migration is out of scope for the library's first landing — these are the intended shapes. +Every current retention consumer maps to the library below. A broad migration is out of scope for the library's first landing — these are the intended shapes. -| Tool | Retainer & strategy | Stops upstream early? | Notes | -|---|---|---|---| -| `glob` | `ItemRetainer`, `head` + `stopWhenFull` | **Yes** — the `(maxItems+1)`th path is the probe; `shouldStop` kills ripgrep. | Path mapping, skipped candidates, `incomplete` stay outside. `omitted` is `atLeast`. | -| `grep` | `ItemRetainer`, `head` + `stopWhenFull` | **Yes** — cap is total matches; stop on the probe match. | Per-match preview truncation, then push a flat match; group + sort the retained subset *after* `finish()`. | -| `bash` | `TextRetainer`, `tail` or `headTail`, reads to completion | No — stopping would lose the true tail and risk pipe backpressure. | Executor still owns spill files, exit status, signal, timeout, background tasks. | -| `web_fetch` | `TextRetainer`, `head` (streaming provider) | Optional — a streaming body can stop; a decode-internally provider keeps its own cap. | The fetch result's `truncated` remains a provider/tool fact. | -| `web_search` | `ItemRetainer`, `head` | Post-hoc today (providers return arrays); a streaming provider can use `stopWhenFull`. | Standardizes the "sources capped" notice. | +| Tool | Retainer & strategy | Notes | +|---|---|---| +| `glob` | `ItemRetainer`, `head` | Collect the full sorted path list for a spill file while retaining the first page inline. Path mapping, skipped candidates, and `incomplete` stay outside. | +| `grep` | `ItemRetainer`, `head` | Collect matches for a spill file while retaining the first page inline. Per-match preview truncation, grouping, sorting, and `incomplete` stay outside. | +| `bash` | `TextRetainer`, `tail` or `headTail` | Executor still owns spill files, exit status, signal, timeout, and background tasks. | +| `web_fetch` | `TextRetainer`, `head` or `headTail` | Provider/resource caps stay provider facts; the retainer supplies only retained text and omission metadata. | +| `web_search` | `ItemRetainer`, `head` | Standardizes the "sources capped" notice when providers return more sources than the model-facing result should include. | `read` is **intentionally out of scope for v1.** Its `read-render` helper owns a file-specific pagination contract — `offset`/`limit`, line numbers, `totalLines`, offset-out-of-range errors, per-line preview truncation, a byte cap over the selected window — which is a line-window renderer, not generic retention. A single `Omitted` count cannot represent both sides of a line window. ## Usage shape ```ts ignore-check -// glob: stop ripgrep the moment truncation is proven. -const retainer = new ItemRetainer({ kind: 'head', maxItems: globMaxResults, stop: 'stopWhenFull' }) +// glob: keep the first page inline while still collecting the full list for spill. +const retainer = new ItemRetainer({ kind: 'head', maxItems: globMaxResults }) +const allEntries: FsGlobEntry[] = [] for await (const entry of candidates) { - const { shouldStop } = retainer.push(entry) - if (shouldStop) { killRipgrep(); break } // the tool owns the actual stop + allEntries.push(entry) + retainer.push(entry) } const { items, truncated, omitted } = retainer.finish() // bash: keep a head + tail, read to process exit. const out = new TextRetainer({ kind: 'headTail', headBytes: headCap, tailBytes: tailCap }) -child.stdout.on('data', (chunk: Buffer) => { out.push(chunk) }) // shouldStop ignored: must drain +child.stdout.on('data', (chunk: Buffer) => { out.push(chunk) }) const { text, omittedBytes } = out.finish() // A footer: the library standardizes the omission clause; the tool owns recovery words. diff --git a/packages/util/retention/package.json b/packages/util/retention/package.json index 2926144ace..db8bab3342 100644 --- a/packages/util/retention/package.json +++ b/packages/util/retention/package.json @@ -1,6 +1,6 @@ { "name": "@deepseek-ai/dsh-retention", - "description": "Zero-dependency bounded-retention primitive: ItemRetainer/TextRetainer + neutral notice helpers (what did we keep, what did we omit, may the caller stop reading)", + "description": "Zero-dependency bounded-retention primitive: ItemRetainer/TextRetainer + neutral notice helpers (what did we keep, what did we omit)", "version": "0.0.1", "private": true, "type": "module", diff --git a/packages/util/retention/src/index.ts b/packages/util/retention/src/index.ts index 8c3b924a16..07547a7d93 100644 --- a/packages/util/retention/src/index.ts +++ b/packages/util/retention/src/index.ts @@ -1,12 +1,11 @@ /** * A dependency-light **retention** library: bounded model-facing output for * tools that must cap how much context they return. A caller feeds items or - * text chunks into a bounded object, gets a per-push {@link PushDecision} about - * whether the upstream may stop, and later gets the retained content plus exact - * or partial omission metadata ({@link RetainedItems} / {@link RetainedText}). + * text chunks into a bounded object, then gets the retained content plus exact + * omission metadata ({@link RetainedItems} / {@link RetainedText}). * * The library owns ONLY the mechanical question "what did we keep, what did we - * omit, and may the caller stop reading now?". Tool-specific code still owns + * omit?". Tool-specific code still owns * business semantics: file grouping, line numbering, exit codes, provider error * states, per-line preview truncation, spill files, and the model-facing prose. * In particular {@link RetainedText.truncated}/{@link RetainedItems.truncated} @@ -23,12 +22,10 @@ * The two retainers differ in resource model, which is why they are two names * rather than one generic collector: * - {@link ItemRetainer} bounds ordered logical units (paths, grep matches, - * search sources). `head` retention only in v1. With `stopWhenFull` it can ask - * the caller to stop the upstream after the first over-cap probe item. + * search sources). `head` retention only in v1. * - {@link TextRetainer} bounds byte-oriented text streams (bash stdout/stderr, * web bodies). `head` / `tail` / `headTail`, preserving UTF-8 boundaries at - * {@link TextRetainer.finish}. Only `head` can stop early; `tail`/`headTail` - * must read to the end to know the true suffix and exact omission. + * {@link TextRetainer.finish}. * * @module @deepseek-ai/dsh-retention */ @@ -36,45 +33,31 @@ /** * How much content the retainer omitted. * - * `atLeast` is the early-stop shape: an {@link ItemRetainer}/{@link TextRetainer} - * with `stopWhenFull` sees the first unit/chunk past the cap, asks the caller to - * stop the upstream, and therefore knows only a LOWER bound — reporting an exact - * count there would be false precision when the true total may be much larger. - * `exact` is the read-to-end shape (`tail`, `headTail`, or `head` with - * `readToEnd`), where every unit/byte was observed. `unknown` is reserved for a - * caller that omits without a count; the retainers themselves never return it. + * `exact` is the normal retainer shape: every unit/byte was observed, so the + * omitted count is precise. `unknown` is reserved for a caller that omits + * without a count; the retainers themselves never return it. */ export type Omitted = | { kind: 'none' } | { kind: 'exact'; count: number } - | { kind: 'atLeast'; count: number } | { kind: 'unknown' } /** * The caller receives this after each `push()`. - * - * `shouldStop` is ADVISORY, not automatic: the tool owns how to stop its upstream - * source — aborting an HTTP body, breaking a file scan, killing ripgrep. The - * retainer cannot reach the upstream; it only reports that keeping more would - * exceed the budget. A `readToEnd` / `tail` / `headTail` retainer never sets it - * (those must drain to the end). */ export interface PushDecision { /** Was this whole unit / all of this chunk's bytes retained (nothing dropped)? */ kept: boolean /** Cumulative: has the retainer omitted anything due to the budget yet? */ truncated: boolean - /** Advisory: keeping more would exceed the budget — the tool may stop its upstream. */ - shouldStop: boolean } /** * Final result for ordered logical units. * * `seen` means units OBSERVED by the retainer, not necessarily the total in the - * upstream source; with an early stop, the true total is intentionally unknown - * (hence {@link Omitted.atLeast}). `kept` is `items.length`, surfaced explicitly - * so a notice formatter need not re-count. + * upstream source. `kept` is `items.length`, surfaced explicitly so a notice + * formatter need not re-count. */ export interface RetainedItems { items: T[] @@ -100,30 +83,19 @@ export interface RetainedText { omittedBytes: Omitted } -/** - * Whether a retainer asks the caller to stop the upstream once keeping more - * would exceed the budget (`stopWhenFull`), or must keep accepting input even - * after the retained output is full (`readToEnd`) — usually to preserve a true - * tail, count exact omission, or drain an upstream process to avoid pipe - * backpressure. Names avoid implementation phrases like "overflow". - */ -export type StopMode = 'stopWhenFull' | 'readToEnd' - /** Item retention strategy. Only `head` in v1; windows/grouped budgets wait for a second consumer. */ export type ItemRetentionStrategy = { /** Keep the first `maxItems` units. Use for `glob`, `grep`, and web sources. */ kind: 'head' maxItems: number - stop: StopMode } /** Text retention strategy: keep a prefix, a suffix, or both, counted in bytes. */ export type TextRetentionStrategy = | { - /** Keep the first `maxBytes` bytes. May stop an upstream body early. */ + /** Keep the first `maxBytes` bytes. */ kind: 'head' maxBytes: number - stop: StopMode } | { /** Keep the final `maxBytes` bytes. Requires reading to the end. */ @@ -164,8 +136,7 @@ function assertBudget(value: number, name: string): void { /** * Bounds an ordered stream of logical units, keeping the first `maxItems` * ({@link ItemRetentionStrategy} `head`). `push()` reports, per unit, whether it - * was kept and — under `stopWhenFull` — whether the caller should stop the - * upstream now that the first over-cap probe unit has been seen. + * was kept and whether the retained result is now truncated. * * Grouping, sorting, path mapping, per-unit preview truncation, and any * `incomplete` state stay OUTSIDE the retainer: it counts and keeps, nothing @@ -174,24 +145,20 @@ function assertBudget(value: number, name: string): void { */ export class ItemRetainer { private readonly maxItems: number - private readonly stop: StopMode private readonly items: T[] = [] private seen = 0 private omittedCount = 0 - /** @param strategy Head strategy: `maxItems` (non-negative integer) and the {@link StopMode}. */ + /** @param strategy Head strategy: `maxItems` (non-negative integer). */ constructor(strategy: ItemRetentionStrategy) { assertBudget(strategy.maxItems, 'maxItems') this.maxItems = strategy.maxItems - this.stop = strategy.stop } /** * Offer one unit. Kept when the retainer is below `maxItems`; otherwise dropped - * and counted as omitted. Under `stopWhenFull` the first dropped unit is the - * probe: `shouldStop` is `true` so the caller can kill ripgrep / cancel the - * stream, and the final {@link Omitted} stays `atLeast` (the true total is - * unknown). Under `readToEnd` the caller keeps pushing so omission is `exact`. + * and counted as omitted. Callers keep pushing all observed units, so the final + * {@link Omitted} count is exact. * * @param item The already-shaped logical unit (path, flat match, source). * @returns The per-push {@link PushDecision}. @@ -202,22 +169,17 @@ export class ItemRetainer { // Reached only below the cap, before any omission (items only grow, the // cap is fixed), so nothing has been dropped yet: truncated is always false. this.items.push(item) - return { kept: true, truncated: false, shouldStop: false } + return { kept: true, truncated: false } } this.omittedCount++ return { kept: false, truncated: true, - // Only ask to stop when the caller opted into it; readToEnd must keep - // draining to reach an exact omission count. - shouldStop: this.stop === 'stopWhenFull', } } /** - * Finalize and report what was kept and omitted. `omitted` is `atLeast` under - * `stopWhenFull` (a lower bound — the caller was asked to stop before the true - * total was known) and `exact` under `readToEnd`. + * Finalize and report what was kept and omitted. * * @returns The {@link RetainedItems} snapshot (safe to group/sort downstream). */ @@ -229,7 +191,7 @@ export class ItemRetainer { seen: this.seen, kept: this.items.length, omitted: truncated - ? { kind: this.stop === 'stopWhenFull' ? 'atLeast' : 'exact', count: this.omittedCount } + ? { kind: 'exact', count: this.omittedCount } : { kind: 'none' }, } } @@ -275,8 +237,6 @@ function trimLeadingContinuationUtf8(bytes: Uint8Array): Uint8Array { * Bounds a byte-oriented text stream, keeping a prefix, a suffix, or both * ({@link TextRetentionStrategy}). All three strategies share one prefix/suffix * accumulator: `head` is prefix-only, `tail` is suffix-only, `headTail` is both. - * Only `head` with `stopWhenFull` sets `shouldStop`; `tail`/`headTail` must read - * to the end to know the true suffix and the exact omitted byte count. * * Bytes, not characters: caps and `omittedBytes` are byte counts for process/ * body safety. Chunks that straddle a codepoint are handled — {@link finish} @@ -288,7 +248,6 @@ function trimLeadingContinuationUtf8(bytes: Uint8Array): Uint8Array { export class TextRetainer { private readonly prefixCap: number private readonly suffixCap: number - private readonly allowStop: boolean private readonly prefixChunks: Uint8Array[] = [] private prefixHeld = 0 private readonly suffixChunks: Uint8Array[] = [] @@ -302,20 +261,17 @@ export class TextRetainer { assertBudget(strategy.maxBytes, 'maxBytes') this.prefixCap = strategy.maxBytes this.suffixCap = 0 - this.allowStop = strategy.stop === 'stopWhenFull' break case 'tail': assertBudget(strategy.maxBytes, 'maxBytes') this.prefixCap = 0 this.suffixCap = strategy.maxBytes - this.allowStop = false break case 'headTail': assertBudget(strategy.headBytes, 'headBytes') assertBudget(strategy.tailBytes, 'tailBytes') this.prefixCap = strategy.headBytes this.suffixCap = strategy.tailBytes - this.allowStop = false break } } @@ -324,9 +280,7 @@ export class TextRetainer { * Offer one chunk (a `Uint8Array`, or a `string` encoded as UTF-8). Prefix * bytes fill up to the prefix cap then stop; suffix bytes roll so only the * last `suffixCap` bytes are retained. `kept` is `true` only when no byte of - * this chunk was dropped. Under `head` + `stopWhenFull`, `shouldStop` turns - * `true` on the chunk that first drops a byte (the caller may then abort the - * body); other strategies never set it. + * this chunk was dropped. * * @param chunk The next bytes of the stream (`Uint8Array` or UTF-8 `string`). * @returns The per-push {@link PushDecision}. @@ -373,12 +327,11 @@ export class TextRetainer { // Dropped = bytes that no side can keep. Compute cumulative omission the // SAME way finish() does (via omittedAt), so push and finish never disagree; // per-push we only need whether THIS chunk pushed the total past what the - // two caps hold, and — for head+stopWhenFull — whether to stop. + // two caps hold. const droppedThisChunk = this.omittedAt(this.total) > this.omittedAt(before) return { kept: !droppedThisChunk, truncated: this.omittedAt(this.total) > 0, - shouldStop: this.allowStop && droppedThisChunk, } } @@ -391,10 +344,7 @@ export class TextRetainer { /** * Finalize: decode the retained prefix and suffix (each trimmed to a UTF-8 - * boundary at its cut) and report the exact or lower-bound omitted byte count. - * `head` + `stopWhenFull` yields `atLeast` (a lower bound — the caller was - * asked to stop before the true size was known); every other case reads to the - * end and yields `exact`. + * boundary at its cut) and report the exact omitted byte count. * * @returns The {@link RetainedText} snapshot (safe to hand to a formatter). */ @@ -423,8 +373,7 @@ export class TextRetainer { // Report omission against the bytes ACTUALLY returned, not the pre-trim // budget: a boundary trim drops partial-codepoint bytes too, so an exact // count derived from the budget alone would overstate the retained text (and - // any "Omitted N bytes" notice built from it would be a lie). total_seen − - // retained stays a valid lower bound under `atLeast` (true total ≥ seen). + // any "Omitted N bytes" notice built from it would be a lie). const omitted = this.total - keptPrefix.length - keptSuffix.length const truncated = omitted > 0 @@ -432,7 +381,7 @@ export class TextRetainer { text, truncated, omittedBytes: truncated - ? { kind: this.allowStop ? 'atLeast' : 'exact', count: omitted } + ? { kind: 'exact', count: omitted } : { kind: 'none' }, } } @@ -454,10 +403,8 @@ function concat(chunks: readonly Uint8Array[]): Uint8Array { /** * Standardized, false-precision-safe wording for one {@link Omitted} value — * the "may standardize omission wording" half the library owns. `exact` prints - * the count (`Omitted 3 items`); `atLeast`/`unknown` print NO count, because an - * early stop knows only that more was dropped, not how much (claiming "omitted - * 1" when the true total may be huge is the false-precision trap the `atLeast` - * variant exists to avoid). `none` is the empty string. + * the count (`Omitted 3 items`); `unknown` prints NO count because the caller + * did not provide one. `none` is the empty string. * * @param omitted The omission metadata from a retainer result. * @param unit The noun for the omitted quantity (`items`, `bytes`, `chars`, `lines`). @@ -469,7 +416,6 @@ export function describeOmitted(omitted: Omitted, unit: RetentionNotice['unit']) return '' case 'exact': return `Omitted ${omitted.count} ${unit}.` - case 'atLeast': case 'unknown': return `More ${unit} were omitted.` } diff --git a/packages/util/retention/tests/retention.spec.ts b/packages/util/retention/tests/retention.spec.ts index ec424595cb..8fac7d8575 100644 --- a/packages/util/retention/tests/retention.spec.ts +++ b/packages/util/retention/tests/retention.spec.ts @@ -11,26 +11,23 @@ import { /** Decode a RetainedText via a round-trip helper for readable UTF-8 assertions. */ const utf8 = (s: string): Uint8Array => new TextEncoder().encode(s) -describe('ItemRetainer — head, stopWhenFull (glob/grep early stop)', () => { - it('keeps the first maxItems and asks to stop on the probe item', () => { - const r = new ItemRetainer({ kind: 'head', maxItems: 2, stop: 'stopWhenFull' }) - expect(r.push('a')).toEqual({ kept: true, truncated: false, shouldStop: false }) - expect(r.push('b')).toEqual({ kept: true, truncated: false, shouldStop: false }) - // The (maxItems + 1)th valid item is the probe: not retained, sets truncated, - // and shouldStop tells the caller to kill the upstream. - expect(r.push('c')).toEqual({ kept: false, truncated: true, shouldStop: true }) +describe('ItemRetainer — head retention', () => { + it('keeps the first maxItems while callers keep draining for an exact omitted count', () => { + const r = new ItemRetainer({ kind: 'head', maxItems: 2 }) + expect(r.push('a')).toEqual({ kept: true, truncated: false }) + expect(r.push('b')).toEqual({ kept: true, truncated: false }) + expect(r.push('c')).toEqual({ kept: false, truncated: true }) const result = r.finish() expect(result.items).toEqual(['a', 'b']) expect(result.kept).toBe(2) expect(result.seen).toBe(3) expect(result.truncated).toBe(true) - // Early stop knows only a lower bound, never an exact total. - expect(result.omitted).toEqual({ kind: 'atLeast', count: 1 }) + expect(result.omitted).toEqual({ kind: 'exact', count: 1 }) }) it('reports none when everything fits', () => { - const r = new ItemRetainer({ kind: 'head', maxItems: 3, stop: 'stopWhenFull' }) + const r = new ItemRetainer({ kind: 'head', maxItems: 3 }) r.push(1) r.push(2) const result = r.finish() @@ -38,15 +35,11 @@ describe('ItemRetainer — head, stopWhenFull (glob/grep early stop)', () => { expect(result.truncated).toBe(false) expect(result.omitted).toEqual({ kind: 'none' }) }) -}) - -describe('ItemRetainer — head, readToEnd (exact omission)', () => { it('keeps draining past the cap and reports an exact omitted count', () => { - const r = new ItemRetainer({ kind: 'head', maxItems: 1, stop: 'readToEnd' }) - expect(r.push('a')).toEqual({ kept: true, truncated: false, shouldStop: false }) - // readToEnd never asks to stop — the caller must keep pushing to count exactly. - expect(r.push('b')).toEqual({ kept: false, truncated: true, shouldStop: false }) - expect(r.push('c')).toEqual({ kept: false, truncated: true, shouldStop: false }) + const r = new ItemRetainer({ kind: 'head', maxItems: 1 }) + expect(r.push('a')).toEqual({ kept: true, truncated: false }) + expect(r.push('b')).toEqual({ kept: false, truncated: true }) + expect(r.push('c')).toEqual({ kept: false, truncated: true }) const result = r.finish() expect(result.items).toEqual(['a']) @@ -56,53 +49,49 @@ describe('ItemRetainer — head, readToEnd (exact omission)', () => { }) describe('ItemRetainer — zero budget', () => { - it('keeps nothing; first item is the probe under stopWhenFull', () => { - const r = new ItemRetainer({ kind: 'head', maxItems: 0, stop: 'stopWhenFull' }) - expect(r.push('a')).toEqual({ kept: false, truncated: true, shouldStop: true }) + it('keeps nothing and counts every pushed item as omitted', () => { + const r = new ItemRetainer({ kind: 'head', maxItems: 0 }) + expect(r.push('a')).toEqual({ kept: false, truncated: true }) const result = r.finish() expect(result.items).toEqual([]) expect(result.kept).toBe(0) - expect(result.omitted).toEqual({ kind: 'atLeast', count: 1 }) + expect(result.omitted).toEqual({ kind: 'exact', count: 1 }) }) it('rejects a non-integer / negative maxItems', () => { - expect(() => new ItemRetainer({ kind: 'head', maxItems: -1, stop: 'readToEnd' })) + expect(() => new ItemRetainer({ kind: 'head', maxItems: -1 })) .toThrow(/maxItems must be a non-negative integer/) - expect(() => new ItemRetainer({ kind: 'head', maxItems: 1.5, stop: 'readToEnd' })) + expect(() => new ItemRetainer({ kind: 'head', maxItems: 1.5 })) .toThrow(/maxItems must be a non-negative integer/) }) }) -describe('TextRetainer — head, stopWhenFull (early body stop)', () => { - it('keeps the prefix and asks to stop on the overflowing chunk', () => { - const r = new TextRetainer({ kind: 'head', maxBytes: 5, stop: 'stopWhenFull' }) - expect(r.push('abc')).toEqual({ kept: true, truncated: false, shouldStop: false }) +describe('TextRetainer — head (exact omission, reads to end)', () => { + it('keeps the prefix and counts omitted bytes exactly', () => { + const r = new TextRetainer({ kind: 'head', maxBytes: 5 }) + expect(r.push('abc')).toEqual({ kept: true, truncated: false }) // 'de' fills the cap exactly (5 bytes) — still fully kept. - expect(r.push('de')).toEqual({ kept: true, truncated: false, shouldStop: false }) - // 'fgh' is wholly dropped: kept:false, and stopWhenFull → shouldStop. - expect(r.push('fgh')).toEqual({ kept: false, truncated: true, shouldStop: true }) + expect(r.push('de')).toEqual({ kept: true, truncated: false }) + expect(r.push('fgh')).toEqual({ kept: false, truncated: true }) const result = r.finish() expect(result.text).toBe('abcde') expect(result.truncated).toBe(true) - // Early stop: a lower bound, not an exact size. - expect(result.omittedBytes).toEqual({ kind: 'atLeast', count: 3 }) + expect(result.omittedBytes).toEqual({ kind: 'exact', count: 3 }) }) it('flags a partially-dropped chunk as not fully kept', () => { - const r = new TextRetainer({ kind: 'head', maxBytes: 4, stop: 'stopWhenFull' }) + const r = new TextRetainer({ kind: 'head', maxBytes: 4 }) r.push('ab') - // 'cde' straddles the cap: 'c','d' fit, 'e' drops → kept:false, shouldStop. - expect(r.push('cde')).toEqual({ kept: false, truncated: true, shouldStop: true }) + // 'cde' straddles the cap: 'c','d' fit, 'e' drops → kept:false. + expect(r.push('cde')).toEqual({ kept: false, truncated: true }) expect(r.finish().text).toBe('abcd') }) -}) -describe('TextRetainer — head, readToEnd (exact omission)', () => { - it('keeps the prefix, drains the rest, and counts exactly', () => { - const r = new TextRetainer({ kind: 'head', maxBytes: 3, stop: 'readToEnd' }) + it('keeps draining past the cap', () => { + const r = new TextRetainer({ kind: 'head', maxBytes: 3 }) r.push('abc') - expect(r.push('defg')).toEqual({ kept: false, truncated: true, shouldStop: false }) + expect(r.push('defg')).toEqual({ kept: false, truncated: true }) const result = r.finish() expect(result.text).toBe('abc') expect(result.omittedBytes).toEqual({ kind: 'exact', count: 4 }) @@ -112,8 +101,7 @@ describe('TextRetainer — head, readToEnd (exact omission)', () => { describe('TextRetainer — tail (exact omission, reads to end)', () => { it('keeps the final maxBytes and reports exact omission', () => { const r = new TextRetainer({ kind: 'tail', maxBytes: 4 }) - // tail never asks to stop — it must read to the end to know the true suffix. - expect(r.push('hello')).toEqual({ kept: false, truncated: true, shouldStop: false }) + expect(r.push('hello')).toEqual({ kept: false, truncated: true }) r.push('world') const result = r.finish() expect(result.text).toBe('orld') // last 4 bytes of 'helloworld' @@ -185,12 +173,12 @@ describe('TextRetainer — headTail (prefix + suffix, omit the middle)', () => { }) describe('TextRetainer — zero budgets', () => { - it('head maxBytes 0 keeps nothing and stops on first byte (stopWhenFull)', () => { - const r = new TextRetainer({ kind: 'head', maxBytes: 0, stop: 'stopWhenFull' }) - expect(r.push('x')).toEqual({ kept: false, truncated: true, shouldStop: true }) + it('head maxBytes 0 keeps nothing and counts every byte exactly', () => { + const r = new TextRetainer({ kind: 'head', maxBytes: 0 }) + expect(r.push('x')).toEqual({ kept: false, truncated: true }) const result = r.finish() expect(result.text).toBe('') - expect(result.omittedBytes).toEqual({ kind: 'atLeast', count: 1 }) + expect(result.omittedBytes).toEqual({ kind: 'exact', count: 1 }) }) it('an empty stream omits nothing', () => { @@ -202,7 +190,7 @@ describe('TextRetainer — zero budgets', () => { }) it('rejects non-integer / negative byte budgets', () => { - expect(() => new TextRetainer({ kind: 'head', maxBytes: -1, stop: 'readToEnd' })) + expect(() => new TextRetainer({ kind: 'head', maxBytes: -1 })) .toThrow(/maxBytes must be a non-negative integer/) expect(() => new TextRetainer({ kind: 'tail', maxBytes: 2.5 })) .toThrow(/maxBytes must be a non-negative integer/) @@ -218,7 +206,7 @@ describe('TextRetainer — UTF-8 boundary handling', () => { // '€' is 3 bytes (E2 82 AC). A 2-byte head cap keeps 'a' (61) + the first // byte of '€' (E2); that partial lead byte must be trimmed, not decoded to // a replacement char. - const r = new TextRetainer({ kind: 'head', maxBytes: 2, stop: 'readToEnd' }) + const r = new TextRetainer({ kind: 'head', maxBytes: 2 }) r.push('a€b') // bytes: 61 E2 82 AC 62 const result = r.finish() expect(result.text).toBe('a') // partial '€' dropped, no U+FFFD @@ -256,7 +244,7 @@ describe('TextRetainer — UTF-8 boundary handling', () => { }) it('preserves a whole multibyte codepoint that fits exactly', () => { - const r = new TextRetainer({ kind: 'head', maxBytes: 3, stop: 'readToEnd' }) + const r = new TextRetainer({ kind: 'head', maxBytes: 3 }) r.push('€x') // '€' is exactly 3 bytes expect(r.finish().text).toBe('€') }) @@ -272,7 +260,7 @@ describe('TextRetainer — UTF-8 boundary handling', () => { }) it('accepts a raw Uint8Array chunk', () => { - const r = new TextRetainer({ kind: 'head', maxBytes: 2, stop: 'readToEnd' }) + const r = new TextRetainer({ kind: 'head', maxBytes: 2 }) r.push(utf8('xy')) r.push(utf8('z')) expect(r.finish().text).toBe('xy') @@ -281,7 +269,7 @@ describe('TextRetainer — UTF-8 boundary handling', () => { it('trims a partial 2-byte codepoint at the head cut', () => { // 'é' is 2 bytes (C3 A9). A 2-byte head cap over 'aé' keeps 'a' (61) + the // lead byte of 'é' (C3) — an incomplete 2-byte sequence to trim. - const r = new TextRetainer({ kind: 'head', maxBytes: 2, stop: 'readToEnd' }) + const r = new TextRetainer({ kind: 'head', maxBytes: 2 }) r.push('aé') // bytes: 61 C3 A9 const result = r.finish() expect(result.text).toBe('a') @@ -291,7 +279,7 @@ describe('TextRetainer — UTF-8 boundary handling', () => { it('trims a partial 4-byte codepoint (emoji) at the head cut', () => { // '😀' is 4 bytes (F0 9F 98 80). A 3-byte head cap keeps 'a' + the first two // bytes of the emoji — an incomplete 4-byte sequence that must be trimmed. - const r = new TextRetainer({ kind: 'head', maxBytes: 3, stop: 'readToEnd' }) + const r = new TextRetainer({ kind: 'head', maxBytes: 3 }) r.push('a😀') // bytes: 61 F0 9F 98 80 const result = r.finish() expect(result.text).toBe('a') @@ -299,7 +287,7 @@ describe('TextRetainer — UTF-8 boundary handling', () => { }) it('keeps a whole 4-byte codepoint that fits exactly', () => { - const r = new TextRetainer({ kind: 'head', maxBytes: 4, stop: 'readToEnd' }) + const r = new TextRetainer({ kind: 'head', maxBytes: 4 }) r.push('😀x') expect(r.finish().text).toBe('😀') }) @@ -308,7 +296,7 @@ describe('TextRetainer — UTF-8 boundary handling', () => { // A cut whose trailing bytes are ALL continuation bytes with no lead in // reach is not a trimmable incomplete sequence — the trimmer bails (no lead // byte found) and leaves them for the non-fatal decoder to replace. - const r = new TextRetainer({ kind: 'head', maxBytes: 2, stop: 'readToEnd' }) + const r = new TextRetainer({ kind: 'head', maxBytes: 2 }) // 0x80 0x80 are bare continuation bytes; 'z' follows so the head keeps just // the two continuation bytes and the cut lands right after them. r.push(new Uint8Array([0x80, 0x80, 0x7a])) @@ -322,7 +310,7 @@ describe('TextRetainer — UTF-8 boundary handling', () => { // 0xF8 is not a valid UTF-8 lead byte (only 0x00–0xF7 lead). The trimmer // recognizes it as "not a lead" (expected length 0) and leaves the byte in // place rather than trimming a phantom partial sequence. - const r = new TextRetainer({ kind: 'head', maxBytes: 1, stop: 'readToEnd' }) + const r = new TextRetainer({ kind: 'head', maxBytes: 1 }) r.push(new Uint8Array([0xf8, 0x61])) // 0xF8 kept, 'a' dropped by the 1-byte cap const result = r.finish() expect(result.omittedBytes).toEqual({ kind: 'exact', count: 1 }) @@ -335,10 +323,7 @@ describe('describeOmitted — false precision safety', () => { expect(describeOmitted({ kind: 'exact', count: 12 }, 'bytes')).toBe('Omitted 12 bytes.') }) - it('prints NO count for atLeast (early stop) and unknown', () => { - // The whole point of atLeast: never claim "omitted 1" when the true count is - // unknown. Both atLeast and unknown collapse to a countless clause. - expect(describeOmitted({ kind: 'atLeast', count: 1 }, 'items')).toBe('More items were omitted.') + it('prints NO count for unknown omission', () => { expect(describeOmitted({ kind: 'unknown' }, 'lines')).toBe('More lines were omitted.') }) @@ -359,10 +344,10 @@ describe('formatRetentionNotice', () => { it('joins the standardized omission clause with the tool recovery guidance', () => { const out = formatRetentionNotice( - notice({ kind: 'atLeast', count: 1 }), + notice({ kind: 'exact', count: 25 }), ({ kept }) => `Results capped at ${kept}. Narrow the pattern, path, or include to see more.`, ) - expect(out).toBe('More items were omitted. Results capped at 100. Narrow the pattern, path, or include to see more.') + expect(out).toBe('Omitted 25 items. Results capped at 100. Narrow the pattern, path, or include to see more.') }) it('omits the empty half when nothing was omitted', () => { From e0f20088d85b40491891dd7a632312a16884f040 Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Thu, 9 Jul 2026 20:44:32 +0800 Subject: [PATCH 14/24] feat: bash-backed glob/grep discovery tools (dsh-tool-fs-search) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements docs/rfc/implemented/feature/2026-07-09-bash-backed-grep-glob- discovery.md: model-facing glob/grep in a new @deepseek-ai/dsh-tool-fs-search package, executing fixed ripgrep templates through ctx.bash.resolve/run — not ctx.fs provider methods — so filesystem backends stay free of a search contract and sandboxed/remote executors substitute cleanly. The tools never call ctx.bash.start(); the tool layer owns quoting (one singleQuote safety boundary), rg --json parsing, ItemRetainer/TextRetainer retention, and the first tool-owned ctx.spillFiles.saveText() handoff (item-level retention the generic post-execute spill policy cannot recover). RFC amendments on the way to implemented/: a shared src/search-core.ts (the SEARCH_* vocabulary + bash-run/raw-spill/spill plumbing was byte-identical across both tools — the missed-extraction smell), and a snapshot-gap note: wiring the acp-agent tree changes the assembled prompt, so goldens need a keyed re-record; the spill notice text is pinned by unit tests instead and only the coding-agent example ships the tools for now. --- docs/config-catalog.md | 22 + docs/module-graph.md | 9 + docs/rfc/INDEX.md | 1 + ...6-07-09-bash-backed-grep-glob-discovery.md | 168 +++++ docs/tool-catalog.md | 59 ++ examples/coding-agent/composition.md | 3 + examples/coding-agent/cordis.yml | 6 + knip.json | 5 + packages/README.md | 2 +- .../core/tools/tests/gen-tool-catalog.spec.ts | 2 +- packages/fs/README.md | 7 +- packages/fs/tool-fs-search/README.md | 46 ++ packages/fs/tool-fs-search/package.json | 49 ++ packages/fs/tool-fs-search/src/glob.ts | 168 +++++ packages/fs/tool-fs-search/src/grep.ts | 314 +++++++++ packages/fs/tool-fs-search/src/index.ts | 110 ++++ packages/fs/tool-fs-search/src/search-core.ts | 257 ++++++++ packages/fs/tool-fs-search/src/shell-quote.ts | 27 + .../tool-fs-search/tests/integration.spec.ts | 161 +++++ .../fs/tool-fs-search/tests/load-path.spec.ts | 50 ++ .../tool-fs-search/tests/shell-quote.spec.ts | 59 ++ .../fs/tool-fs-search/tests/tools.spec.ts | 623 ++++++++++++++++++ packages/fs/tool-fs-search/tsconfig.json | 20 + pnpm-lock.yaml | 37 ++ scripts/gen-tool-catalog.ts | 18 + tsconfig.build.json | 1 + tsconfig.json | 1 + 27 files changed, 2220 insertions(+), 5 deletions(-) create mode 100644 docs/rfc/implemented/feature/2026-07-09-bash-backed-grep-glob-discovery.md create mode 100644 packages/fs/tool-fs-search/README.md create mode 100644 packages/fs/tool-fs-search/package.json create mode 100644 packages/fs/tool-fs-search/src/glob.ts create mode 100644 packages/fs/tool-fs-search/src/grep.ts create mode 100644 packages/fs/tool-fs-search/src/index.ts create mode 100644 packages/fs/tool-fs-search/src/search-core.ts create mode 100644 packages/fs/tool-fs-search/src/shell-quote.ts create mode 100644 packages/fs/tool-fs-search/tests/integration.spec.ts create mode 100644 packages/fs/tool-fs-search/tests/load-path.spec.ts create mode 100644 packages/fs/tool-fs-search/tests/shell-quote.spec.ts create mode 100644 packages/fs/tool-fs-search/tests/tools.spec.ts create mode 100644 packages/fs/tool-fs-search/tsconfig.json diff --git a/docs/config-catalog.md b/docs/config-catalog.md index c4a982e12a..01c8ead2f0 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -724,6 +724,28 @@ export interface Config { Source: [`packages/fs/tool-fs/src/index.ts:48`](../packages/fs/tool-fs/src/index.ts) +## `@deepseek-ai/dsh-tool-fs-search` + +Requires: `tools` · `systemPrompt` · `bash` + +```ts config-catalog +/** Plugin config (all optional — `Config` supplies the defaults). */ +export interface Config { + /** Max paths one `glob` call retains inline; later paths go to the formatted spill file. */ + globMaxResults?: number + /** Max flat matches one `grep` call retains inline; later matches go to the formatted spill file. */ + grepMaxMatches?: number + /** Max bytes retained for one matched-line preview (the cut preserves UTF-8 boundaries). */ + grepMaxLineBytes?: number + /** Max complete raw `rg` stdout bytes a search will parse; larger raw output fails with `SEARCH_RAW_OUTPUT_OVERFLOW`. */ + rawOutputMaxBytes?: number + /** Cooperative tool-call timeout budget (ms) on both tools, enforced by `@deepseek-ai/dsh-timeout-policy` through `exec.signal`. */ + timeoutMs?: number +} +``` + +Source: [`packages/fs/tool-fs-search/src/index.ts:59`](../packages/fs/tool-fs-search/src/index.ts) + ## `@deepseek-ai/dsh-tool-subagent` Requires: `tools` · `subagents` diff --git a/docs/module-graph.md b/docs/module-graph.md index c283bfe2a6..641566b03c 100644 --- a/docs/module-graph.md +++ b/docs/module-graph.md @@ -35,6 +35,7 @@ flowchart TD pkg_fs_local["fs-local"] pkg_fs_policy["fs-policy"] pkg_tool_fs["tool-fs"] + pkg_tool_fs_search["tool-fs-search"] end subgraph group_compact["packages/compact"] pkg_compact["compact"] @@ -161,6 +162,13 @@ flowchart TD pkg_tool_fs --> pkg_session pkg_tool_fs --> pkg_system_prompt pkg_tool_fs --> pkg_tools + pkg_tool_fs_search --> pkg_bash + pkg_tool_fs_search --> pkg_llm + pkg_tool_fs_search --> pkg_retention + pkg_tool_fs_search --> pkg_session + pkg_tool_fs_search --> pkg_spill + pkg_tool_fs_search --> pkg_system_prompt + pkg_tool_fs_search --> pkg_tools pkg_subagent --> pkg_agent pkg_subagent --> pkg_llm pkg_subagent --> pkg_tools @@ -278,6 +286,7 @@ flowchart TD | [`agent-loop`](../packages/core/agent-loop) | `core` | [`agent`](../packages/core/agent), [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`session-persistence`](../packages/session-persistence/session-persistence), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools) | | [`tool-bash`](../packages/bash/tool-bash) | `bash` | [`agent`](../packages/core/agent), [`bash`](../packages/bash/bash), [`llm`](../packages/llm/llm), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools) | | [`tool-fs`](../packages/fs/tool-fs) | `fs` | [`fs`](../packages/fs/fs), [`llm`](../packages/llm/llm), [`session`](../packages/core/session), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools) | +| [`tool-fs-search`](../packages/fs/tool-fs-search) | `fs` | [`bash`](../packages/bash/bash), [`llm`](../packages/llm/llm), [`retention`](../packages/util/retention), [`session`](../packages/core/session), [`spill`](../packages/spill/spill), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools) | | [`subagent`](../packages/subagent/subagent) | `subagent` | [`agent`](../packages/core/agent), [`llm`](../packages/llm/llm), [`tools`](../packages/core/tools) | | [`tool-web`](../packages/web/tool-web) | `web` | [`llm`](../packages/llm/llm), [`system-prompt`](../packages/core/system-prompt), [`tools`](../packages/core/tools), [`web`](../packages/web/web) | | [`spill-policy`](../packages/spill/spill-policy) | `spill` | [`llm`](../packages/llm/llm), [`retention`](../packages/util/retention), [`session`](../packages/core/session), [`spill`](../packages/spill/spill), [`tools`](../packages/core/tools) | diff --git a/docs/rfc/INDEX.md b/docs/rfc/INDEX.md index 475f3f01ae..c5febfe749 100644 --- a/docs/rfc/INDEX.md +++ b/docs/rfc/INDEX.md @@ -63,6 +63,7 @@ Generated by `pnpm run gen-rfc-index` from the RFC tree — never edit by hand; | [Subagent lifecycle enrichment — lastAssistantMessage (observe-only)](implemented/feature/2026-06-30-subagent-observe-enrich.md) | 2026-06-30 | | [Explicit model-facing tool order](implemented/feature/2026-07-06-explicit-tool-order.md) | 2026-07-06 | | [Repeat-tool-call guard plugin](implemented/feature/2026-07-08-repeat-tool-guard.md) | 2026-07-08 | +| [Bash-backed grep and glob discovery tools](implemented/feature/2026-07-09-bash-backed-grep-glob-discovery.md) | 2026-07-09 | ### Simplification diff --git a/docs/rfc/implemented/feature/2026-07-09-bash-backed-grep-glob-discovery.md b/docs/rfc/implemented/feature/2026-07-09-bash-backed-grep-glob-discovery.md new file mode 100644 index 0000000000..185444872f --- /dev/null +++ b/docs/rfc/implemented/feature/2026-07-09-bash-backed-grep-glob-discovery.md @@ -0,0 +1,168 @@ +# RFC: Bash-backed grep and glob discovery tools + +Status: implemented + +## Problem + +The harness needs model-facing `glob` and `grep` tools, but making them `ctx.fs` provider methods turns a local product convenience into a universal filesystem backend contract. Local workspace discovery is naturally a process-backed `rg` workflow; remote or virtual filesystem backends may expose their own search API, may not share a local `ripgrep` view, or may not support discovery at all. The v1 should not require every filesystem backend to implement search before the file read/write/edit seam has proven that need. + +Search output also has two distinct budgets. The tool needs enough raw `rg` output to compute a stable logical result, but the model should receive only a bounded preview plus a recovery path when the formatted result is larger than the inline budget. The generic spill policy only sees the final tool result, so it cannot recover matches that a search tool already omitted. Search therefore needs tool-owned retention and best-effort formatted-result spill. + +## Decision + +`glob` and `grep` are model-facing tools in `@deepseek-ai/dsh-tool-fs-search`, backed by the bash seam, not by new `ctx.fs` provider methods. The package registers model-facing filesystem discovery tools, but execution uses `ctx.bash.resolve(request)` followed by `ctx.bash.run(spec)` with fixed `rg` command templates assembled by the tool. The tool layer owns schemas, argument validation, shell quoting, result parsing, result formatting, retention, formatted-result spill handoff, and timeout declaration. The bash executor owns request defaulting/capping, subprocess execution, process-group termination, environment scrubbing, raw output capture, and backend substitution across local, sandboxed, or remote bash implementations. + +The tools do not use `ctx.bash.start()` and do not create model-visible background tasks. They run as ordinary foreground tools from the agent loop's perspective: the tool call returns only after the `rg` command exits, times out, is aborted, or fails. `defineTool({ timeoutMs })` declares the cooperative tool-call budget, `@deepseek-ai/dsh-timeout-policy` enforces it through `exec.signal`, and the tool forwards that signal into the bash request before `resolve()` / `run()`. The bash backend's own timeout remains a second safety cap; whichever aborts first wins. + +The tools align `path` with Claude Code's search tools while binding resolution to the bash workdir, not to `ctx.fs`. The tool derives the bash request workdir from `exec.agent?.session.header.cwd`, mirroring `dsh-tool-bash` and `dsh-tool-fs`; when no session cwd exists, it omits `request.workdir` so the bash implementation applies its configured cwd or process cwd through `resolve()`. For `grep`, `path` is an optional ripgrep target and may be a file or directory; omitted means the resolved bash workdir. For `glob`, `path` is an optional directory search root; omitted means the resolved bash workdir. Relative `path` values resolve against that workdir. Returned paths are displayed relative to the resolved bash workdir when possible and are intended to be follow-up-readable only in co-located deployments where the bash workdir and filesystem `read` root are the same workspace. v1 documents that deployment requirement but does not perform runtime cross-service validation. Remote or virtual filesystem search is deferred until there is a shared workspace/root contract or a provider-specific search backend. + +The package does not inject `fs`. It injects `tools`, `systemPrompt`, and `bash`; it deliberately reads `spillFiles` with `ctx.get('spillFiles')` instead of static inject because formatted-result spill is optional. Existing `@deepseek-ai/dsh-tool-fs` deployments that only want `read` / `write` / `edit` do not need to load bash. + +### Package shape + +The v1 package stays small. Inside `@deepseek-ai/dsh-tool-fs-search`, the source layout is: + +```text +src/index.ts +src/glob.ts +src/grep.ts +src/search-core.ts +src/shell-quote.ts +``` + +`glob.ts` and `grep.ts` own their parameter validation, command construction, result parsing, formatting, and registration. `shell-quote.ts` is one shared helper because shell quoting is the safety boundary both tools must use; `search-core.ts` is the other (an implementation-time amendment to the original four-file plan): the `SEARCH_*` error vocabulary, the bash-run + raw-output acquisition, the formatted-spill handoff, and workdir-relative display are byte-identical between the two tools, and duplicating that delicate plumbing per tool is exactly the missed extraction the symmetry convention flags. Command builders must not hand-roll quoting or concatenate unquoted model-controlled values into the shell command. + +### Schemas and config + +`glob` exposes the small discovery shape: + +```ts +interface GlobArgs { + pattern: string + path?: string +} +``` + +`grep` exposes the OpenCode-style minimal shape: + +```ts +interface GrepArgs { + pattern: string + path?: string + include?: string +} +``` + +Routine budgets stay out of the model-facing schema. `@deepseek-ai/dsh-tool-fs-search` owns these defaulted, validated config fields: + +| Field | Default | Role | +|---|---:|---| +| `globMaxResults` | `100` | Max paths retained inline; matches Claude Code's default `GlobTool` result limit. | +| `grepMaxMatches` | `250` | Max flat matches retained inline; matches Claude Code's default `GrepTool` `head_limit`. | +| `grepMaxLineBytes` | `2000` | Max bytes retained for one matched-line preview, applied with `TextRetainer({ kind: 'head', maxBytes: grepMaxLineBytes })`. | +| `rawOutputMaxBytes` | `20000000` | Max complete raw `rg` stdout the tool will parse; matches Claude Code's ripgrep raw buffer. | +| `timeoutMs` | `30000` | Tool-call timeout attached to both tool definitions and enforced by `@deepseek-ai/dsh-timeout-policy`. | + +`globMaxResults` and `grepMaxMatches` use `ItemRetainer({ kind: 'head' })`. `grepMaxLineBytes` uses `TextRetainer({ kind: 'head', maxBytes: grepMaxLineBytes })` for each matched line so preview cuts preserve UTF-8 boundaries. This follows the [tool result retention library](../../implemented/architecture/2026-07-06-tool-result-retention-library.md) mapping for discovery items: collect the complete result, retain head items inline, and keep path mapping, grouping, and per-line preview outside the retainer. `grep` does not expose `case_insensitive`, `head_limit`, `offset`, `count`, multiline, context lines, output modes, or file type filters in v1. A model that needs surrounding context reads the matched file with `read`; a model that needs later results reads the formatted spill file with `read offset/limit`. + +The Claude Code values are reference points for the two-layer budget, not model-facing schema precedent. Its dedicated search tools buffer raw ripgrep output up to 20 MB for internal processing, use a 20-second ripgrep timeout on non-WSL platforms (60 seconds on WSL), then apply search-specific caps before the model sees a result: `GrepTool` defaults to `head_limit = 250` and persists formatted results above 20,000 characters, while `GlobTool` defaults to 100 paths and persists formatted results above 100,000 characters. This RFC mirrors the raw-buffer and inline-count defaults, chooses a 30-second default search timeout, and uses this harness's `ctx.spillFiles.saveText()` path for formatted-result recovery. + +The `path` field follows the same split as Claude Code: `grep.path` is a file-or-directory ripgrep target, while `glob.path` is a directory search root. v1 does not expose a separate cwd/workdir argument on these tools. + +`include` is one positive glob filter, not a list and not an exclude syntax. Reject comma-separated or negated include patterns up front with a structured argument error. Every model-controlled value used in a shell command, including `pattern`, `path`, and `include`, must pass through the package-private shell quoting helper. + +### Execution + +`glob` builds a fixed `rg --files` command rooted at the resolved directory search root (`path` when supplied, else the bash workdir): `rg --files --glob --sort=modified --no-ignore --hidden`, plus VCS metadata excludes for `.git`, `.svn`, `.hg`, `.bzr`, `.jj`, and `.sl`. This aligns with Claude Code on hidden/ignored-file discovery and modified-time ordering while keeping VCS internals out of broad searches. The tool parses one path per line, maps results back to paths relative to the bash workdir when possible, pushes each path into `ItemRetainer({ kind: 'head', maxItems: globMaxResults })`, and formats the full sorted path list for a spill file when the retained result is capped. + +`grep` builds a fixed line-oriented `rg --json` command against the supplied file/directory target (`path` when supplied, else the bash workdir) so file path, line number, and line text are parsed without colon-splitting ambiguity. It consumes `match` records, treats malformed JSON or malformed match records as `SEARCH_FAILED`, maps result paths relative to the bash workdir when possible, applies per-line preview retention with `grepMaxLineBytes`, pushes each match into `ItemRetainer({ kind: 'head', maxItems: grepMaxMatches })`, then groups only the retained preview matches by file for inline output. The spill file stores the full formatted match list, not only the omitted tail, so `read offset/limit` works against the same logical result the model saw. + +Raw `rg` stdout is an internal transport detail. If `ctx.bash.run()` returns untruncated stdout, the tool parses `stdout.text`. If stdout is truncated and `stdout.spillPath` is present, the tool reads that local raw spill file up to `rawOutputMaxBytes + 1` bytes and parses it only when the complete file fits within `rawOutputMaxBytes`. If the raw spill file is larger than `rawOutputMaxBytes`, or stdout is truncated without a spill path, the tool fails with a clear search error telling the model to narrow `pattern`, `path`, or `include`. The tool never exposes raw `rg` output or bash raw spill paths to the model. + +Only stdout is a parse source. Stderr is diagnostic text for invalid patterns, missing `rg`, and search failures; if bash truncates stderr, the tool uses the retained stderr tail with a truncation note and does not read `stderr.spillPath`. + +If `ctx.bash.run()` reports `aborted` because the tool timeout or caller cancellation fired, the tool returns a structured failure rather than pretending there were no matches. If bash reports its own timeout first, the tool likewise fails with a clear timeout message. Nonzero ripgrep exit semantics are tool-owned: exit 0 is success with matches, exit 1 is success with no matches, invalid pattern / missing `rg` / inaccessible search workdir are failures. + +Search failures use a package-owned `HarnessError` subclass with `SEARCH_*` codes, not `FsErrorCode`, because these tools are not `ctx.fs` provider operations. The v1 vocabulary is `SEARCH_INVALID_PATTERN`, `SEARCH_FAILED`, `SEARCH_RAW_OUTPUT_OVERFLOW`, and `SEARCH_ABORTED`. Model argument validation failures such as missing required fields, blank strings, or unsupported negated/list `include` values remain ordinary tool argument errors. + +### Formatted result spill + +`ctx.spillFiles` is optional and used only for model-facing formatted results. This is the first tool-owned spill call pattern in the codebase, and it is intentional because search retention is item-level policy: `globMaxResults` caps paths and `grepMaxMatches` caps matches while the tool still holds the complete logical result. The generic `dsh-spill-policy` caps final text bytes on `tools/post-execute`; by then a search tool would already have omitted later paths or matches, so the policy cannot recover them. + +When a search produces more logical results than the inline cap and `ctx.spillFiles` is present, the tool saves the complete formatted result with `saveText()`. The spill owner is the calling agent's session header id (`exec.agent?.session.header.id`); without that owner, the search keeps the inline result and reports that the complete result could not be saved. The spill source is the tool execution identity: `{ toolName: exec.name, callId: exec.callId, label: 'result' }`. The suggested filenames are `grep-results.txt` and `glob-results.txt`; the spill backend still sanitizes them as hints, never paths. + +When spill storage is absent, the call has no session owner, or saving fails, the tool still returns the inline page and a footer explaining that the complete result could not be saved. Search success must not turn into an `isError` result solely because formatted-result spill storage is unavailable. + +The bash raw spill file and the formatted search spill file are different artifacts. The raw bash spill file is a local executor implementation detail used only so the search tool can parse complete `rg` stdout. The formatted spill file is the stable model-facing recovery path produced by `ctx.spillFiles.saveText()`. + +### Result shape + +A capped `glob` result with successful formatted spill returns the inline page and a spill notice: + +```text + + +(Showing N of M paths. Full sorted result saved to: /.../session-abc123/9f8e7d-glob-results.txt. Use read with offset/limit to inspect it.) +``` + +A capped `grep` result with successful formatted spill returns grouped preview matches and a spill notice: + +```text +Found N of M matches + + +Line 12: ... + +(Full grep result saved to: /.../session-abc123/9f8e7d-grep-results.txt. Use read with offset/limit to inspect it.) +``` + +If the complete logical result fits under the inline cap, no formatted spill file is created. If the complete logical result is too large but formatted spill is unavailable, the footer says that the result was capped and the complete result could not be saved. The `truncated` / omitted count is a budget fact, not an incomplete-search fact; timeout, invalid regex, missing `rg`, inaccessible workdirs, raw-output overflow, binary skips, and parse failures stay in tool-domain error or incomplete fields. + +## Alternatives considered + +**Put `glob` / `grep` on `ctx.fs`.** Rejected for v1: it forces every filesystem backend to grow a search API and makes local ripgrep behavior part of the provider seam. Search is useful product behavior, but it is not a universal text-storage primitive like `readText` or `writeText`. + +**Directly spawn ripgrep from `dsh-fs-local`.** Rejected for this RFC's v1: direct spawn gives the cleanest argv boundary, stdout/stderr control, and early-stop control, but it duplicates process execution concerns that the bash seam already owns: environment scrubbing, process-group kill, timeout propagation, sandbox/remote executor substitution, and raw output spill. It remains a reasonable optimization if bash-backed search proves too shell-string-sensitive or if raw bash spill recovery is not portable enough. + +**Use `ctx.bash.start()` for streaming early stop.** Rejected: `start()` creates model-visible background task semantics: task ids, owner tokens, `bash_output`, `bash_kill`, completion notifications, and no built-in timeout. `grep` needs a foreground tool result, not a background bash workflow. If streaming search becomes necessary, the right abstraction is a foreground streaming process handle on the bash/process seam, not borrowing the public background-task API. + +**Expose bash raw spill paths to the model.** Rejected: a bash raw spill path contains raw `rg` stdout (`rg --json` records for grep), not the stable formatted search result. The search tool may read raw spill internally, but model recovery uses a formatted result saved through `ctx.spillFiles.saveText()`. + +**Add `spillFiles.saveFile()` for bash output normalization first.** Rejected for this RFC's v1: `saveFile()` would help a future bash normalization pass move existing executor spill files into session-scoped spill storage, but search still has to parse raw `rg` output before producing the model-facing artifact. `saveText()` is sufficient for the formatted search result, and raw bash spill remains an executor-local recovery detail. + +**Rely on the generic `dsh-spill-policy`.** Rejected: generic post-execute spill sees only the final tool result. If `grep` / `glob` return the first page inline, the generic policy cannot recover omitted results. The search tools must save the complete formatted result themselves before returning the bounded model-facing text. + +**Expose Claude Code's full `GrepTool` schema.** Rejected for v1: `output_mode`, context flags, multiline, `head_limit`, `offset`, `case_insensitive`, and type filters make the model-facing surface into a ripgrep wrapper. This harness keeps routine budgets and continuation mechanics in deployment policy and spill files. + +**Keep early-stop search and skip formatted spill files.** Rejected for this proposal: early stop is more efficient but gives the model no path to inspect later results. The chosen v1 optimizes result recoverability and implementation simplicity, with `timeoutMs`, `rawOutputMaxBytes`, bash backend caps, and formatted spill files as safety backstops. + +**Expand the bash seam with a raw-output reader first.** Deferred: a remote bash backend may eventually need a portable `readRawOutput(ref, maxBytes)` style API instead of local `spillPath` reads. v1 uses the existing local-readable `stdout.spillPath` to avoid widening the bash seam for one consumer. + +## Testing + +- Tests prove an aborted `exec.signal` reaches the bash backend (same-reference spec assertion plus the `SEARCH_ABORTED` result), and cover command construction/quoting (malicious patterns, paths with spaces, leading-dash values, quotes, newlines, glob metacharacters — unit assertions plus a real `bash -c` round-trip for every hostile value), `grep.path` as file and directory targets, `glob.path` as a directory search root, invalid pattern handling, no matches, malformed `rg --json` output, matched-line preview truncation, raw-output overflow, timeout/abort, formatted spill success/failure, the package-owned `SEARCH_*` error codes, and the no-background-task invariant. +- The first-party tool-owned spill precedent is covered directly: spill backend present, spill backend absent, `saveText()` failure, and missing spill owner. +- The package has real Loader-path coverage for the namespace plugin export shape (`name`, `inject`, `Config`, and `apply`, with no default export). +- A real-executor integration suite (`dsh-bash-local` + a real `rg`) verifies the world: hostile patterns stay inert, per-session cwd resolution, VCS-metadata exclusion, modification-time ordering, and real ripgrep stderr classification. It self-skips where `rg` is not on PATH (a CI accommodation mirroring the keyless e2e skip); the fake-executor suite alone carries the per-file 100% coverage gate. +- Snapshot gap note for the transcript-visible spill notice: this landed with the gap note, not a snapshot. The snapshot tier replays the acp-agent tree, and adding the search plugin there changes the assembled system prompt — every golden would need re-recording with a real key, which the implementing environment did not hold. The spill notice's exact transcript text is pinned by unit tests (`formatGlobOutput`/`formatGrepOutput` and the through-the-registry spill tests); wiring the plugin into the acp-agent tree plus a `test:snapshot:record` pass is the follow-up for the next key-holding session. + +## Consequences + +- `glob` and `grep` are model-facing tools in `@deepseek-ai/dsh-tool-fs-search`, not `ctx.fs` provider methods and not part of the existing `@deepseek-ai/dsh-tool-fs` root plugin. The package injects `tools`, `systemPrompt`, and `bash`; it does not inject `fs`, and `ctx.spillFiles` stays optional via `ctx.get('spillFiles')`. +- The schemas are exactly `glob(pattern, path?)` and `grep(pattern, path?, include?)`; search caps and timeout are defaulted, validated Config fields (`globMaxResults`, `grepMaxMatches`, `grepMaxLineBytes`, `rawOutputMaxBytes`, `timeoutMs`). +- The tools execute through `ctx.bash.resolve(request)` → `ctx.bash.run(spec)`, forward `exec.signal`, never call `ctx.bash.start()`, and never expose a bash task id. The bash request workdir comes from `exec.agent?.session.header.cwd` when available; the resolved `spec.workdir` drives execution and relative-path display. +- When bash stdout is truncated, the tools parse the full raw stdout only through a local `stdout.spillPath` that fits within `rawOutputMaxBytes`; missing spill paths or over-cap raw output are clear search failures, and raw `rg` output is never exposed to the model. +- Oversized complete formatted results are saved through `ctx.spillFiles.saveText()` when available while inline results stay bounded; spill failure, a missing backend, or a missing owner preserves the inline result and reports the unsaved remainder — never an `isError`. +- The package README, the generated config catalog, and exported JSDoc document the Config fields and `SEARCH_*` codes; the coding-agent example ships the tools (the acp-agent tree waits on the snapshot re-record above); the fs group README records the co-located bash/filesystem deployment requirement. + +## Risks + +Full-run `grep` can be slower than an early-stop search on broad patterns. The v1 accepts that cost for simpler implementation and complete-result recovery, bounded by tool timeout, bash timeout, `rawOutputMaxBytes`, and output caps. If this proves too slow, the direct-ripgrep or foreground-streaming alternatives remain available. + +Shell command construction is the sharpest safety edge. Because `ctx.bash` accepts a command string rather than an argv vector, the implementation must centralize shell quoting and test malicious patterns, paths with spaces, leading-dash patterns, quotes, newlines, and glob metacharacters. + +The v1 assumes a co-located bash/filesystem deployment. If bash searches one workspace and the `read` tool resolves paths against another, returned paths may not be follow-up-readable. The package documents this requirement but does not verify it at runtime. + +Raw bash spill recovery is local-path-shaped in v1. A remote or sandboxed bash backend may return no readable `spillPath` or may require a future raw-output read API. In that case broad searches fail clearly instead of pretending a truncated raw result is complete. + +Spill paths are local filesystem paths in v1. The formatted-result design works for local deployments where `read` can open spill files; remote or workspace-confined deployments need either an allowlist for spill paths or a future virtual spill URI bridge. diff --git a/docs/tool-catalog.md b/docs/tool-catalog.md index 3645ff40e9..824d86bfdd 100644 --- a/docs/tool-catalog.md +++ b/docs/tool-catalog.md @@ -17,6 +17,7 @@ This table connects model-visible tool names to the plugin package and service s | --- | --- | --- | --- | --- | --- | | `@deepseek-ai/dsh-tool-bash` | `bash`, `bash_kill`, `bash_output` | `ctx.tools`, `ctx.bash` | `tool/call`, `tool/result`, `context/message via agent.inject() for background completion notices` | - | The bash/bash_output/bash_kill tools are model-facing consumers of the bash executor seam. | | `@deepseek-ai/dsh-tool-fs` | `edit`, `read`, `write` | `ctx.tools`, `ctx.fs`, `ctx.systemPrompt` | `tool/call`, `fs/write-intent or fs/edit-intent for mutations`, `fs/observed after successful file operations`, `tool/result` | - | The read-before-write/edit policy is added by `@deepseek-ai/dsh-fs-policy` (an `fs/*` event-gate plugin, no schema change); a deployment that loads these tools is expected to also load it. The tool schemas above are identical with or without the policy plugin. | +| `@deepseek-ai/dsh-tool-fs-search` | `glob`, `grep` | `ctx.tools`, `ctx.bash`, `ctx.systemPrompt` | `tool/call`, `tool/result` | - | glob and grep are bash-backed discovery tools: they run fixed ripgrep commands through ctx.bash as ordinary foreground calls (never background tasks). Capped results save the complete formatted list through the optional ctx.spillFiles backend; returned paths are follow-up-readable in co-located bash/filesystem deployments. | | `@deepseek-ai/dsh-tool-subagent` | `subagent` | `ctx.tools`, `ctx.subagents` | `tool/call`, `tool/result`, `child session events through the chosen provider` | `subagent`, `subagent_fork` | The registered tool name is the load-time `toolName` config (default `subagent`); the schema above is that default. The shipped example agents load this package once per subagent backend, so the model additionally sees `subagent_fork` (bound to the fork backend) with an identical schema — see `examples/coding-agent/cordis.yml` and `examples/acp-agent/cordis.yml`. | | `@deepseek-ai/dsh-tool-todo` | `todo_write` | `ctx.tools`, `owning Agent session` | `tool/call`, `todo/write`, `tool/result` | - | todo_write is session-owned state; UIs render the latest todo/write event as a checklist or ACP plan. | | `@deepseek-ai/dsh-tool-web` | `web_fetch`, `web_search` | `ctx.tools`, `ctx.web`, `ctx.systemPrompt` | `tool/call`, `tool/result` | - | web_search and web_fetch keep provider selection behind ctx.web so model-visible schemas stay stable across backend swaps. | @@ -199,6 +200,64 @@ Source: [`packages/fs/tool-fs/src/index.ts`](../packages/fs/tool-fs/src/index.ts The read-before-write/edit policy is added by `@deepseek-ai/dsh-fs-policy` (an `fs/*` event-gate plugin, no schema change); a deployment that loads these tools is expected to also load it. The tool schemas above are identical with or without the policy plugin. +## `@deepseek-ai/dsh-tool-fs-search` + +### `glob` + +Find files whose paths match a glob pattern. Returns matching paths sorted by modification time, including hidden and ignored files (VCS metadata directories are excluded). Returns the first 100 paths inline; a capped result reports where the complete list was saved. + +```json +{ + "type": "object", + "properties": { + "pattern": { + "type": "string", + "description": "Glob pattern to match file paths against (e.g. \"**/*.ts\", \"src/**/*.test.js\")." + }, + "path": { + "type": "string", + "description": "Directory to search in. Defaults to the session workspace; a relative path resolves against it." + } + }, + "required": [ + "pattern" + ] +} +``` + +Source: [`packages/fs/tool-fs-search/src/index.ts`](../packages/fs/tool-fs-search/src/index.ts) + +### `grep` + +Search file contents with a ripgrep regular expression. Returns matching lines with line numbers, grouped by file. Returns the first 250 matches inline; a capped result reports where the complete match list was saved. Use read on a matched file for surrounding context. + +```json +{ + "type": "object", + "properties": { + "pattern": { + "type": "string", + "description": "Regular expression to search for (ripgrep syntax)." + }, + "path": { + "type": "string", + "description": "File or directory to search. Defaults to the session workspace; a relative path resolves against it." + }, + "include": { + "type": "string", + "description": "One glob filter for which files to search (e.g. \"*.ts\", \"*.{js,jsx}\"). Not a list; negation is not supported." + } + }, + "required": [ + "pattern" + ] +} +``` + +Source: [`packages/fs/tool-fs-search/src/index.ts`](../packages/fs/tool-fs-search/src/index.ts) + +glob and grep are bash-backed discovery tools: they run fixed ripgrep commands through ctx.bash as ordinary foreground calls (never background tasks). Capped results save the complete formatted list through the optional ctx.spillFiles backend; returned paths are follow-up-readable in co-located bash/filesystem deployments. + ## `@deepseek-ai/dsh-tool-subagent` ### `subagent` diff --git a/examples/coding-agent/composition.md b/examples/coding-agent/composition.md index 979737d05f..c6c624f0fb 100644 --- a/examples/coding-agent/composition.md +++ b/examples/coding-agent/composition.md @@ -43,6 +43,8 @@ flowchart LR cfg --> plugin_coding_fs_policy plugin_coding_tool_fs["tool-fs
@deepseek-ai/dsh-tool-fs"] cfg --> plugin_coding_tool_fs + plugin_coding_tool_fs_search["tool-fs-search
@deepseek-ai/dsh-tool-fs-search"] + cfg --> plugin_coding_tool_fs_search plugin_coding_spill_local["spill-local
@deepseek-ai/dsh-spill-local"] cfg --> plugin_coding_spill_local plugin_coding_spill_policy["spill-policy
@deepseek-ai/dsh-spill-policy"] @@ -65,6 +67,7 @@ flowchart LR | `fs-local` | `@deepseek-ai/dsh-fs-local` | | `fs-policy` | `@deepseek-ai/dsh-fs-policy` | | `tool-fs` | `@deepseek-ai/dsh-tool-fs` | +| `tool-fs-search` | `@deepseek-ai/dsh-tool-fs-search` | | `spill-local` | `@deepseek-ai/dsh-spill-local` | | `spill-policy` | `@deepseek-ai/dsh-spill-policy` | diff --git a/examples/coding-agent/cordis.yml b/examples/coding-agent/cordis.yml index c740aa2907..5861e6a602 100644 --- a/examples/coding-agent/cordis.yml +++ b/examples/coding-agent/cordis.yml @@ -121,6 +121,12 @@ - id: tool-fs name: '@deepseek-ai/dsh-tool-fs' +# Bash-backed discovery tools (glob/grep): fixed ripgrep commands through the +# local bash executor above — not ctx.fs. Capped results save the complete +# formatted list through the spill backend below (ctx.spillFiles, optional). +- id: tool-fs-search + name: '@deepseek-ai/dsh-tool-fs-search' + # Tool-output spill stack: a local backend that saves oversized tool text under # a private session-scoped dir, and the tools/post-execute policy that replaces # an over-budget plain-text result with a preview + the spill path (the model diff --git a/knip.json b/knip.json index 4b63fa01c4..18870b3f0b 100644 --- a/knip.json +++ b/knip.json @@ -83,6 +83,11 @@ "packages/fs/tool-fs": { "entry": ["tests/**/*.spec.ts", "tests/**/*.e2e.ts"], "project": ["src/**/*.ts", "tests/**/*.ts"] + }, + "packages/fs/tool-fs-search": { + "entry": ["tests/**/*.spec.ts"], + "project": ["src/**/*.ts", "tests/**/*.ts"], + "ignoreBinaries": ["rg"] } } } diff --git a/packages/README.md b/packages/README.md index 7060760311..fcb7b08cb4 100644 --- a/packages/README.md +++ b/packages/README.md @@ -12,7 +12,7 @@ Packages are grouped by role at `packages///`. The group directory i | [`llm/`](llm/README.md) | LLM capability family: the abstract service + provider adapters | Product — stable surface | | [`bash/`](bash/README.md) | Bash capability family: the executor seam, a local impl, and the model-facing tool | Product — stable surface | | [`code-runtime/`](code-runtime/README.md) | Code-execution capability family: the abstract runtime seam for model-written programs + a worker-thread backend | Product — stable surface | -| [`fs/`](fs/README.md) | Filesystem capability family: the abstract seam, a local impl, and the model-facing file tools | Product — stable surface | +| [`fs/`](fs/README.md) | Filesystem capability family: the abstract seam, a local impl, the model-facing file tools, and the bash-backed discovery tools | Product — stable surface | | [`compact/`](compact/README.md) | Compaction capability family: the abstract seam + a basic backend (tool deferred) | Product — stable surface | | [`subagent/`](subagent/README.md) | Subagent capability family: the provider-registry seam and the model-facing delegation tool | Product — stable surface | | [`web/`](web/README.md) | Web capability family: the abstract seam, search/fetch provider impls, and the model-facing web tools | Product — stable surface | diff --git a/packages/core/tools/tests/gen-tool-catalog.spec.ts b/packages/core/tools/tests/gen-tool-catalog.spec.ts index cacc2eef66..57d3e5dbf1 100644 --- a/packages/core/tools/tests/gen-tool-catalog.spec.ts +++ b/packages/core/tools/tests/gen-tool-catalog.spec.ts @@ -35,7 +35,7 @@ describe('gen-tool-catalog collectToolCatalog', () => { it('boots every shipped tool package and harvests its model-facing schemas', async () => { const catalog = await collectToolCatalog() const names = catalog.flatMap(entry => entry.schemas.map(s => s.name)).sort() - expect(names).toEqual(['bash', 'bash_kill', 'bash_output', 'edit', 'read', 'subagent', 'todo_write', 'web_fetch', 'web_search', 'write']) + expect(names).toEqual(['bash', 'bash_kill', 'bash_output', 'edit', 'glob', 'grep', 'read', 'subagent', 'todo_write', 'web_fetch', 'web_search', 'write']) // Every tool carries a JSON-Schema `parameters` object (what the model sees). for (const entry of catalog) { for (const schema of entry.schemas) { diff --git a/packages/fs/README.md b/packages/fs/README.md index ec3bb62afb..039cb39ae9 100644 --- a/packages/fs/README.md +++ b/packages/fs/README.md @@ -1,6 +1,6 @@ # fs/ - filesystem capability family -The filesystem stack: a provider seam (text IO + atomic mutation with an optional version guard), a local implementation, a policy gate plugin (observed-state + read-before-edit + version-guarded write/edit), and the model-facing file tools + executor. All **product** packages. +The filesystem stack: a provider seam (text IO + atomic mutation with an optional version guard), a local implementation, a policy gate plugin (observed-state + read-before-edit + version-guarded write/edit), the model-facing file tools + executor, and the bash-backed discovery tools. All **product** packages. | Package | Role | ctx key | |---|---|---| @@ -8,9 +8,10 @@ The filesystem stack: a provider seam (text IO + atomic mutation with an optiona | `fs-local/` | Local-filesystem `FileSystem` implementation | (registers `ctx.fs`) | | `fs-policy/` | Policy gate plugin: observed-state + read-before-edit + version-guarded write/edit, via the `fs/*` event gate | (no service — `fs/*` listeners) | | `tool-fs/` | Model-facing `read`/`write`/`edit` tools AND the executor (reads via `ctx.fs`, owns read windowing, dispatches `fs/*`) | (registers on `ctx.tools`) | +| `tool-fs-search/` | Model-facing `glob`/`grep` discovery tools, backed by fixed ripgrep commands through the bash seam (`ctx.bash`), NOT by `ctx.fs` provider methods | (registers on `ctx.tools`) | -The interface lives at `fs/fs/`. A sandboxed, remote, or project-scoped filesystem backend can replace `fs-local` without touching the seam, the policy gate, or the model-facing tool schemas. The policy (`fs-policy/`) is a plugin that participates only through the `fs/*` event gate, not a service the tool injects — so dropping it gracefully loses the policy and leaves the unconstrained bare provider rather than breaking the tool. A deployment that loads `tool-fs/` is expected to also load it. +The interface lives at `fs/fs/`. A sandboxed, remote, or project-scoped filesystem backend can replace `fs-local` without touching the seam, the policy gate, or the model-facing tool schemas. The policy (`fs-policy/`) is a plugin that participates only through the `fs/*` event gate, not a service the tool injects — so dropping it gracefully loses the policy and leaves the unconstrained bare provider rather than breaking the tool. A deployment that loads `tool-fs/` is expected to also load it. Discovery (`tool-fs-search/`) deliberately does NOT extend the provider seam: search is a process-backed `rg` workflow on the bash executor, so filesystem backends stay free of a universal search contract; its results are follow-up-readable when the bash workdir and the `read` root are the same workspace (the co-located deployment its README documents). ## No timeouts on file IO -`read`/`write`/`edit` take **no** `timeoutMs`, and the provider seam arms no deadline — unlike bash and web, which consume [`@deepseek-ai/dsh-timeout`](../util/timeout/README.md). A local syscall is best-effort-abortable at most: a timeout could not force an in-progress `fsync`/`rename` to stop, so a deadline here would be a knob that cannot deliver on its promise. Adding one would also be an implicit default in the exact place explicit-over-implicit forbids. Both reference agents (Claude Code, Codex) leave file IO untimed for the same reason; cancellation still propagates through the tool-execution signal for best-effort abort at syscall boundaries. +`read`/`write`/`edit` take **no** `timeoutMs`, and the provider seam arms no deadline — unlike bash and web (which consume [`@deepseek-ai/dsh-timeout`](../util/timeout/README.md)) and the bash-backed `glob`/`grep` (whose declared `timeoutMs` is enforced by `@deepseek-ai/dsh-timeout-policy`): those are process-backed, where a deadline can really kill the work. A local syscall is best-effort-abortable at most: a timeout could not force an in-progress `fsync`/`rename` to stop, so a deadline here would be a knob that cannot deliver on its promise. Adding one would also be an implicit default in the exact place explicit-over-implicit forbids. Both reference agents (Claude Code, Codex) leave file IO untimed for the same reason; cancellation still propagates through the tool-execution signal for best-effort abort at syscall boundaries. diff --git a/packages/fs/tool-fs-search/README.md b/packages/fs/tool-fs-search/README.md new file mode 100644 index 0000000000..e1ed680060 --- /dev/null +++ b/packages/fs/tool-fs-search/README.md @@ -0,0 +1,46 @@ +# @deepseek-ai/dsh-tool-fs-search + +The **model-facing filesystem discovery tools** — `glob`, `grep` — backed by the **bash executor seam**, not by `ctx.fs` provider methods. Each call assembles a fixed ripgrep command (every model-controlled value through one package-private shell-quoting helper), runs it via `ctx.bash.resolve(request)` → `ctx.bash.run(spec)` as an ordinary foreground tool call, parses the raw `rg` output, and returns a bounded, workdir-relative result. The package injects `tools`, `systemPrompt`, and `bash` — deliberately **not** `fs`; `ctx.spillFiles` is read opportunistically with `ctx.get()` because formatted-result spill is optional. + +```ts ignore-check +// Default deployment: a bash executor, then the discovery tools. +await ctx.plugin(LocalBashExecutor, { cwd: process.cwd() }) // @deepseek-ai/dsh-bash-local +await ctx.plugin(ToolFsSearch) // this package — registers glob/grep +// Optional: a spill backend makes capped results fully recoverable. +await ctx.plugin(LocalSpillFiles) // @deepseek-ai/dsh-spill-local +``` + +Why bash-backed: local workspace discovery is naturally a process-backed `rg` workflow, and putting search on `ctx.fs` would force every filesystem backend to grow a search API. The bash executor owns request defaulting/capping, subprocess execution, process-group termination, environment scrubbing, raw output capture, and backend substitution (local, sandboxed, remote); this package owns schemas, argument validation, shell quoting, parsing, retention, formatted-result spill, and timeout declaration. The tools never call `ctx.bash.start()` and never expose a bash task id — the call returns only after `rg` exits, times out, is aborted, or fails. + +## Deployment requirement: co-located bash + filesystem + +Returned paths are displayed relative to the resolved bash workdir (the calling agent's session cwd when present, else the executor's configured default) and are follow-up-readable with `read` only when the bash workdir and the filesystem root are the same workspace. v1 documents that requirement and performs no runtime cross-service validation; remote or virtual filesystem search waits for a shared workspace contract or a provider-specific search backend. + +## Config + +All keys are optional; the defaults are the shipped search caps. + +| Key | Default | Meaning | +|---|---|---| +| `globMaxResults` | `100` | Max paths one `glob` call retains inline (matches Claude Code's `GlobTool` limit); later paths go to the formatted spill file. | +| `grepMaxMatches` | `250` | Max flat matches one `grep` call retains inline (matches Claude Code's `GrepTool` `head_limit`); later matches go to the formatted spill file. | +| `grepMaxLineBytes` | `2000` | Byte cap per matched-line preview; the cut preserves UTF-8 boundaries and is marked `(line truncated)`. | +| `rawOutputMaxBytes` | `20000000` | Max complete raw `rg` stdout a search will parse (matches Claude Code's ripgrep raw buffer); larger raw output fails with `SEARCH_RAW_OUTPUT_OVERFLOW`. | +| `timeoutMs` | `30000` | Cooperative tool-call budget attached to both tool definitions, enforced by `@deepseek-ai/dsh-timeout-policy` through `exec.signal`; the bash backend's own timeout stays a second safety cap. | + +## Tools + +| Tool | Arguments | Behavior | +|---|---|---| +| `glob` | `pattern`, `path?` | `rg --files --glob --sort=modified --no-ignore --hidden` plus VCS metadata excludes (`.git`, `.svn`, `.hg`, `.bzr`, `.jj`, `.sl`). `path` is an optional **directory** search root; omitted means the resolved bash workdir. Returns one path per line, modification-time ordered. | +| `grep` | `pattern`, `path?`, `include?` | Line-oriented `rg --json` parse (no colon-splitting ambiguity). `pattern` is a ripgrep regex; `path` is an optional **file or directory** target; `include` is ONE positive glob filter — a comma-separated list or a negated (`!…`) value is rejected up front (brace alternation like `*.{ts,tsx}` is fine). Returns matches grouped by file as `Line N: `. | + +Routine budgets stay out of the model-facing schema (no `head_limit`/`offset`/`case_insensitive`/output modes): a model that needs surrounding context reads the matched file with `read`; one that needs later results reads the formatted spill file with `read offset/limit`. + +## Two budgets, two artifacts + +Raw `rg` stdout is an internal transport detail. When the executor truncates it, the tool recovers the complete stream from the executor's **raw bash spill file** — read locally, capped at `rawOutputMaxBytes`, never shown to the model. The model-facing recovery artifact is different: when a search yields more logical results than the inline cap, the tool saves the COMPLETE formatted result through `ctx.spillFiles.saveText()` (suggested names `glob-results.txt` / `grep-results.txt`, owner = the calling session, source = the tool execution identity) and appends a footer naming the saved path. This is the first tool-owned spill call in the codebase — deliberate, because retention here is item-level: the generic `@deepseek-ai/dsh-spill-policy` only sees the final text on `tools/post-execute`, by which point a capped search has already omitted later paths/matches. A missing spill backend, a call with no session owner, or a `saveText()` failure keeps the inline page and reports that the complete result could not be saved — never an `isError`. + +## Errors + +Search failures carry the package-owned `SearchError` (a `HarnessError` subclass), surfaced as `{ name, code }` on `isError` results: `SEARCH_INVALID_PATTERN` (ripgrep rejected the regex/glob), `SEARCH_FAILED` (missing `rg`, inaccessible target, signal kill, malformed `--json` output), `SEARCH_RAW_OUTPUT_OVERFLOW` (raw output over `rawOutputMaxBytes`, or truncated with no recovery file), and `SEARCH_ABORTED` (tool timeout, caller cancellation, or the bash executor's own timeout). ripgrep exit semantics are tool-owned: exit 0 is success with results, exit 1 is a successful empty search (`No files found` / `No matches found`), and only other exits are failures. Model argument mistakes (blank pattern, a list-valued `include`) stay ordinary tool argument errors. diff --git a/packages/fs/tool-fs-search/package.json b/packages/fs/tool-fs-search/package.json new file mode 100644 index 0000000000..002d54569a --- /dev/null +++ b/packages/fs/tool-fs-search/package.json @@ -0,0 +1,49 @@ +{ + "name": "@deepseek-ai/dsh-tool-fs-search", + "description": "Model-facing filesystem discovery tools (glob, grep) backed by the DeepSeek Harness bash seam (ctx.bash)", + "version": "0.0.1", + "private": true, + "type": "module", + "main": "lib/index.js", + "types": "lib/types/index.d.ts", + "exports": { + ".": { + "types": "./lib/types/index.d.ts", + "default": "./lib/index.js" + }, + "./src/*": "./src/*", + "./package.json": "./package.json" + }, + "files": [ + "lib/index.js", + "lib/types/**/*.d.ts", + "lib/types/**/*.d.ts.map", + "src" + ], + "license": "BSD-3-Clause", + "dependencies": { + "schemastery": "^3.18.0" + }, + "peerDependencies": { + "@deepseek-ai/dsh-bash": "^0.0.1", + "@deepseek-ai/dsh-llm": "^0.0.1", + "@deepseek-ai/dsh-retention": "^0.0.1", + "@deepseek-ai/dsh-session": "^0.0.1", + "@deepseek-ai/dsh-spill": "^0.0.1", + "@deepseek-ai/dsh-system-prompt": "^0.0.1", + "@deepseek-ai/dsh-tools": "^0.0.1", + "cordis": "^4.0.0-rc.6" + }, + "devDependencies": { + "@deepseek-ai/dsh-agent": "workspace:^", + "@deepseek-ai/dsh-bash": "workspace:^", + "@deepseek-ai/dsh-bash-local": "workspace:^", + "@deepseek-ai/dsh-llm": "workspace:^", + "@deepseek-ai/dsh-retention": "workspace:^", + "@deepseek-ai/dsh-session": "workspace:^", + "@deepseek-ai/dsh-spill": "workspace:^", + "@deepseek-ai/dsh-system-prompt": "workspace:^", + "@deepseek-ai/dsh-tools": "workspace:^", + "cordis": "^4.0.0-rc.6" + } +} diff --git a/packages/fs/tool-fs-search/src/glob.ts b/packages/fs/tool-fs-search/src/glob.ts new file mode 100644 index 0000000000..e565699e2c --- /dev/null +++ b/packages/fs/tool-fs-search/src/glob.ts @@ -0,0 +1,168 @@ +/** + * The model-facing `glob` tool: discover files whose paths match a glob + * pattern, sorted by modification time. Execution goes through the bash seam + * (`ctx.bash`) with a fixed `rg --files` command — this module owns the + * model-facing schema, argument validation, shell-safe command construction, + * result parsing, retention, and formatting; process concerns (defaulting, + * scrubbing, kill, backend substitution) stay behind `ctx.bash`. + * + * @module @deepseek-ai/dsh-tool-fs-search/glob + */ + +import type { Context } from 'cordis' +import { defineTool } from '@deepseek-ai/dsh-tools' +import type { GenericCallView } from '@deepseek-ai/dsh-tools' +import type { ContentBlock } from '@deepseek-ai/dsh-llm' +import { ItemRetainer } from '@deepseek-ai/dsh-retention' +import type { RetainedItems } from '@deepseek-ai/dsh-retention' +import type {} from '@deepseek-ai/dsh-bash' +import type {} from '@deepseek-ai/dsh-system-prompt' +import { runRipgrep, toWorkdirRelative, trySaveFormattedResult } from './search-core.ts' +import { singleQuote } from './shell-quote.ts' + +/** + * Default cap on paths retained inline by one `glob` call (the `globMaxResults` + * config), matching Claude Code's default `GlobTool` result limit. + */ +export const GLOB_MAX_RESULTS = 100 + +/** + * Directory names ripgrep must never descend into for a discovery listing: VCS + * metadata stores. `--no-ignore --hidden` would otherwise surface them in every + * broad search. Each is excluded with a negated any-depth `--glob` (see + * {@link buildGlobCommand}), which matches — and prunes — the directory + * wherever it appears. + */ +export const GLOB_VCS_EXCLUDES: readonly string[] = ['.git', '.svn', '.hg', '.bzr', '.jj', '.sl'] + +/** Resolved glob-tool caps — plugin config after defaulting (see `Config` in index.ts). */ +export interface GlobToolCaps { + /** Max paths retained inline; later paths go to the formatted spill file. */ + maxResults: number + /** Cap on the complete raw `rg` stdout the tool will parse. */ + rawOutputMaxBytes: number + /** Cooperative tool-call budget (ms) attached as `ToolDefinition.timeoutMs`. */ + timeoutMs: number +} + +/** Validated `glob` arguments. */ +export interface GlobInput { + pattern: string + path?: string +} + +/** + * Validate value constraints the schema DSL can't express: a non-blank + * `pattern`, and a non-blank `path` when given. Throws a plain `Error` (an + * ordinary tool argument error) otherwise. + * + * @param args - the schema-validated `glob` arguments. + * @returns the accepted input, unchanged. + */ +export function parseGlobArgs(args: { pattern: string; path?: string }): GlobInput { + if (args.pattern.trim().length === 0) throw new Error('pattern must be a non-empty string') + if (args.path !== undefined && args.path.trim().length === 0) throw new Error('path must be a non-empty string when given') + return { pattern: args.pattern, ...args.path !== undefined ? { path: args.path } : {} } +} + +/** + * Build the fixed `rg --files` command for one `glob` call. Every + * model-controlled value ({@link GlobInput.pattern}, {@link GlobInput.path}) + * passes through {@link singleQuote}; the search root rides behind `--` so a + * leading-dash path can never be parsed as a flag. `--sort=modified` orders by + * modification time, `--no-ignore --hidden` searches ignored and hidden files, + * and {@link GLOB_VCS_EXCLUDES} keeps VCS metadata out. + * + * @param input - the validated arguments. + * @returns the complete, shell-safe command string. + */ +export function buildGlobCommand(input: GlobInput): string { + const parts = [ + 'rg --files', + `--glob=${singleQuote(input.pattern)}`, + '--sort=modified --no-ignore --hidden', + ...GLOB_VCS_EXCLUDES.map(name => `--glob=${singleQuote(`!**/${name}`)}`), + ] + if (input.path !== undefined) parts.push('--', singleQuote(input.path)) + return parts.join(' ') +} + +/** + * Format the model-facing `glob` result: the retained paths, then — when the + * result was capped — a footer carrying either the formatted-spill recovery + * path or the could-not-save explanation. The omitted count is a budget fact: + * the search itself completed. + * + * @param retained - the retention outcome over every discovered path. + * @param spillPath - the saved complete-result path, or `undefined` when unsaved. + * @returns the model-facing text. + */ +export function formatGlobOutput(retained: RetainedItems, spillPath: string | undefined): string { + const body = retained.items.join('\n') + if (!retained.truncated) return body + const recovery = spillPath !== undefined + ? `Full sorted result saved to: ${spillPath}. Use read with offset/limit to inspect it.` + : 'The complete result could not be saved; narrow pattern or path to see more.' + return `${body}\n\n(Showing ${retained.kept} of ${retained.seen} paths. ${recovery})` +} + +/** + * Pending-call presentation: a search card titled by the pattern (and root). + * + * @param args - the raw tool arguments; `pattern` and `path` feed the title. + * @returns the generic card view (`kind: 'search'`) shown while the call runs. + */ +export function presentGlobCall(args: { pattern: string; path?: string }): GenericCallView { + const where = args.path !== undefined ? ` in ${args.path}` : '' + return { card: 'generic', title: `Glob ${args.pattern}${where}`, kind: 'search', rawInput: args.pattern } +} + +/** + * Register the `glob` tool and its system-prompt guidance. + * + * @param ctx - the plugin context; registrations are effects scoped to it, and + * execution uses its `bash` service. + * @param caps - the deployment's resolved glob caps (plugin config after defaulting). + */ +export function applyGlobTool(ctx: Context, caps: GlobToolCaps): void { + ctx.systemPrompt.section({ + name: 'tool:glob', + order: 103, + text: 'Use the glob tool — not shell find or ls — to discover files by path pattern. Results are sorted by modification time and include hidden and ignored files.', + }) + + ctx.tools.register(defineTool({ + name: 'glob', + description: 'Find files whose paths match a glob pattern. Returns matching paths sorted by modification time, ' + + 'including hidden and ignored files (VCS metadata directories are excluded). ' + + `Returns the first ${caps.maxResults} paths inline; a capped result reports where the complete list was saved.`, + parameters: { + pattern: { type: 'string', required: true, description: 'Glob pattern to match file paths against (e.g. "**/*.ts", "src/**/*.test.js").' }, + path: { type: 'string', description: 'Directory to search in. Defaults to the session workspace; a relative path resolves against it.' }, + }, + timeoutMs: caps.timeoutMs, + async execute(args, exec): Promise { + const input = parseGlobArgs(args) + const run = await runRipgrep(ctx, exec, 'glob', buildGlobCommand(input), caps.rawOutputMaxBytes) + if (run.noMatches) return [{ type: 'text', text: 'No files found' }] + + const retainer = new ItemRetainer({ kind: 'head', maxItems: caps.maxResults }) + const all: string[] = [] + for (const line of run.stdout.split('\n')) { + if (line.length === 0) continue + const displayPath = toWorkdirRelative(line, run.workdir) + all.push(displayPath) + retainer.push(displayPath) + } + const retained = retainer.finish() + + // The complete sorted list is the recovery artifact; save it only when + // the inline page omitted paths (an uncapped result needs no spill file). + const spillPath = retained.truncated + ? await trySaveFormattedResult(ctx, exec, 'glob-results.txt', all.join('\n')) + : undefined + return [{ type: 'text', text: formatGlobOutput(retained, spillPath) }] + }, + presentCall: presentGlobCall, + })) +} diff --git a/packages/fs/tool-fs-search/src/grep.ts b/packages/fs/tool-fs-search/src/grep.ts new file mode 100644 index 0000000000..e5e64e5222 --- /dev/null +++ b/packages/fs/tool-fs-search/src/grep.ts @@ -0,0 +1,314 @@ +/** + * The model-facing `grep` tool: search file contents with a ripgrep regular + * expression. Execution goes through the bash seam (`ctx.bash`) with a fixed + * line-oriented `rg --json` command so file path, line number, and line text + * parse without colon-splitting ambiguity — this module owns the model-facing + * schema, argument validation, shell-safe command construction, `--json` + * record parsing, per-line preview retention, match retention, grouping, and + * formatting; process concerns stay behind `ctx.bash`. + * + * @module @deepseek-ai/dsh-tool-fs-search/grep + */ + +import type { Context } from 'cordis' +import { defineTool } from '@deepseek-ai/dsh-tools' +import type { GenericCallView } from '@deepseek-ai/dsh-tools' +import type { ContentBlock } from '@deepseek-ai/dsh-llm' +import { ItemRetainer, TextRetainer } from '@deepseek-ai/dsh-retention' +import type { RetainedItems } from '@deepseek-ai/dsh-retention' +import type {} from '@deepseek-ai/dsh-bash' +import type {} from '@deepseek-ai/dsh-system-prompt' +import { SearchError, runRipgrep, toWorkdirRelative, trySaveFormattedResult } from './search-core.ts' +import { singleQuote } from './shell-quote.ts' + +/** + * Default cap on flat matches retained inline by one `grep` call (the + * `grepMaxMatches` config), matching Claude Code's default `GrepTool` + * `head_limit`. + */ +export const GREP_MAX_MATCHES = 250 + +/** + * Default cap in bytes on one matched-line preview (the `grepMaxLineBytes` + * config); the cut preserves UTF-8 boundaries. + */ +export const GREP_MAX_LINE_BYTES = 2000 + +/** Resolved grep-tool caps — plugin config after defaulting (see `Config` in index.ts). */ +export interface GrepToolCaps { + /** Max flat matches retained inline; later matches go to the formatted spill file. */ + maxMatches: number + /** Max bytes retained per matched-line preview. */ + maxLineBytes: number + /** Cap on the complete raw `rg` stdout the tool will parse. */ + rawOutputMaxBytes: number + /** Cooperative tool-call budget (ms) attached as `ToolDefinition.timeoutMs`. */ + timeoutMs: number +} + +/** Validated `grep` arguments. */ +export interface GrepInput { + pattern: string + path?: string + include?: string +} + +/** One parsed match: the file, the 1-based line number, and the (possibly previewed) line text. */ +export interface GrepMatch { + path: string + lineNumber: number + line: string +} + +/** + * Reject an `include` that is not ONE positive glob filter: blank strings, + * negated patterns (`!…`), and comma-separated lists. A comma inside a brace + * group is fine — `*.{ts,tsx}` is one glob with alternation, not a list. + */ +function validateInclude(include: string): void { + if (include.trim().length === 0) throw new Error('include must be a non-empty glob when given') + if (include.startsWith('!')) throw new Error('include must be a positive glob filter; negated patterns ("!…") are not supported') + let braceDepth = 0 + for (const char of include) { + if (char === '{') braceDepth++ + else if (char === '}') braceDepth = Math.max(0, braceDepth - 1) + else if (char === ',' && braceDepth === 0) { + throw new Error('include must be one glob, not a comma-separated list (use {a,b} alternation instead)') + } + } +} + +/** + * Validate value constraints the schema DSL can't express: a non-EMPTY + * `pattern` (whitespace is a legitimate regex), a non-blank `path` when given, + * and a single positive `include` glob ({@link GrepInput}). Throws a plain + * `Error` (an ordinary tool argument error) otherwise. + * + * @param args - the schema-validated `grep` arguments. + * @returns the accepted input, unchanged. + */ +export function parseGrepArgs(args: { pattern: string; path?: string; include?: string }): GrepInput { + if (args.pattern.length === 0) throw new Error('pattern must be a non-empty string') + if (args.path !== undefined && args.path.trim().length === 0) throw new Error('path must be a non-empty string when given') + if (args.include !== undefined) validateInclude(args.include) + return { + pattern: args.pattern, + ...args.path !== undefined ? { path: args.path } : {}, + ...args.include !== undefined ? { include: args.include } : {}, + } +} + +/** + * Build the fixed line-oriented `rg --json` command for one `grep` call. Every + * model-controlled value ({@link GrepInput.pattern}, {@link GrepInput.path}, + * {@link GrepInput.include}) passes through {@link singleQuote}; the pattern + * and include ride in `--flag=value` form and the target behind `--`, so a + * leading-dash value can never be parsed as a flag. + * + * @param input - the validated arguments. + * @returns the complete, shell-safe command string. + */ +export function buildGrepCommand(input: GrepInput): string { + const parts = ['rg --json', `--regexp=${singleQuote(input.pattern)}`] + if (input.include !== undefined) parts.push(`--glob=${singleQuote(input.include)}`) + if (input.path !== undefined) parts.push('--', singleQuote(input.path)) + return parts.join(' ') +} + +/** + * The uniform malformed-output failure: raw `rg --json` is an internal + * transport, so a shape surprise is a search failure, not a partial result. + */ +function malformedRecord(detail: string, cause?: unknown): SearchError { + return new SearchError(`grep received malformed ripgrep --json output (${detail})`, 'SEARCH_FAILED', cause !== undefined ? { cause } : undefined) +} + +/** + * Parse one `rg --json` NDJSON line into a match, `undefined` for the + * non-match record types (`begin`/`end`/`context`/`summary`). A line that is + * not JSON, or a `match` record missing its path / line number / line content, + * throws {@link SearchError} `SEARCH_FAILED`. A match whose line is not valid + * UTF-8 (ripgrep sends base64 `bytes` instead of `text`) yields a placeholder + * preview rather than failing the whole search. + */ +function parseRecord(line: string): GrepMatch | undefined { + let parsed: unknown + try { + parsed = JSON.parse(line) + } catch (error: unknown) { + throw malformedRecord('a line is not JSON', error) + } + if (typeof parsed !== 'object' || parsed === null) throw malformedRecord('a record is not an object') + const record = parsed as { type?: unknown; data?: unknown } + // Non-match record types (begin/end/context/summary — and any future type) + // are transport framing, not results: skipped, not malformed. + if (record.type !== 'match') return undefined + if (typeof record.data !== 'object' || record.data === null) throw malformedRecord('a match record has no data') + const data = record.data as { path?: unknown; line_number?: unknown; lines?: unknown } + const pathText = typeof data.path === 'object' && data.path !== null ? (data.path as { text?: unknown }).text : undefined + if (typeof pathText !== 'string') throw malformedRecord('a match record has no path text') + if (typeof data.line_number !== 'number') throw malformedRecord('a match record has no line number') + if (typeof data.lines !== 'object' || data.lines === null) throw malformedRecord('a match record has no line content') + const lines = data.lines as { text?: unknown; bytes?: unknown } + if (typeof lines.text === 'string') { + return { path: pathText, lineNumber: data.line_number, line: lines.text.replace(/\r?\n$/, '') } + } + if (typeof lines.bytes === 'string') { + return { path: pathText, lineNumber: data.line_number, line: '(line is not valid UTF-8)' } + } + throw malformedRecord('a match record has neither line text nor bytes') +} + +/** + * Parse complete `rg --json` stdout into flat matches, in output order (ripgrep + * emits one file's matches contiguously). Only `match` records are consumed. + * + * @param stdout - the complete raw `rg --json` stdout. + * @returns the flat matches; empty for output with no match records. + */ +export function parseGrepMatches(stdout: string): GrepMatch[] { + const matches: GrepMatch[] = [] + for (const line of stdout.split('\n')) { + if (line.length === 0) continue + const match = parseRecord(line) + if (match !== undefined) matches.push(match) + } + return matches +} + +/** + * Bound one matched-line preview to `maxBytes` (UTF-8 boundary preserved) and + * mark the cut. The cap is a per-line budget fact; the complete line stays in + * the searched file for `read`. + * + * @param line - the matched line text (trailing newline already stripped). + * @param maxBytes - the preview budget in bytes. + * @returns the preview, suffixed with ` (line truncated)` when bytes were cut. + */ +export function previewLine(line: string, maxBytes: number): string { + const retainer = new TextRetainer({ kind: 'head', maxBytes }) + retainer.push(line) + const kept = retainer.finish() + return kept.truncated ? `${kept.text} (line truncated)` : kept.text +} + +/** `match` / `matches` for a count. */ +function matchNoun(count: number): string { + return count === 1 ? 'match' : 'matches' +} + +/** + * Group flat matches by file (first-seen order) into the model-facing body: + * each file's display path, then one `Line N: ` row per match. + * + * @param matches - the flat matches to render. + * @returns the grouped body text. + */ +export function formatGrepMatches(matches: GrepMatch[]): string { + const byFile = new Map() + for (const match of matches) { + const group = byFile.get(match.path) + if (group !== undefined) group.push(match) + else byFile.set(match.path, [match]) + } + const sections: string[] = [] + for (const [path, group] of byFile) { + sections.push(`${path}\n${group.map(m => `Line ${m.lineNumber}: ${m.line}`).join('\n')}`) + } + return sections.join('\n\n') +} + +/** + * Format the model-facing `grep` result: a found-count header, the retained + * matches grouped by file, then — when the result was capped — a footer + * carrying either the formatted-spill recovery path or the could-not-save + * explanation. The omitted count is a budget fact: the search itself completed. + * + * @param retained - the retention outcome over every parsed match. + * @param spillPath - the saved complete-result path, or `undefined` when unsaved. + * @returns the model-facing text. + */ +export function formatGrepOutput(retained: RetainedItems, spillPath: string | undefined): string { + const header = retained.truncated + ? `Found ${retained.kept} of ${retained.seen} matches` + : `Found ${retained.seen} ${matchNoun(retained.seen)}` + const body = formatGrepMatches(retained.items) + if (!retained.truncated) return `${header}\n\n${body}` + const recovery = spillPath !== undefined + ? `Full grep result saved to: ${spillPath}. Use read with offset/limit to inspect it.` + : 'The complete result could not be saved; narrow pattern, path, or include to see more.' + return `${header}\n\n${body}\n\n(${recovery})` +} + +/** + * Pending-call presentation: a search card titled by the pattern (and target / + * include filter). + * + * @param args - the raw tool arguments; `pattern`, `path`, and `include` feed the title. + * @returns the generic card view (`kind: 'search'`) shown while the call runs. + */ +export function presentGrepCall(args: { pattern: string; path?: string; include?: string }): GenericCallView { + const where = args.path !== undefined ? ` in ${args.path}` : '' + const filter = args.include !== undefined ? ` (${args.include})` : '' + return { card: 'generic', title: `Grep ${args.pattern}${where}${filter}`, kind: 'search', rawInput: args.pattern } +} + +/** + * Register the `grep` tool and its system-prompt guidance. + * + * @param ctx - the plugin context; registrations are effects scoped to it, and + * execution uses its `bash` service. + * @param caps - the deployment's resolved grep caps (plugin config after defaulting). + */ +export function applyGrepTool(ctx: Context, caps: GrepToolCaps): void { + ctx.systemPrompt.section({ + name: 'tool:grep', + order: 104, + text: 'Use the grep tool — not shell grep or rg — to search file contents. Use read on a matched file when you need surrounding context.', + }) + + ctx.tools.register(defineTool({ + name: 'grep', + description: 'Search file contents with a ripgrep regular expression. Returns matching lines with line numbers, grouped by file. ' + + `Returns the first ${caps.maxMatches} matches inline; a capped result reports where the complete match list was saved. ` + + 'Use read on a matched file for surrounding context.', + parameters: { + pattern: { type: 'string', required: true, description: 'Regular expression to search for (ripgrep syntax).' }, + path: { type: 'string', description: 'File or directory to search. Defaults to the session workspace; a relative path resolves against it.' }, + include: { type: 'string', description: 'One glob filter for which files to search (e.g. "*.ts", "*.{js,jsx}"). Not a list; negation is not supported.' }, + }, + timeoutMs: caps.timeoutMs, + async execute(args, exec): Promise { + const input = parseGrepArgs(args) + const run = await runRipgrep(ctx, exec, 'grep', buildGrepCommand(input), caps.rawOutputMaxBytes) + if (run.noMatches) return [{ type: 'text', text: 'No matches found' }] + + const retainer = new ItemRetainer({ kind: 'head', maxItems: caps.maxMatches }) + const all: GrepMatch[] = [] + for (const raw of parseGrepMatches(run.stdout)) { + const match: GrepMatch = { + path: toWorkdirRelative(raw.path, run.workdir), + lineNumber: raw.lineNumber, + line: previewLine(raw.line, caps.maxLineBytes), + } + all.push(match) + retainer.push(match) + } + const retained = retainer.finish() + + // The spill file stores the FULL formatted match list (same grouped, + // per-line-previewed shape the model saw), so read offset/limit pages the + // same logical result; save only when the inline page omitted matches. + const spillPath = retained.truncated + ? await trySaveFormattedResult( + ctx, + exec, + 'grep-results.txt', + `Found ${all.length} ${matchNoun(all.length)}\n\n${formatGrepMatches(all)}`, + ) + : undefined + return [{ type: 'text', text: formatGrepOutput(retained, spillPath) }] + }, + presentCall: presentGrepCall, + })) +} diff --git a/packages/fs/tool-fs-search/src/index.ts b/packages/fs/tool-fs-search/src/index.ts new file mode 100644 index 0000000000..1fec6a999e --- /dev/null +++ b/packages/fs/tool-fs-search/src/index.ts @@ -0,0 +1,110 @@ +/** + * The model-facing filesystem discovery tool suite (`glob`, `grep`) over the + * bash executor seam (`ctx.bash`). This single plugin registers both tools. + * + * ## Bash-backed, not a `ctx.fs` provider method + * + * Local workspace discovery is a process-backed `rg` workflow, so these tools + * execute through `ctx.bash.resolve(request)` → `ctx.bash.run(spec)` with fixed + * ripgrep command templates — never `ctx.bash.start()`, never a model-visible + * background task. The tool layer owns schemas, argument validation, shell + * quoting ({@link module:@deepseek-ai/dsh-tool-fs-search/shell-quote}), result + * parsing, retention, formatted-result spill, and timeout declaration; the + * bash executor owns request defaulting/capping, subprocess execution, + * process-group termination, environment scrubbing, raw output capture, and + * backend substitution. The package injects `tools`, `systemPrompt`, and + * `bash` — deliberately NOT `fs`, and `ctx.spillFiles` is read opportunistically + * with `ctx.get()` because formatted-result spill is optional. + * + * Returned paths are displayed relative to the resolved bash workdir and are + * follow-up-readable only in co-located deployments where the bash workdir and + * the filesystem `read` root are the same workspace — a documented v1 + * deployment requirement, not runtime-validated. + * + * @module @deepseek-ai/dsh-tool-fs-search + */ + +import type { Context } from 'cordis' +import z from 'schemastery' +import { GLOB_MAX_RESULTS, applyGlobTool } from './glob.ts' +import { GREP_MAX_LINE_BYTES, GREP_MAX_MATCHES, applyGrepTool } from './grep.ts' +import { RAW_OUTPUT_MAX_BYTES, SEARCH_TIMEOUT_MS } from './search-core.ts' + +export { GLOB_MAX_RESULTS, GLOB_VCS_EXCLUDES, applyGlobTool, buildGlobCommand, formatGlobOutput, parseGlobArgs, presentGlobCall } from './glob.ts' +export type { GlobInput, GlobToolCaps } from './glob.ts' +export { + GREP_MAX_LINE_BYTES, + GREP_MAX_MATCHES, + applyGrepTool, + buildGrepCommand, + formatGrepMatches, + formatGrepOutput, + parseGrepArgs, + parseGrepMatches, + presentGrepCall, + previewLine, +} from './grep.ts' +export type { GrepInput, GrepMatch, GrepToolCaps } from './grep.ts' +export { RAW_OUTPUT_MAX_BYTES, SEARCH_TIMEOUT_MS, SearchError, runRipgrep, toWorkdirRelative, trySaveFormattedResult } from './search-core.ts' +export type { RipgrepRun, SearchErrorCode } from './search-core.ts' +export { singleQuote } from './shell-quote.ts' + +/** Cordis plugin name used by loader diagnostics. */ +export const name = 'tool-fs-search' + +/** Services required by the search tool suite (`spillFiles` is optional, read via `ctx.get()`). */ +export const inject = ['tools', 'systemPrompt', 'bash'] + +/** Plugin config (all optional — `Config` supplies the defaults). */ +export interface Config { + /** Max paths one `glob` call retains inline; later paths go to the formatted spill file. */ + globMaxResults?: number + /** Max flat matches one `grep` call retains inline; later matches go to the formatted spill file. */ + grepMaxMatches?: number + /** Max bytes retained for one matched-line preview (the cut preserves UTF-8 boundaries). */ + grepMaxLineBytes?: number + /** Max complete raw `rg` stdout bytes a search will parse; larger raw output fails with `SEARCH_RAW_OUTPUT_OVERFLOW`. */ + rawOutputMaxBytes?: number + /** Cooperative tool-call timeout budget (ms) on both tools, enforced by `@deepseek-ai/dsh-timeout-policy` through `exec.signal`. */ + timeoutMs?: number +} + +export const Config: z = z.object({ + globMaxResults: z.number().default(GLOB_MAX_RESULTS), + grepMaxMatches: z.number().default(GREP_MAX_MATCHES), + grepMaxLineBytes: z.number().default(GREP_MAX_LINE_BYTES), + rawOutputMaxBytes: z.number().default(RAW_OUTPUT_MAX_BYTES), + timeoutMs: z.number().default(SEARCH_TIMEOUT_MS), +}) + +/** The shape after schemastery applied the defaults. */ +type ResolvedConfig = Required + +/** Every search cap counts items/bytes/milliseconds — a positive integer, or retention and timeout arithmetic misbehaves silently. */ +function assertPositiveInteger(name: string, value: number): void { + if (!Number.isInteger(value) || value < 1) { + throw new Error(`tool-fs-search: ${name} must be a positive integer`) + } +} + +/** Register the `glob`/`grep` filesystem discovery tool suite. */ +export function apply(ctx: Context, config: Config): void { + // schemastery (Config) has already filled every defaulted field. + const resolved = config as ResolvedConfig + assertPositiveInteger('globMaxResults', resolved.globMaxResults) + assertPositiveInteger('grepMaxMatches', resolved.grepMaxMatches) + assertPositiveInteger('grepMaxLineBytes', resolved.grepMaxLineBytes) + assertPositiveInteger('rawOutputMaxBytes', resolved.rawOutputMaxBytes) + assertPositiveInteger('timeoutMs', resolved.timeoutMs) + applyGlobTool(ctx, { + maxResults: resolved.globMaxResults, + rawOutputMaxBytes: resolved.rawOutputMaxBytes, + timeoutMs: resolved.timeoutMs, + }) + applyGrepTool(ctx, { + maxMatches: resolved.grepMaxMatches, + maxLineBytes: resolved.grepMaxLineBytes, + rawOutputMaxBytes: resolved.rawOutputMaxBytes, + timeoutMs: resolved.timeoutMs, + }) +} diff --git a/packages/fs/tool-fs-search/src/search-core.ts b/packages/fs/tool-fs-search/src/search-core.ts new file mode 100644 index 0000000000..2d9d0b63dd --- /dev/null +++ b/packages/fs/tool-fs-search/src/search-core.ts @@ -0,0 +1,257 @@ +/** + * Shared execution plumbing for the `glob` / `grep` search tools: the + * package-owned `SEARCH_*` error vocabulary, one bash-seam run helper that + * turns a fixed `rg` command into complete raw stdout, the best-effort + * formatted-result spill handoff, and workdir-relative path display. + * + * Both tools execute through `ctx.bash.resolve(request)` → `ctx.bash.run(spec)` + * as ordinary foreground tool calls — never `ctx.bash.start()`, never a + * model-visible background task. Raw `rg` stdout is an internal transport + * detail: when the executor truncates it, the ONLY recovery source is the + * executor's local raw spill file, read here up to `rawOutputMaxBytes` and + * never exposed to the model. The model-facing recovery artifact is the + * formatted result saved through `ctx.spillFiles.saveText()` + * ({@link trySaveFormattedResult}) — a different artifact from the bash raw + * spill file. + * + * @module @deepseek-ai/dsh-tool-fs-search/search-core + */ + +import { readFile, stat } from 'node:fs/promises' +import { isAbsolute, relative, sep } from 'node:path' +import type { Context } from 'cordis' +import { HarnessError } from '@deepseek-ai/dsh-llm' +import type { BashRunResult, CollectedOutput } from '@deepseek-ai/dsh-bash' +import type { SaveTextSpill } from '@deepseek-ai/dsh-spill' +import type { ToolExecution } from '@deepseek-ai/dsh-tools' + +/** + * Default cap on the complete raw `rg` stdout the tools will parse (the + * `rawOutputMaxBytes` config), matching Claude Code's ripgrep raw buffer. + */ +export const RAW_OUTPUT_MAX_BYTES = 20_000_000 + +/** + * Default cooperative tool-call timeout budget in milliseconds (the `timeoutMs` + * config), attached to both tool definitions for + * `@deepseek-ai/dsh-timeout-policy` to enforce through `exec.signal`. + */ +export const SEARCH_TIMEOUT_MS = 30_000 + +/** + * Stable, machine-routable codes for search failures. Package-owned (not + * `FsErrorCode`) because these tools are bash-backed discovery, not `ctx.fs` + * provider operations: `SEARCH_INVALID_PATTERN` — ripgrep rejected the regex or + * glob; `SEARCH_FAILED` — the search could not run or its output could not be + * parsed (missing `rg`, inaccessible target, signal kill, malformed `--json`); + * `SEARCH_RAW_OUTPUT_OVERFLOW` — raw `rg` output exceeded `rawOutputMaxBytes` + * (or was truncated with no recovery file); `SEARCH_ABORTED` — the tool + * timeout, caller cancellation, or the bash executor's own timeout cut the + * search short. + */ +export type SearchErrorCode = + | 'SEARCH_INVALID_PATTERN' + | 'SEARCH_FAILED' + | 'SEARCH_RAW_OUTPUT_OVERFLOW' + | 'SEARCH_ABORTED' + +/** + * Typed search failure. Extends {@link HarnessError} so it carries a stable + * {@link SearchErrorCode} and chains `cause`; the tool registry surfaces + * `{ name, code }` on `isError` results so retry/permission/UI layers can + * branch without parsing messages. + */ +export class SearchError extends HarnessError { + override readonly code: SearchErrorCode + + constructor(message: string, code: SearchErrorCode, options?: ErrorOptions) { + super(message, code, options) + this.code = code + } +} + +/** The completed acquisition of one `rg` run: complete stdout plus the resolved workdir. */ +export interface RipgrepRun { + /** Complete raw stdout — inline executor text, or the raw spill file's content. */ + stdout: string + /** True when ripgrep exited 1: a successful search with zero results. */ + noMatches: boolean + /** The resolved working directory the command ran in (the display-relativization base). */ + workdir: string +} + +/** + * The retained stderr tail as a diagnostic excerpt, with a truncation note when + * the executor dropped bytes (the tool never reads `stderr.spillPath`). + */ +function stderrExcerpt(stderr: CollectedOutput): string { + const text = stderr.text.trim() + if (text.length === 0) return '' + return stderr.truncated ? `${text} [stderr truncated]` : text +} + +/** Classify a nonzero-exit `rg` run into the search error vocabulary (invalid pattern vs missing `rg` vs everything else). */ +function classifyRunFailure(toolName: string, result: BashRunResult): SearchError { + const stderr = stderrExcerpt(result.stderr) + if (/regex parse error|error parsing glob/i.test(stderr)) { + return new SearchError(`${toolName} pattern rejected by ripgrep: ${stderr}`, 'SEARCH_INVALID_PATTERN') + } + if (result.exitCode === 127 || /command not found/i.test(stderr)) { + return new SearchError(`${toolName} requires ripgrep (rg) on the bash executor's PATH${stderr.length > 0 ? `: ${stderr}` : ''}`, 'SEARCH_FAILED') + } + return new SearchError(`${toolName} search failed (exit ${result.exitCode})${stderr.length > 0 ? `: ${stderr}` : ''}`, 'SEARCH_FAILED') +} + +/** + * Acquire the COMPLETE raw stdout of a finished run. Untruncated stdout is used + * as-is; truncated stdout is recovered from the executor's local raw spill file + * only when the complete file fits within `rawOutputMaxBytes`. A missing spill + * path or an over-cap file is a clear failure telling the model to narrow the + * search — never a silently-partial parse. + */ +async function completeStdout(toolName: string, result: BashRunResult, rawOutputMaxBytes: number): Promise { + if (!result.stdout.truncated) return result.stdout.text + const narrow = 'narrow pattern, path, or include and retry' + const spillPath = result.stdout.spillPath + if (spillPath === undefined) { + throw new SearchError( + `${toolName} produced more raw output than the bash executor retained and no raw spill file is available; ${narrow}`, + 'SEARCH_RAW_OUTPUT_OVERFLOW', + ) + } + try { + const { size } = await stat(spillPath) + if (size > rawOutputMaxBytes) { + throw new SearchError( + `${toolName} produced ${size} bytes of raw output, over the ${rawOutputMaxBytes}-byte cap; ${narrow}`, + 'SEARCH_RAW_OUTPUT_OVERFLOW', + ) + } + return await readFile(spillPath, 'utf8') + } catch (error: unknown) { + if (error instanceof SearchError) throw error + throw new SearchError(`${toolName} could not read the executor's raw output spill file`, 'SEARCH_FAILED', { cause: error }) + } +} + +/** + * Run one fixed `rg` command through the bash seam and return its complete raw + * stdout. The bash request workdir is the calling agent's session cwd + * (`exec.agent.session.header.cwd`) when available — mirroring `dsh-tool-bash` / + * `dsh-tool-fs` — else omitted so the implementation's `resolve()` applies its + * configured default. `exec.signal` is forwarded so the cooperative tool + * timeout (`@deepseek-ai/dsh-timeout-policy`) and caller cancellation kill the + * command; the bash backend's own timeout stays a second safety cap. + * + * Exit semantics are tool-owned: exit 0 is success with results, exit 1 is + * success with zero results (`noMatches`), anything else throws a + * {@link SearchError} (abort/timeout → `SEARCH_ABORTED`, invalid pattern → + * `SEARCH_INVALID_PATTERN`, the rest → `SEARCH_FAILED` / + * `SEARCH_RAW_OUTPUT_OVERFLOW`). + * + * @param ctx - the plugin context; execution uses its `bash` service. + * @param exec - the tool-execution context; supplies the session cwd and the abort signal. + * @param toolName - `glob` or `grep`, used in error messages. + * @param command - the fully-quoted `rg` command string (every model value already through `singleQuote`). + * @param rawOutputMaxBytes - cap on the complete raw stdout the tool will parse. + * @returns the complete stdout, the zero-result flag, and the resolved workdir. + */ +export async function runRipgrep( + ctx: Context, + exec: ToolExecution, + toolName: string, + command: string, + rawOutputMaxBytes: number, +): Promise { + const cwd = exec.agent?.session.header.cwd + const spec = ctx.bash.resolve({ + command, + ...cwd !== undefined ? { workdir: cwd } : {}, + ...exec.signal ? { signal: exec.signal } : {}, + }) + const result = await ctx.bash.run(spec) + if (result.aborted) { + throw new SearchError(`${toolName} was aborted before completion (tool timeout or caller cancellation)`, 'SEARCH_ABORTED') + } + if (result.timedOut) { + throw new SearchError(`${toolName} timed out after ${result.timeoutMs}ms in the bash executor; narrow pattern, path, or include and retry`, 'SEARCH_ABORTED') + } + if (result.signal !== null || result.exitCode === null) { + throw new SearchError(`${toolName} search command was killed by signal ${result.signal ?? '(unknown)'}`, 'SEARCH_FAILED') + } + if (result.exitCode !== 0 && result.exitCode !== 1) { + throw classifyRunFailure(toolName, result) + } + const stdout = await completeStdout(toolName, result, rawOutputMaxBytes) + return { stdout, noMatches: result.exitCode === 1, workdir: spec.workdir } +} + +/** + * Map an `rg` output path to its display form: absolute paths inside the + * resolved bash workdir become workdir-relative; everything else (relative + * output, paths outside the workdir) passes through unchanged. Display-only — + * returned paths are follow-up-readable in co-located bash/filesystem + * deployments where both resolve the same workspace (the documented v1 + * deployment requirement). + * + * @param path - one path as ripgrep printed it. + * @param workdir - the resolved bash workdir the command ran in. + * @returns the workdir-relative display path when possible, else `path` unchanged. + */ +export function toWorkdirRelative(path: string, workdir: string): string { + if (!isAbsolute(path)) return path + const rel = relative(workdir, path) + if (rel.length === 0) return '.' + if (rel === '..' || rel.startsWith(`..${sep}`)) return path + return rel +} + +/** + * Best-effort save of one COMPLETE formatted search result through + * `ctx.spillFiles.saveText()` — the model-facing recovery path for a capped + * result. `spillFiles` is read with `ctx.get()` (not static inject) because + * formatted-result spill is optional; the spill owner is the calling agent's + * session header id and the source is the tool execution identity. A missing + * backend, a call with no session owner, or a `saveText()` rejection logs a + * warning and returns `undefined` — the caller keeps the inline result and + * reports that the complete result could not be saved; search success never + * turns into `isError` because spill storage is unavailable. + * + * @param ctx - the plugin context; `spillFiles` is looked up opportunistically. + * @param exec - the tool-execution context; supplies the owning session, tool name, and call id. + * @param suggestedName - the backend-sanitized filename hint (e.g. `grep-results.txt`). + * @param content - the complete formatted result to persist. + * @returns the saved spill path, or `undefined` when the result could not be saved. + */ +export async function trySaveFormattedResult( + ctx: Context, + exec: ToolExecution, + suggestedName: string, + content: string, +): Promise { + const sessionId = exec.agent?.session.header.id + if (sessionId === undefined) { + ctx.logger.warn(`tool-fs-search: no session owner for ${exec.name} result; complete result not saved`) + return undefined + } + const spillFiles = ctx.get('spillFiles') + if (!spillFiles) { + ctx.logger.warn(`tool-fs-search: no ctx.spillFiles backend loaded; complete ${exec.name} result not saved`) + return undefined + } + const save: SaveTextSpill = { + owner: { sessionId }, + source: { toolName: exec.name, callId: exec.callId, label: 'result' }, + suggestedName, + content, + } + try { + const { path } = await spillFiles.saveText(save) + return path + } catch (error: unknown) { + // Best-effort: a storage failure must never fail the search or hide the + // inline result — the footer reports the unsaved remainder instead. + ctx.logger.warn(`tool-fs-search: saveText failed for ${exec.name}: ${String(error)}; complete result not saved`) + return undefined + } +} diff --git a/packages/fs/tool-fs-search/src/shell-quote.ts b/packages/fs/tool-fs-search/src/shell-quote.ts new file mode 100644 index 0000000000..9453b8e255 --- /dev/null +++ b/packages/fs/tool-fs-search/src/shell-quote.ts @@ -0,0 +1,27 @@ +/** + * The one shell-quoting helper both search tools MUST route every + * model-controlled value through before it enters an `rg` command string. The + * bash seam (`ctx.bash`) accepts a command STRING, not an argv vector, so this + * is the safety boundary that stops a `pattern`, `path`, or `include` from + * breaking out of its argument and injecting shell syntax. + * + * Command builders in `glob.ts` / `grep.ts` must never hand-roll quoting or + * concatenate an unquoted model value — they call {@link singleQuote}. + * + * @module @deepseek-ai/dsh-tool-fs-search/shell-quote + */ + +/** + * POSIX single-quote a string for safe use as ONE shell word. Wraps the value + * in single quotes and rewrites every embedded single quote as `'\''` (close + * quote, an escaped literal quote, reopen quote). Inside single quotes the shell + * treats every other byte literally — spaces, newlines, `$`, backticks, `;`, + * `|`, `&`, glob metacharacters, and a leading `-` are all inert — so the result + * is a single, injection-safe argument regardless of the input. + * + * @param value - the raw, possibly model-controlled string to quote. + * @returns the value wrapped as one safe single-quoted shell word. + */ +export function singleQuote(value: string): string { + return `'${value.replaceAll("'", "'\\''")}'` +} diff --git a/packages/fs/tool-fs-search/tests/integration.spec.ts b/packages/fs/tool-fs-search/tests/integration.spec.ts new file mode 100644 index 0000000000..74a418238b --- /dev/null +++ b/packages/fs/tool-fs-search/tests/integration.spec.ts @@ -0,0 +1,161 @@ +/** + * Integration tests: the REAL local bash executor (`dsh-bash-local`) plus a + * REAL ripgrep binary, exercised through `ctx.tools.execute()`. These verify + * the WORLD — actual files on disk are discovered and grepped, hostile + * patterns stay inert in a real shell, and real `rg` stderr classifies into + * the `SEARCH_*` vocabulary. The whole suite self-skips when `rg` is not on + * PATH (a CI accommodation mirroring the keyless e2e skip); the fake-executor + * suite (tools.spec.ts) carries the coverage gate. + */ + +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { spawnSync } from 'node:child_process' +import { mkdir, mkdtemp, rm, utimes, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Context } from 'cordis' +import { CallId } from '@deepseek-ai/dsh-llm' +import SystemPrompt from '@deepseek-ai/dsh-system-prompt' +import ToolRegistry from '@deepseek-ai/dsh-tools' +import { LocalBashExecutor } from '@deepseek-ai/dsh-bash-local' +import * as ToolFsSearch from '@deepseek-ai/dsh-tool-fs-search' + +const hasRg = spawnSync('rg', ['--version'], { encoding: 'utf8' }).status === 0 + +let dir: string +let ctx: Context + +let callCounter = 0 +function call(name: string, args: unknown, agentObj?: object) { + return ctx.tools.execute({ + callId: CallId(`it-${++callCounter}`), + name, + arguments: args, + ...agentObj ? { agent: agentObj as never } : {}, + }) +} + +function text(result: { content: { type: string; text?: string }[] }): string { + return result.content.filter(b => b.type === 'text').map(b => b.text).join('') +} + +describe.skipIf(!hasRg)('search tools over the real bash executor + real rg', () => { + beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), 'dsh-search-int-')) + await mkdir(join(dir, 'src'), { recursive: true }) + await mkdir(join(dir, '.git'), { recursive: true }) + await mkdir(join(dir, 'spaced dir'), { recursive: true }) + await writeFile(join(dir, 'src', 'alpha.ts'), 'export const alpha = 1\n// TODO: refit alpha\n') + await writeFile(join(dir, 'src', 'beta.ts'), 'export const beta = 2\n') + await writeFile(join(dir, 'notes.md'), 'alpha appears here too\n') + await writeFile(join(dir, '.hidden.ts'), 'export const hidden = 3\n') + await writeFile(join(dir, '.git', 'config.ts'), 'never listed\n') + await writeFile(join(dir, 'spaced dir', "wei'rd \"name\".ts"), 'const inside = true\n') + // Deterministic --sort=modified order: alpha oldest, beta newest. + await utimes(join(dir, 'src', 'alpha.ts'), new Date(2000, 0, 1), new Date(2000, 0, 1)) + await utimes(join(dir, 'src', 'beta.ts'), new Date(2020, 0, 1), new Date(2020, 0, 1)) + + ctx = new Context() + await ctx.plugin(SystemPrompt) + await ctx.plugin(ToolRegistry) + await ctx.plugin(LocalBashExecutor, { cwd: dir, timeoutMs: 20_000 }) + await ctx.plugin(ToolFsSearch) + }) + + afterEach(async () => { + await rm(dir, { recursive: true, force: true }) + }) + + describe('glob', () => { + it('discovers files by pattern, sorted by modification time, hidden included, .git excluded', async () => { + const result = await call('glob', { pattern: '**/*.ts' }) + expect(result.isError).toBe(false) + const paths = text(result).split('\n') + expect(paths.indexOf('src/alpha.ts')).toBeLessThan(paths.indexOf('src/beta.ts')) + expect(paths).toContain('.hidden.ts') + expect(paths).toContain("spaced dir/wei'rd \"name\".ts") + expect(paths).not.toContain('.git/config.ts') + expect(paths).not.toContain('notes.md') + }) + + it('scopes to a directory search root (path arg)', async () => { + const result = await call('glob', { pattern: '*.ts', path: 'src' }) + expect(text(result).split('\n').sort()).toEqual(['src/alpha.ts', 'src/beta.ts']) + }) + + it('reports zero discoveries as No files found', async () => { + expect(text(await call('glob', { pattern: '*.nomatch' }))).toBe('No files found') + }) + + it('classifies an invalid glob as SEARCH_INVALID_PATTERN', async () => { + const result = await call('glob', { pattern: '[' }) + expect(result.isError).toBe(true) + expect(result.error).toMatchObject({ name: 'SearchError', code: 'SEARCH_INVALID_PATTERN' }) + }) + }) + + describe('grep', () => { + it('greps a directory tree with grouped, line-numbered output', async () => { + const result = await call('grep', { pattern: 'alpha' }) + expect(result.isError).toBe(false) + const output = text(result) + expect(output).toContain('Found 3 matches') + expect(output).toContain('src/alpha.ts\nLine 1: export const alpha = 1\nLine 2: // TODO: refit alpha') + expect(output).toContain('notes.md\nLine 1: alpha appears here too') + }) + + it('greps a single FILE target', async () => { + const result = await call('grep', { pattern: 'alpha', path: 'notes.md' }) + expect(text(result)).toBe('Found 1 match\n\nnotes.md\nLine 1: alpha appears here too') + }) + + it('greps a directory target with an include filter', async () => { + const result = await call('grep', { pattern: 'alpha', path: '.', include: '*.ts' }) + const output = text(result) + expect(output).toContain('alpha.ts') + expect(output).not.toContain('notes.md') + }) + + it('a hostile pattern stays inert (no command substitution, the world untouched)', async () => { + const canary = join(dir, 'pwned') + const result = await call('grep', { pattern: `$(touch ${canary})` }) + expect(result.isError).toBe(false) // exit 1: found nothing, executed nothing + expect(text(result)).toBe('No matches found') + expect(spawnSync('test', ['-e', canary]).status).not.toBe(0) + }) + + it('a leading-dash pattern is a pattern, not a flag', async () => { + await writeFile(join(dir, 'dashes.txt'), 'value --flag value\n') + const result = await call('grep', { pattern: '--flag', path: 'dashes.txt' }) + expect(text(result)).toBe('Found 1 match\n\ndashes.txt\nLine 1: value --flag value') + }) + + it('classifies a real rg regex error as SEARCH_INVALID_PATTERN', async () => { + const result = await call('grep', { pattern: '(unclosed' }) + expect(result.isError).toBe(true) + expect(result.error).toMatchObject({ code: 'SEARCH_INVALID_PATTERN' }) + }) + + it('classifies a missing target as SEARCH_FAILED', async () => { + const result = await call('grep', { pattern: 'x', path: 'no-such-dir' }) + expect(result.isError).toBe(true) + expect(result.error).toMatchObject({ code: 'SEARCH_FAILED' }) + }) + }) + + describe('per-session cwd', () => { + it('resolves the search in the SESSION workspace, not the executor config cwd', async () => { + const sessionDir = await mkdtemp(join(tmpdir(), 'dsh-search-session-')) + try { + await writeFile(join(sessionDir, 'only-here.ts'), 'const sessionFile = true\n') + const agentObj = { session: { header: { id: 'session-int', cwd: sessionDir } } } + const globbed = await call('glob', { pattern: '*.ts' }, agentObj) + expect(text(globbed)).toBe('only-here.ts') + const grepped = await call('grep', { pattern: 'sessionFile' }, agentObj) + expect(text(grepped)).toContain('only-here.ts\nLine 1: const sessionFile = true') + } finally { + await rm(sessionDir, { recursive: true, force: true }) + } + }) + }) +}) diff --git a/packages/fs/tool-fs-search/tests/load-path.spec.ts b/packages/fs/tool-fs-search/tests/load-path.spec.ts new file mode 100644 index 0000000000..d3c28619a3 --- /dev/null +++ b/packages/fs/tool-fs-search/tests/load-path.spec.ts @@ -0,0 +1,50 @@ +/** + * Real-load-path guard for @deepseek-ai/dsh-tool-fs-search. `tool-fs-search` is + * a NAMESPACE plugin with `inject` — so a stray `export default apply` would + * make the cordis Loader's `unwrapExports` (`exports.default ?? exports`) + * collapse the module to the bare `apply` function, DROPPING `inject`. The + * plugin would then read `ctx.bash` without having injected it and throw + * `cannot get property … without inject` the moment it loads (postmortem 0001). + * + * A hand-built `ctx.plugin({ apply, inject })` mount CANNOT catch that — it + * bypasses `unwrapExports`. So this test unwraps the module through the REAL + * `Loader.prototype.unwrapExports` and mounts the result over a bash executor, + * exercising the exact path the Loader uses. Prove the guard bites: add + * `export default apply` to `src/index.ts`, watch this go red, revert. + */ + +import { describe, expect, it } from 'vitest' +import { Context } from 'cordis' +import Loader from '@cordisjs/plugin-loader' +import SystemPrompt from '@deepseek-ai/dsh-system-prompt' +import ToolRegistry from '@deepseek-ai/dsh-tools' +import { LocalBashExecutor } from '@deepseek-ai/dsh-bash-local' +import * as toolFsSearch from '@deepseek-ai/dsh-tool-fs-search' + +describe('dsh-tool-fs-search real-load-path guard', () => { + it('has no default export and keeps name/inject/Config through unwrapExports', () => { + expect('default' in toolFsSearch).toBe(false) + + const loader = Object.create(Loader.prototype) as Loader + const unwrapped = loader.unwrapExports(toolFsSearch) as Record + expect(unwrapped).toBe(toolFsSearch) + expect(unwrapped.name).toBe('tool-fs-search') + expect(unwrapped.inject).toEqual(['tools', 'systemPrompt', 'bash']) + expect(typeof unwrapped.Config).toBe('function') + expect(typeof unwrapped.apply).toBe('function') + }) + + it('boots over ctx.bash through the unwrapped module without an inject error', async () => { + const ctx = new Context() + await ctx.plugin(SystemPrompt) + await ctx.plugin(ToolRegistry) + await ctx.plugin(LocalBashExecutor, {}) + + const loader = Object.create(Loader.prototype) as Loader + const unwrapped = loader.unwrapExports(toolFsSearch) as Parameters[0] + // A collapsed export shape (dropped inject) would throw "without inject" here. + const fiber = await ctx.plugin(unwrapped) + expect(ctx.tools.schemas().map(s => s.name)).toEqual(expect.arrayContaining(['glob', 'grep'])) + await fiber.dispose() + }) +}) diff --git a/packages/fs/tool-fs-search/tests/shell-quote.spec.ts b/packages/fs/tool-fs-search/tests/shell-quote.spec.ts new file mode 100644 index 0000000000..84c8506be1 --- /dev/null +++ b/packages/fs/tool-fs-search/tests/shell-quote.spec.ts @@ -0,0 +1,59 @@ +/** + * Unit tests for the shell-quoting safety boundary, plus a REAL round-trip: + * every adversarial value, quoted, must survive `bash -c "printf '%s' "` + * byte-for-byte — proving the quoting is inert in an actual shell, not just + * against a mental model of one. + */ + +import { describe, expect, it } from 'vitest' +import { spawnSync } from 'node:child_process' +import { singleQuote } from '@deepseek-ai/dsh-tool-fs-search' + +/** Adversarial values a model could pass as pattern / path / include. */ +const HOSTILE: readonly string[] = [ + 'plain', + 'with spaces', + "it's got 'quotes'", + '"double quoted"', + '$(rm -rf /tmp/nope)', + '`touch /tmp/nope`', + '$HOME and ${PATH}', + 'semi;colon && chain || pipe | bg &', + 'newline\nin the middle', + '-leading-dash', + '--leading-double-dash', + '*?[a-z]{x,y}', + '!bang', + '\\backslash\\', + '~tilde', + '# not a comment', + '>redirect &1', +] + +describe('singleQuote', () => { + it('wraps a plain value in single quotes', () => { + expect(singleQuote('abc')).toBe("'abc'") + }) + + it("rewrites embedded single quotes as '\\''", () => { + expect(singleQuote("a'b")).toBe("'a'\\''b'") + expect(singleQuote("''")).toBe("''\\'''\\'''") + }) + + it.each(HOSTILE.map(value => [JSON.stringify(value), value] as const))( + 'round-trips %s through a real bash -c unchanged', + (_label, value) => { + const result = spawnSync('bash', ['-c', `printf '%s' ${singleQuote(value)}`], { encoding: 'utf8' }) + expect(result.status).toBe(0) + expect(result.stdout).toBe(value) + }, + ) + + it('a quoted command substitution does not execute (the world stays untouched)', () => { + const canary = `/tmp/dsh-quote-canary-${process.pid}` + const result = spawnSync('bash', ['-c', `printf '%s' ${singleQuote(`$(touch ${canary})`)}`], { encoding: 'utf8' }) + expect(result.stdout).toBe(`$(touch ${canary})`) + // The canary file must NOT exist — the substitution stayed literal. + expect(spawnSync('test', ['-e', canary]).status).not.toBe(0) + }) +}) diff --git a/packages/fs/tool-fs-search/tests/tools.spec.ts b/packages/fs/tool-fs-search/tests/tools.spec.ts new file mode 100644 index 0000000000..6a539956a4 --- /dev/null +++ b/packages/fs/tool-fs-search/tests/tools.spec.ts @@ -0,0 +1,623 @@ +/** + * Consumer-surface tests for the search tools over a FAKE bash executor and a + * FAKE spill backend, exercised through `ctx.tools.execute()` so nothing + * bypasses the tool registry. The fake executor makes every seam outcome + * scriptable — truncated stdout with/without a raw spill file, abort/timeout, + * signal kills, ripgrep exit codes — so these tests verify schemas, argument + * validation, shell-safe command construction, workdir derivation, signal + * forwarding, `SEARCH_*` error classification, retention, formatted-result + * spill handoff, and the no-background-task invariant. Real-`rg` behavior is + * pinned separately in integration.spec.ts. + */ + +import { afterEach, describe, expect, it } from 'vitest' +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Context } from 'cordis' +import { CallId } from '@deepseek-ai/dsh-llm' +import SystemPrompt, { renderPrompt } from '@deepseek-ai/dsh-system-prompt' +import ToolRegistry from '@deepseek-ai/dsh-tools' +import { BashExecutor } from '@deepseek-ai/dsh-bash' +import type { BashExecRequest, BashExecSpec, BashRunResult, BashTask, BashTaskId, BashTaskRead, OwnerToken } from '@deepseek-ai/dsh-bash' +import { SpillFiles, SpillPath } from '@deepseek-ai/dsh-spill' +import type { SaveTextSpill, SpillRef } from '@deepseek-ai/dsh-spill' +import * as ToolFsSearch from '@deepseek-ai/dsh-tool-fs-search' +import { + buildGlobCommand, + buildGrepCommand, + formatGrepMatches, + parseGrepMatches, + presentGlobCall, + presentGrepCall, + previewLine, + toWorkdirRelative, +} from '@deepseek-ai/dsh-tool-fs-search' + +/** A successful run result over the given stdout; overrides script the failure shapes. */ +function runResult(stdout: string, overrides?: Partial): BashRunResult { + return { + exitCode: 0, + signal: null, + timedOut: false, + aborted: false, + timeoutMs: 60_000, + stdout: { text: stdout, truncated: false }, + stderr: { text: '', truncated: false }, + ...overrides, + } +} + +/** + * A scriptable fake executor: `resolve()` mirrors the real request→spec + * defaulting (workdir falls back to `/work`), `run()` returns whatever the + * test armed via `handler`, and `start()` throws — the search tools must NEVER + * create a background task. + */ +class FakeBash extends BashExecutor { + requests: BashExecRequest[] = [] + specs: BashExecSpec[] = [] + startCalls = 0 + handler: (spec: BashExecSpec) => BashRunResult = () => runResult('') + + override resolve(request: BashExecRequest): BashExecSpec { + this.requests.push(request) + return { + command: request.command, + workdir: request.workdir ?? '/work', + timeoutMs: request.timeoutMs ?? 60_000, + signal: request.signal, + owner: request.owner, + } + } + override run(spec: BashExecSpec): Promise { + this.specs.push(spec) + return Promise.resolve(this.handler(spec)) + } + override start(): BashTask { + this.startCalls++ + throw new Error('search tools must never start a background task') + } + override get(): BashTask | undefined { + return undefined + } + override ownerOf(): OwnerToken | undefined { + return undefined + } + override list(): BashTask[] { + return [] + } + override readOutput(id: BashTaskId): BashTaskRead { + throw new Error(`unknown bash task ${id}`) + } + override kill(id: BashTaskId): boolean { + throw new Error(`unknown bash task ${id}`) + } +} + +/** A recording spill backend; arm `failWith` to script a storage failure. */ +class FakeSpill extends SpillFiles { + saves: SaveTextSpill[] = [] + failWith?: Error + + override saveText(input: SaveTextSpill): Promise { + if (this.failWith) return Promise.reject(this.failWith) + this.saves.push(input) + return Promise.resolve({ path: SpillPath(`/spill/${input.suggestedName}`), bytes: Buffer.byteLength(input.content, 'utf8') }) + } +} + +interface SetupOptions { + config?: ToolFsSearch.Config + spill?: boolean +} + +async function setup(options: SetupOptions = {}) { + const ctx = new Context() + await ctx.plugin(SystemPrompt) + await ctx.plugin(ToolRegistry) + await ctx.plugin(FakeBash) + if (options.spill === true) await ctx.plugin(FakeSpill) + const fiber = await ctx.plugin(ToolFsSearch, options.config) + const bash = ctx.bash as FakeBash + const spill = options.spill === true ? ctx.get('spillFiles') as FakeSpill : undefined + return { ctx, bash, spill, fiber } +} + +/** A stand-in agent whose session header carries the given cwd (and a stable id). */ +const agent = (cwd?: string) => ({ session: { header: { id: 'session-1', ...cwd !== undefined ? { cwd } : {} } } }) + +let callCounter = 0 +function call(ctx: Context, name: string, args: unknown, options: { agent?: object; signal?: AbortSignal } = {}) { + return ctx.tools.execute({ + callId: CallId(`call-${++callCounter}`), + name, + arguments: args, + ...options.agent ? { agent: options.agent as never } : {}, + ...options.signal ? { signal: options.signal } : {}, + }) +} + +function text(result: { content: { type: string; text?: string }[] }): string { + return result.content.filter(b => b.type === 'text').map(b => b.text).join('') +} + +/** One rg --json match record line. */ +function matchLine(path: string, lineNumber: number, lineText: string): string { + return JSON.stringify({ type: 'match', data: { path: { text: path }, lines: { text: lineText }, line_number: lineNumber, absolute_offset: 0, submatches: [] } }) +} + +describe('registration', () => { + it('registers glob and grep with their prompt sections', async () => { + const { ctx } = await setup() + expect(ctx.tools.schemas().map(s => s.name).sort()).toEqual(['glob', 'grep']) + const prompt = renderPrompt(await ctx.systemPrompt.assemble()) + expect(prompt).toContain('Use the glob tool') + expect(prompt).toContain('Use the grep tool') + }) + + it('stays pending until ctx.bash exists (inject)', async () => { + const ctx = new Context() + await ctx.plugin(SystemPrompt) + await ctx.plugin(ToolRegistry) + await ctx.plugin(ToolFsSearch) // no bash executor + expect(ctx.tools.schemas()).toHaveLength(0) + }) + + it('unregisters everything on fiber disposal (HMR safety)', async () => { + const { ctx, fiber } = await setup() + expect(ctx.tools.schemas()).toHaveLength(2) + await fiber.dispose() + expect(ctx.tools.schemas()).toHaveLength(0) + const sections = (await ctx.systemPrompt.assemble()).sections.map(s => s.name) + expect(sections).not.toContain('tool:glob') + expect(sections).not.toContain('tool:grep') + }) + + it('attaches the configured timeoutMs to both tool definitions', async () => { + const { ctx } = await setup({ config: { timeoutMs: 5000 } }) + expect(ctx.tools.get('glob')?.timeoutMs).toBe(5000) + expect(ctx.tools.get('grep')?.timeoutMs).toBe(5000) + }) + + it('defaults the timeout budget to 30 seconds', async () => { + const { ctx } = await setup() + expect(ctx.tools.get('glob')?.timeoutMs).toBe(30_000) + expect(ctx.tools.get('grep')?.timeoutMs).toBe(30_000) + }) +}) + +describe('config validation', () => { + it.each([ + ['globMaxResults', { globMaxResults: 0 }], + ['grepMaxMatches', { grepMaxMatches: -1 }], + ['grepMaxLineBytes', { grepMaxLineBytes: 1.5 }], + ['rawOutputMaxBytes', { rawOutputMaxBytes: 0 }], + ['timeoutMs', { timeoutMs: -100 }], + ] as const)('rejects a non-positive or fractional %s at load', async (name, config) => { + const ctx = new Context() + await ctx.plugin(SystemPrompt) + await ctx.plugin(ToolRegistry) + await ctx.plugin(FakeBash) + await expect(ctx.plugin(ToolFsSearch, config)).rejects.toThrow(new RegExp(`tool-fs-search: ${name} must be a positive integer`)) + }) +}) + +describe('command construction (shell-safe)', () => { + it('glob: fixed rg --files template with quoted pattern and VCS excludes', () => { + const command = buildGlobCommand({ pattern: '**/*.ts' }) + expect(command).toBe( + "rg --files --glob='**/*.ts' --sort=modified --no-ignore --hidden " + + "--glob='!**/.git' --glob='!**/.svn' --glob='!**/.hg' --glob='!**/.bzr' --glob='!**/.jj' --glob='!**/.sl'", + ) + }) + + it('glob: the search root rides behind -- and is quoted', () => { + const command = buildGlobCommand({ pattern: '*.md', path: 'docs dir' }) + expect(command).toContain("-- 'docs dir'") + }) + + it('grep: fixed rg --json template with the pattern in --regexp= form', () => { + expect(buildGrepCommand({ pattern: 'foo.*bar' })).toBe("rg --json --regexp='foo.*bar'") + }) + + it('grep: include and path are quoted, include in --glob= form, path behind --', () => { + const command = buildGrepCommand({ pattern: 'x', path: '-leading-dash', include: '*.{ts,tsx}' }) + expect(command).toBe("rg --json --regexp='x' --glob='*.{ts,tsx}' -- '-leading-dash'") + }) + + it.each([ + ['a command-substitution pattern', '$(rm -rf /)', "'$(rm -rf /)'"], + ['a backtick pattern', '`touch pwned`', "'`touch pwned`'"], + ['a pattern with double quotes and spaces', 'say "hi there"', '\'say "hi there"\''], + ['a pattern with single quotes', "it's", '\'it\'\\\'\'s\''], + ['a pattern with newlines', 'a\nb', "'a\nb'"], + ['a leading-dash pattern', '--flag', "'--flag'"], + ['glob metacharacters', '*?[a-z]{x,y}', "'*?[a-z]{x,y}'"], + ])('quotes %s into one inert shell word', (_label, raw, quoted) => { + expect(buildGrepCommand({ pattern: raw })).toBe(`rg --json --regexp=${quoted}`) + }) +}) + +describe('workdir derivation and signal forwarding', () => { + it('forwards the session cwd as the request workdir', async () => { + const { ctx, bash } = await setup() + bash.handler = () => runResult('a.ts\n') + await call(ctx, 'glob', { pattern: '*' }, { agent: agent('/sessions/s1') }) + expect(bash.requests[0]?.workdir).toBe('/sessions/s1') + expect(bash.specs[0]?.workdir).toBe('/sessions/s1') + }) + + it('omits the request workdir without a session cwd so resolve() defaults apply', async () => { + const { ctx, bash } = await setup() + bash.handler = () => runResult('a.ts\n') + await call(ctx, 'glob', { pattern: '*' }, { agent: agent() }) + expect(bash.requests[0]).not.toHaveProperty('workdir') + expect(bash.specs[0]?.workdir).toBe('/work') + // A non-agent caller takes the same default path. + await call(ctx, 'grep', { pattern: 'x' }) + expect(bash.requests[1]).not.toHaveProperty('workdir') + }) + + it('forwards exec.signal into the bash spec (the abort reaches the backend)', async () => { + const { ctx, bash } = await setup() + const controller = new AbortController() + controller.abort() + bash.handler = spec => runResult('', { aborted: spec.signal?.aborted === true }) + const result = await call(ctx, 'grep', { pattern: 'x' }, { signal: controller.signal }) + expect(bash.specs[0]?.signal).toBe(controller.signal) + expect(result.isError).toBe(true) + expect(result.error).toMatchObject({ name: 'SearchError', code: 'SEARCH_ABORTED' }) + expect(text(result)).toContain('aborted') + }) + + it('reports the bash executor timeout as SEARCH_ABORTED with the budget', async () => { + const { ctx, bash } = await setup() + bash.handler = () => runResult('', { timedOut: true, timeoutMs: 1234, exitCode: null, signal: 'SIGTERM' }) + const result = await call(ctx, 'glob', { pattern: '*' }) + expect(result.isError).toBe(true) + expect(result.error).toMatchObject({ code: 'SEARCH_ABORTED' }) + expect(text(result)).toContain('timed out after 1234ms') + }) +}) + +describe('exit semantics and failure classification', () => { + it('exit 1 is a successful empty search', async () => { + const { ctx, bash } = await setup() + bash.handler = () => runResult('', { exitCode: 1 }) + const glob = await call(ctx, 'glob', { pattern: '*.nope' }) + expect(glob.isError).toBe(false) + expect(text(glob)).toBe('No files found') + const grep = await call(ctx, 'grep', { pattern: 'nope' }) + expect(grep.isError).toBe(false) + expect(text(grep)).toBe('No matches found') + }) + + it('a regex parse error classifies as SEARCH_INVALID_PATTERN', async () => { + const { ctx, bash } = await setup() + bash.handler = () => runResult('', { exitCode: 2, stderr: { text: 'rg: regex parse error:\n (\nerror: unclosed group', truncated: false } }) + const result = await call(ctx, 'grep', { pattern: '(' }) + expect(result.error).toMatchObject({ code: 'SEARCH_INVALID_PATTERN' }) + expect(text(result)).toContain('regex parse error') + }) + + it('a glob parse error classifies as SEARCH_INVALID_PATTERN', async () => { + const { ctx, bash } = await setup() + bash.handler = () => runResult('', { exitCode: 2, stderr: { text: 'rg: error parsing glob \'[\': unclosed character class', truncated: false } }) + const result = await call(ctx, 'glob', { pattern: '[' }) + expect(result.error).toMatchObject({ code: 'SEARCH_INVALID_PATTERN' }) + }) + + it('a missing rg binary classifies as SEARCH_FAILED naming ripgrep', async () => { + const { ctx, bash } = await setup() + bash.handler = () => runResult('', { exitCode: 127, stderr: { text: 'bash: line 1: rg: command not found', truncated: false } }) + const result = await call(ctx, 'glob', { pattern: '*' }) + expect(result.error).toMatchObject({ code: 'SEARCH_FAILED' }) + expect(text(result)).toContain('requires ripgrep (rg)') + // The same classification holds from either evidence alone: the 127 exit + // with silent stderr, or a shell's command-not-found text on another exit. + bash.handler = () => runResult('', { exitCode: 127 }) + expect(text(await call(ctx, 'glob', { pattern: '*' }))).toContain('requires ripgrep (rg)') + bash.handler = () => runResult('', { exitCode: 2, stderr: { text: 'sh: rg: command not found', truncated: false } }) + expect(text(await call(ctx, 'grep', { pattern: 'x' }))).toContain('requires ripgrep (rg)') + }) + + it('other nonzero exits are SEARCH_FAILED carrying the stderr excerpt', async () => { + const { ctx, bash } = await setup() + bash.handler = () => runResult('', { exitCode: 2, stderr: { text: 'rg: missing.dir: IO error: no such file or directory', truncated: false } }) + const result = await call(ctx, 'grep', { pattern: 'x', path: 'missing.dir' }) + expect(result.error).toMatchObject({ code: 'SEARCH_FAILED' }) + expect(text(result)).toContain('IO error') + }) + + it('a nonzero exit with EMPTY stderr still reports the exit code', async () => { + const { ctx, bash } = await setup() + bash.handler = () => runResult('', { exitCode: 3 }) + const result = await call(ctx, 'glob', { pattern: '*' }) + expect(result.error).toMatchObject({ code: 'SEARCH_FAILED' }) + expect(text(result)).toContain('exit 3') + }) + + it('truncated stderr gains a truncation note and stderr.spillPath is never read', async () => { + const { ctx, bash } = await setup() + bash.handler = () => runResult('', { + exitCode: 2, + stderr: { text: 'tail of diagnostics', truncated: true, spillPath: '/does/not/exist-and-never-read' }, + }) + const result = await call(ctx, 'grep', { pattern: 'x' }) + expect(text(result)).toContain('tail of diagnostics [stderr truncated]') + }) + + it('a signal kill (not timeout, not abort) is SEARCH_FAILED', async () => { + const { ctx, bash } = await setup() + bash.handler = () => runResult('', { exitCode: null, signal: 'SIGKILL' }) + const result = await call(ctx, 'grep', { pattern: 'x' }) + expect(result.error).toMatchObject({ code: 'SEARCH_FAILED' }) + expect(text(result)).toContain('SIGKILL') + }) + + it('a null exit with no signal (defensive) is SEARCH_FAILED', async () => { + const { ctx, bash } = await setup() + bash.handler = () => runResult('', { exitCode: null, signal: null }) + const result = await call(ctx, 'glob', { pattern: '*' }) + expect(result.error).toMatchObject({ code: 'SEARCH_FAILED' }) + }) +}) + +describe('raw output acquisition', () => { + let dir: string + afterEach(async () => { + await rm(dir, { recursive: true, force: true }) + }) + + it('parses the complete raw spill file when stdout is truncated', async () => { + dir = await mkdtemp(join(tmpdir(), 'dsh-search-raw-')) + const spillPath = join(dir, 'raw.txt') + await writeFile(spillPath, 'one.ts\ntwo.ts\nthree.ts\n') + const { ctx, bash } = await setup() + bash.handler = () => runResult('', { stdout: { text: 'one.ts\n', truncated: true, spillPath } }) + const result = await call(ctx, 'glob', { pattern: '*.ts' }) + expect(result.isError).toBe(false) + expect(text(result)).toBe('one.ts\ntwo.ts\nthree.ts') + }) + + it('fails with SEARCH_RAW_OUTPUT_OVERFLOW when the raw spill file exceeds the cap', async () => { + dir = await mkdtemp(join(tmpdir(), 'dsh-search-raw-')) + const spillPath = join(dir, 'raw.txt') + await writeFile(spillPath, 'x'.repeat(64)) + const { ctx, bash } = await setup({ config: { rawOutputMaxBytes: 16 } }) + bash.handler = () => runResult('', { stdout: { text: 'x', truncated: true, spillPath } }) + const result = await call(ctx, 'glob', { pattern: '*' }) + expect(result.error).toMatchObject({ code: 'SEARCH_RAW_OUTPUT_OVERFLOW' }) + expect(text(result)).toContain('narrow pattern, path, or include') + }) + + it('fails with SEARCH_RAW_OUTPUT_OVERFLOW when truncated stdout has no spill path', async () => { + dir = await mkdtemp(join(tmpdir(), 'dsh-search-raw-')) + const { ctx, bash } = await setup() + bash.handler = () => runResult('', { stdout: { text: 'partial', truncated: true } }) + const result = await call(ctx, 'grep', { pattern: 'x' }) + expect(result.error).toMatchObject({ code: 'SEARCH_RAW_OUTPUT_OVERFLOW' }) + }) + + it('fails with SEARCH_FAILED when the raw spill file cannot be read', async () => { + dir = await mkdtemp(join(tmpdir(), 'dsh-search-raw-')) + const { ctx, bash } = await setup() + bash.handler = () => runResult('', { stdout: { text: 'partial', truncated: true, spillPath: join(dir, 'gone.txt') } }) + const result = await call(ctx, 'glob', { pattern: '*' }) + expect(result.error).toMatchObject({ code: 'SEARCH_FAILED' }) + expect(text(result)).toContain('raw output spill file') + }) +}) + +describe('glob results', () => { + it('lists workdir-relative paths (absolute output under the workdir is relativized)', async () => { + const { ctx, bash } = await setup() + bash.handler = () => runResult('/sessions/s1/src/a.ts\n/elsewhere/b.ts\nrel/c.ts\n') + const result = await call(ctx, 'glob', { pattern: '*' }, { agent: agent('/sessions/s1') }) + expect(text(result)).toBe('src/a.ts\n/elsewhere/b.ts\nrel/c.ts') + }) + + it('validates arguments (blank pattern, blank path)', async () => { + const { ctx } = await setup() + expect(text(await call(ctx, 'glob', { pattern: ' ' }))).toContain('pattern must be a non-empty string') + expect(text(await call(ctx, 'glob', { pattern: '*', path: ' ' }))).toContain('path must be a non-empty string') + }) + + it('caps at globMaxResults and saves the FULL sorted list through spillFiles', async () => { + const { ctx, bash, spill } = await setup({ config: { globMaxResults: 2 }, spill: true }) + bash.handler = () => runResult('a.ts\nb.ts\nc.ts\nd.ts\n') + const result = await call(ctx, 'glob', { pattern: '*.ts' }, { agent: agent('/w') }) + expect(result.isError).toBe(false) + expect(text(result)).toBe('a.ts\nb.ts\n\n(Showing 2 of 4 paths. Full sorted result saved to: /spill/glob-results.txt. Use read with offset/limit to inspect it.)') + expect(spill?.saves).toHaveLength(1) + expect(spill?.saves[0]).toMatchObject({ + owner: { sessionId: 'session-1' }, + source: { toolName: 'glob', label: 'result' }, + suggestedName: 'glob-results.txt', + content: 'a.ts\nb.ts\nc.ts\nd.ts', + }) + expect(spill?.saves[0]?.source.callId).toBeDefined() + }) + + it('does not create a spill file when the result fits inline', async () => { + const { ctx, bash, spill } = await setup({ spill: true }) + bash.handler = () => runResult('a.ts\nb.ts\n') + const result = await call(ctx, 'glob', { pattern: '*' }, { agent: agent('/w') }) + expect(text(result)).toBe('a.ts\nb.ts') + expect(spill?.saves).toHaveLength(0) + }) + + it.each([ + ['no spill backend loaded', { fail: false, spill: false, ownerless: false }], + ['saveText fails', { fail: true, spill: true, ownerless: false }], + ['no session owner', { fail: false, spill: true, ownerless: true }], + ])('keeps the inline page and reports the unsaved remainder when %s', async (_label, mode) => { + const { ctx, bash, spill } = await setup({ config: { globMaxResults: 1 }, spill: mode.spill }) + if (mode.fail && spill) spill.failWith = new Error('disk full') + bash.handler = () => runResult('a.ts\nb.ts\n') + const result = await call(ctx, 'glob', { pattern: '*' }, mode.ownerless ? {} : { agent: agent('/w') }) + expect(result.isError).toBe(false) // spill unavailability never fails the search + expect(text(result)).toBe('a.ts\n\n(Showing 1 of 2 paths. The complete result could not be saved; narrow pattern or path to see more.)') + }) +}) + +describe('grep results', () => { + it('groups matches by file with line numbers', async () => { + const { ctx, bash } = await setup() + bash.handler = () => runResult([ + JSON.stringify({ type: 'begin', data: { path: { text: 'a.ts' } } }), + matchLine('a.ts', 3, 'const x = 1\n'), + matchLine('a.ts', 9, 'const y = 2\n'), + JSON.stringify({ type: 'end', data: { path: { text: 'a.ts' } } }), + matchLine('b.ts', 1, 'const z = 3'), + JSON.stringify({ type: 'summary', data: {} }), + '', + ].join('\n')) + const result = await call(ctx, 'grep', { pattern: 'const' }) + expect(result.isError).toBe(false) + expect(text(result)).toBe('Found 3 matches\n\na.ts\nLine 3: const x = 1\nLine 9: const y = 2\n\nb.ts\nLine 1: const z = 3') + }) + + it('reports a single match in the singular', async () => { + const { ctx, bash } = await setup() + bash.handler = () => runResult(`${matchLine('a.ts', 1, 'hit')}\n`) + expect(text(await call(ctx, 'grep', { pattern: 'hit' }))).toBe('Found 1 match\n\na.ts\nLine 1: hit') + }) + + it('relativizes absolute match paths against the resolved workdir', async () => { + const { ctx, bash } = await setup() + bash.handler = () => runResult(`${matchLine('/sessions/s1/deep/a.ts', 2, 'hit')}\n`) + const result = await call(ctx, 'grep', { pattern: 'hit', path: '/sessions/s1' }, { agent: agent('/sessions/s1') }) + expect(text(result)).toContain('deep/a.ts\nLine 2: hit') + }) + + it('previews a long matched line at grepMaxLineBytes preserving UTF-8', async () => { + const { ctx, bash } = await setup({ config: { grepMaxLineBytes: 7 } }) + // 'héllo wörld' cut at 7 bytes lands mid-'é'? h(1)é(2)l(1)l(1)o(1)=6, space=7 → clean cut at 7. + // Use a multibyte straddle instead: 'aé' repeated — cut at 7 bytes: a(1)é(2)a(1)é(2)=6 +a(1)=7 → next é straddles: trimmed. + bash.handler = () => runResult(`${matchLine('a.txt', 1, 'aéaéaéaé')}\n`) + const result = await call(ctx, 'grep', { pattern: 'a' }) + expect(text(result)).toContain('Line 1: aéaéa (line truncated)') + }) + + it('renders a non-UTF-8 line (rg bytes form) as a placeholder instead of failing', async () => { + const { ctx, bash } = await setup() + const record = JSON.stringify({ type: 'match', data: { path: { text: 'bin.dat' }, lines: { bytes: 'AAECww==' }, line_number: 4 } }) + bash.handler = () => runResult(`${record}\n`) + expect(text(await call(ctx, 'grep', { pattern: 'x' }))).toContain('Line 4: (line is not valid UTF-8)') + }) + + it('strips a CRLF terminator from the matched line text', () => { + const matches = parseGrepMatches(`${matchLine('a.txt', 1, 'windows line\r\n')}\n`) + expect(matches[0]?.line).toBe('windows line') + }) + + it('caps at grepMaxMatches and spills the full formatted match list', async () => { + const { ctx, bash, spill } = await setup({ config: { grepMaxMatches: 2 }, spill: true }) + bash.handler = () => runResult([ + matchLine('a.ts', 1, 'one'), + matchLine('a.ts', 2, 'two'), + matchLine('b.ts', 3, 'three'), + '', + ].join('\n')) + const result = await call(ctx, 'grep', { pattern: 'e' }, { agent: agent('/w') }) + expect(text(result)).toBe('Found 2 of 3 matches\n\na.ts\nLine 1: one\nLine 2: two\n\n(Full grep result saved to: /spill/grep-results.txt. Use read with offset/limit to inspect it.)') + expect(spill?.saves[0]).toMatchObject({ + source: { toolName: 'grep', label: 'result' }, + suggestedName: 'grep-results.txt', + content: 'Found 3 matches\n\na.ts\nLine 1: one\nLine 2: two\n\nb.ts\nLine 3: three', + }) + }) + + it('reports the unsaved remainder when capped with no spill backend', async () => { + const { ctx, bash } = await setup({ config: { grepMaxMatches: 1 } }) + bash.handler = () => runResult(`${matchLine('a.ts', 1, 'one')}\n${matchLine('a.ts', 2, 'two')}\n`) + const result = await call(ctx, 'grep', { pattern: 'o' }, { agent: agent('/w') }) + expect(result.isError).toBe(false) + expect(text(result)).toBe('Found 1 of 2 matches\n\na.ts\nLine 1: one\n\n(The complete result could not be saved; narrow pattern, path, or include to see more.)') + }) + + it('validates arguments (empty pattern, blank path, bad include)', async () => { + const { ctx } = await setup() + expect(text(await call(ctx, 'grep', { pattern: '' }))).toContain('pattern must be a non-empty string') + expect(text(await call(ctx, 'grep', { pattern: 'x', path: ' ' }))).toContain('path must be a non-empty string') + expect(text(await call(ctx, 'grep', { pattern: 'x', include: ' ' }))).toContain('include must be a non-empty glob') + expect(text(await call(ctx, 'grep', { pattern: 'x', include: '!*.ts' }))).toContain('negated patterns') + expect(text(await call(ctx, 'grep', { pattern: 'x', include: '*.ts,*.js' }))).toContain('comma-separated list') + }) + + it('accepts a whitespace-only pattern (a legitimate regex) and brace alternation in include', async () => { + const { ctx, bash } = await setup() + bash.handler = () => runResult('', { exitCode: 1 }) + const result = await call(ctx, 'grep', { pattern: ' ', include: '*.{ts,tsx}' }) + expect(result.isError).toBe(false) + }) +}) + +describe('rg --json transport failures (SEARCH_FAILED)', () => { + it.each([ + ['a non-JSON line', 'not json at all'], + ['a non-object record', '42'], + ['a match record with no data', JSON.stringify({ type: 'match' })], + ['a match record with no path text', JSON.stringify({ type: 'match', data: { path: {}, lines: { text: 'x' }, line_number: 1 } })], + ['a match record with a non-object path', JSON.stringify({ type: 'match', data: { path: 'a.ts', lines: { text: 'x' }, line_number: 1 } })], + ['a match record with no line number', JSON.stringify({ type: 'match', data: { path: { text: 'a.ts' }, lines: { text: 'x' } } })], + ['a match record with no line content', JSON.stringify({ type: 'match', data: { path: { text: 'a.ts' }, line_number: 1 } })], + ['a match record with neither text nor bytes', JSON.stringify({ type: 'match', data: { path: { text: 'a.ts' }, lines: {}, line_number: 1 } })], + ])('%s fails the search', async (_label, line) => { + const { ctx, bash } = await setup() + bash.handler = () => runResult(`${line}\n`) + const result = await call(ctx, 'grep', { pattern: 'x' }) + expect(result.isError).toBe(true) + expect(result.error).toMatchObject({ name: 'SearchError', code: 'SEARCH_FAILED' }) + }) +}) + +describe('the no-background-task invariant', () => { + it('never calls ctx.bash.start() across successful and failed searches', async () => { + const { ctx, bash } = await setup() + bash.handler = () => runResult('a.ts\n') + await call(ctx, 'glob', { pattern: '*' }) + bash.handler = () => runResult('', { exitCode: 2, stderr: { text: 'boom', truncated: false } }) + await call(ctx, 'grep', { pattern: 'x' }) + expect(bash.startCalls).toBe(0) + }) +}) + +describe('presentation', () => { + it('glob titles carry the pattern and optional root', () => { + expect(presentGlobCall({ pattern: '**/*.ts' })).toMatchObject({ card: 'generic', title: 'Glob **/*.ts', kind: 'search' }) + expect(presentGlobCall({ pattern: '*.md', path: 'docs' }).title).toBe('Glob *.md in docs') + }) + + it('grep titles carry the pattern, target, and include filter', () => { + expect(presentGrepCall({ pattern: 'todo' })).toMatchObject({ card: 'generic', title: 'Grep todo', kind: 'search' }) + expect(presentGrepCall({ pattern: 'todo', path: 'src', include: '*.ts' }).title).toBe('Grep todo in src (*.ts)') + }) +}) + +describe('helpers', () => { + it('toWorkdirRelative maps inside-workdir absolutes and passes everything else through', () => { + expect(toWorkdirRelative('/w/a/b.ts', '/w')).toBe('a/b.ts') + expect(toWorkdirRelative('/w', '/w')).toBe('.') + expect(toWorkdirRelative('/other/b.ts', '/w')).toBe('/other/b.ts') + expect(toWorkdirRelative('/w-sibling/b.ts', '/w')).toBe('/w-sibling/b.ts') + expect(toWorkdirRelative('rel/b.ts', '/w')).toBe('rel/b.ts') + // Normalization makes this land OUTSIDE the workdir → original path kept. + expect(toWorkdirRelative('/w/../up.ts', '/w')).toBe('/w/../up.ts') + }) + + it('previewLine keeps a within-budget line untouched', () => { + expect(previewLine('short', 100)).toBe('short') + }) + + it('formatGrepMatches groups by first-seen file order', () => { + const grouped = formatGrepMatches([ + { path: 'b.ts', lineNumber: 2, line: 'x' }, + { path: 'a.ts', lineNumber: 1, line: 'y' }, + { path: 'b.ts', lineNumber: 5, line: 'z' }, + ]) + expect(grouped).toBe('b.ts\nLine 2: x\nLine 5: z\n\na.ts\nLine 1: y') + }) +}) diff --git a/packages/fs/tool-fs-search/tsconfig.json b/packages/fs/tool-fs-search/tsconfig.json new file mode 100644 index 0000000000..9241aca15b --- /dev/null +++ b/packages/fs/tool-fs-search/tsconfig.json @@ -0,0 +1,20 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "lib/types" + }, + "include": ["src"], + "references": [ + { "path": "../../../vendor/cosmokit" }, + { "path": "../../../vendor/cordis" }, + { "path": "../../../vendor/schemastery" }, + { "path": "../../util/retention" }, + { "path": "../../llm/llm" }, + { "path": "../../core/session" }, + { "path": "../../core/tools" }, + { "path": "../../core/system-prompt" }, + { "path": "../../bash/bash" }, + { "path": "../../spill/spill" } + ] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 18129c26cc..9d0c641453 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -403,6 +403,43 @@ importers: specifier: ^4.0.0-rc.6 version: 4.0.0-rc.6(@cordisjs/plugin-include@1.0.4)(@cordisjs/plugin-loader@1.0.0-rc.4) + packages/fs/tool-fs-search: + dependencies: + schemastery: + specifier: ^3.18.0 + version: 3.18.0 + devDependencies: + '@deepseek-ai/dsh-agent': + specifier: workspace:^ + version: link:../../core/agent + '@deepseek-ai/dsh-bash': + specifier: workspace:^ + version: link:../../bash/bash + '@deepseek-ai/dsh-bash-local': + specifier: workspace:^ + version: link:../../bash/bash-local + '@deepseek-ai/dsh-llm': + specifier: workspace:^ + version: link:../../llm/llm + '@deepseek-ai/dsh-retention': + specifier: workspace:^ + version: link:../../util/retention + '@deepseek-ai/dsh-session': + specifier: workspace:^ + version: link:../../core/session + '@deepseek-ai/dsh-spill': + specifier: workspace:^ + version: link:../../spill/spill + '@deepseek-ai/dsh-system-prompt': + specifier: workspace:^ + version: link:../../core/system-prompt + '@deepseek-ai/dsh-tools': + specifier: workspace:^ + version: link:../../core/tools + cordis: + specifier: ^4.0.0-rc.6 + version: 4.0.0-rc.6(@cordisjs/plugin-include@1.0.4)(@cordisjs/plugin-loader@1.0.0-rc.4) + packages/guard/repeat-tool-guard: dependencies: schemastery: diff --git a/scripts/gen-tool-catalog.ts b/scripts/gen-tool-catalog.ts index 24739e1388..e808dfdd1b 100644 --- a/scripts/gen-tool-catalog.ts +++ b/scripts/gen-tool-catalog.ts @@ -48,6 +48,7 @@ import SubagentService from '@deepseek-ai/dsh-subagent' import * as SubagentMock from '@deepseek-ai/dsh-subagent-mock' import * as ToolBash from '@deepseek-ai/dsh-tool-bash' import * as ToolFs from '@deepseek-ai/dsh-tool-fs' +import * as ToolFsSearch from '@deepseek-ai/dsh-tool-fs-search' import * as ToolTodo from '@deepseek-ai/dsh-tool-todo' import * as ToolSubagent from '@deepseek-ai/dsh-tool-subagent' import * as ToolWeb from '@deepseek-ai/dsh-tool-web' @@ -129,6 +130,23 @@ const TOOL_PACKAGES: ToolPackage[] = [ note: 'The read-before-write/edit policy is added by `@deepseek-ai/dsh-fs-policy` (an `fs/*` event-gate plugin, no schema change); a deployment that loads these tools is expected to also load it. The tool schemas above are identical with or without the policy plugin.', }, + { + pkg: '@deepseek-ai/dsh-tool-fs-search', + dir: 'tool-fs-search', + source: 'packages/fs/tool-fs-search/src/index.ts', + requires: ['ctx.tools', 'ctx.bash', 'ctx.systemPrompt'], + writes: ['tool/call', 'tool/result'], + async mount(ctx) { + // The tools inject `bash` (search executes fixed `rg` commands through + // the executor seam, not ctx.fs); boot the local executor to satisfy it. + // `ctx.spillFiles` is optional (read via ctx.get) and does not affect the + // schemas, so no spill backend is mounted. + await ctx.plugin(LocalBashExecutor) + await ctx.plugin(ToolFsSearch) + }, + note: + 'glob and grep are bash-backed discovery tools: they run fixed ripgrep commands through ctx.bash as ordinary foreground calls (never background tasks). Capped results save the complete formatted list through the optional ctx.spillFiles backend; returned paths are follow-up-readable in co-located bash/filesystem deployments.', + }, { pkg: '@deepseek-ai/dsh-tool-subagent', dir: 'tool-subagent', diff --git a/tsconfig.build.json b/tsconfig.build.json index 625efd3a44..4bb8da6759 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -36,6 +36,7 @@ { "path": "./packages/fs/fs-local" }, { "path": "./packages/fs/fs-policy" }, { "path": "./packages/fs/tool-fs" }, + { "path": "./packages/fs/tool-fs-search" }, { "path": "./packages/web/web" }, { "path": "./packages/web/web-search-exa" }, { "path": "./packages/web/web-search-perplexity" }, diff --git a/tsconfig.json b/tsconfig.json index 91fc52ebf3..e65121cf19 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -45,6 +45,7 @@ { "path": "./packages/fs/fs-local" }, { "path": "./packages/fs/fs-policy" }, { "path": "./packages/fs/tool-fs" }, + { "path": "./packages/fs/tool-fs-search" }, { "path": "./packages/compact/compact" }, { "path": "./packages/compact/compact-basic" }, { "path": "./packages/web/web" }, From e94305d99ebcee7c93208b3e347b4ee791ada0b2 Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Thu, 9 Jul 2026 21:12:41 +0800 Subject: [PATCH 15/24] fix: address codex review round 1 Two functional gaps in the search tools change: - Enforce rawOutputMaxBytes on UNTRUNCATED inline stdout too. The cap was only checked on the truncated->raw-spill path, so an executor retaining more inline than the search cap (or a deployment lowering the cap below the bash retention) could smuggle an over-cap parse through, contradicting the documented SEARCH_RAW_OUTPUT_OVERFLOW contract. Covered by a new over-cap-inline test. - Load @deepseek-ai/dsh-timeout-policy in the coding-agent tree. The search tools declare timeoutMs but nothing in the demo enforced it, so the advertised 30s budget silently degraded to the bash executor's 60s backstop. The keyless smoke boots the amended tree. --- examples/coding-agent/composition.md | 3 +++ examples/coding-agent/cordis.yml | 7 ++++++ packages/fs/tool-fs-search/src/search-core.ts | 23 ++++++++++++++----- .../fs/tool-fs-search/tests/tools.spec.ts | 12 ++++++++++ 4 files changed, 39 insertions(+), 6 deletions(-) diff --git a/examples/coding-agent/composition.md b/examples/coding-agent/composition.md index c6c624f0fb..9896c44420 100644 --- a/examples/coding-agent/composition.md +++ b/examples/coding-agent/composition.md @@ -45,6 +45,8 @@ flowchart LR cfg --> plugin_coding_tool_fs plugin_coding_tool_fs_search["tool-fs-search
@deepseek-ai/dsh-tool-fs-search"] cfg --> plugin_coding_tool_fs_search + plugin_coding_timeout_policy["timeout-policy
@deepseek-ai/dsh-timeout-policy"] + cfg --> plugin_coding_timeout_policy plugin_coding_spill_local["spill-local
@deepseek-ai/dsh-spill-local"] cfg --> plugin_coding_spill_local plugin_coding_spill_policy["spill-policy
@deepseek-ai/dsh-spill-policy"] @@ -68,6 +70,7 @@ flowchart LR | `fs-policy` | `@deepseek-ai/dsh-fs-policy` | | `tool-fs` | `@deepseek-ai/dsh-tool-fs` | | `tool-fs-search` | `@deepseek-ai/dsh-tool-fs-search` | +| `timeout-policy` | `@deepseek-ai/dsh-timeout-policy` | | `spill-local` | `@deepseek-ai/dsh-spill-local` | | `spill-policy` | `@deepseek-ai/dsh-spill-policy` | diff --git a/examples/coding-agent/cordis.yml b/examples/coding-agent/cordis.yml index 5861e6a602..c175bbaf97 100644 --- a/examples/coding-agent/cordis.yml +++ b/examples/coding-agent/cordis.yml @@ -127,6 +127,13 @@ - id: tool-fs-search name: '@deepseek-ai/dsh-tool-fs-search' +# The tool-call timeout enforcer: arms each declared ToolDefinition.timeoutMs +# (the search tools above declare 30s) as a deadline on exec.signal. Without +# it a declared budget is advisory and only the bash executor's own timeout +# backstop applies. +- id: timeout-policy + name: '@deepseek-ai/dsh-timeout-policy' + # Tool-output spill stack: a local backend that saves oversized tool text under # a private session-scoped dir, and the tools/post-execute policy that replaces # an over-budget plain-text result with a preview + the spill path (the model diff --git a/packages/fs/tool-fs-search/src/search-core.ts b/packages/fs/tool-fs-search/src/search-core.ts index 2d9d0b63dd..b73c31bd1f 100644 --- a/packages/fs/tool-fs-search/src/search-core.ts +++ b/packages/fs/tool-fs-search/src/search-core.ts @@ -103,15 +103,26 @@ function classifyRunFailure(toolName: string, result: BashRunResult): SearchErro } /** - * Acquire the COMPLETE raw stdout of a finished run. Untruncated stdout is used - * as-is; truncated stdout is recovered from the executor's local raw spill file - * only when the complete file fits within `rawOutputMaxBytes`. A missing spill - * path or an over-cap file is a clear failure telling the model to narrow the - * search — never a silently-partial parse. + * Acquire the COMPLETE raw stdout of a finished run, enforcing + * `rawOutputMaxBytes` on BOTH transports: inline executor text (an executor + * retaining more than this package's cap must not smuggle an over-cap parse + * through the untruncated path) and the executor's local raw spill file, read + * only when the complete file fits the cap. A missing spill path or over-cap + * output is a clear failure telling the model to narrow the search — never a + * silently-partial parse. */ async function completeStdout(toolName: string, result: BashRunResult, rawOutputMaxBytes: number): Promise { - if (!result.stdout.truncated) return result.stdout.text const narrow = 'narrow pattern, path, or include and retry' + if (!result.stdout.truncated) { + const inlineBytes = Buffer.byteLength(result.stdout.text, 'utf8') + if (inlineBytes > rawOutputMaxBytes) { + throw new SearchError( + `${toolName} produced ${inlineBytes} bytes of raw output, over the ${rawOutputMaxBytes}-byte cap; ${narrow}`, + 'SEARCH_RAW_OUTPUT_OVERFLOW', + ) + } + return result.stdout.text + } const spillPath = result.stdout.spillPath if (spillPath === undefined) { throw new SearchError( diff --git a/packages/fs/tool-fs-search/tests/tools.spec.ts b/packages/fs/tool-fs-search/tests/tools.spec.ts index 6a539956a4..55b54555ae 100644 --- a/packages/fs/tool-fs-search/tests/tools.spec.ts +++ b/packages/fs/tool-fs-search/tests/tools.spec.ts @@ -392,6 +392,18 @@ describe('raw output acquisition', () => { expect(text(result)).toContain('narrow pattern, path, or include') }) + it('fails with SEARCH_RAW_OUTPUT_OVERFLOW when UNTRUNCATED inline stdout exceeds the cap', async () => { + // An executor retaining more inline than this package's cap (or a + // deployment lowering rawOutputMaxBytes below the bash retention) must not + // smuggle an over-cap parse through the untruncated path. + dir = await mkdtemp(join(tmpdir(), 'dsh-search-raw-')) + const { ctx, bash } = await setup({ config: { rawOutputMaxBytes: 16 } }) + bash.handler = () => runResult(`${'x'.repeat(64)}\n`) + const result = await call(ctx, 'grep', { pattern: 'x' }) + expect(result.error).toMatchObject({ name: 'SearchError', code: 'SEARCH_RAW_OUTPUT_OVERFLOW' }) + expect(text(result)).toContain('narrow pattern, path, or include') + }) + it('fails with SEARCH_RAW_OUTPUT_OVERFLOW when truncated stdout has no spill path', async () => { dir = await mkdtemp(join(tmpdir(), 'dsh-search-raw-')) const { ctx, bash } = await setup() From 590f520949dcd57bb52aebd713a5fa593bf209de Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Thu, 9 Jul 2026 21:28:42 +0800 Subject: [PATCH 16/24] fix: address codex review round 2 Translate ctx.bash.run() REJECTIONS into the SEARCH_* taxonomy. The seam contract has run() reject for infrastructure failures (a pre-aborted signal, an unusable/deleted session workdir, a missing shell); the bare await let those escape as plain Errors, so the tool registry produced isError results without the structured SearchError { name, code } the package documents. A pre-aborted spec.signal now maps to SEARCH_ABORTED and any other start failure to SEARCH_FAILED, original error chained as cause. Covered by fake-executor tests for both branches plus real-executor integration tests pinning the exact pre-aborted-signal and deleted-cwd paths. --- packages/fs/tool-fs-search/src/search-core.ts | 19 +++++++++++++-- .../tool-fs-search/tests/integration.spec.ts | 23 +++++++++++++++++++ .../fs/tool-fs-search/tests/tools.spec.ts | 21 +++++++++++++++++ 3 files changed, 61 insertions(+), 2 deletions(-) diff --git a/packages/fs/tool-fs-search/src/search-core.ts b/packages/fs/tool-fs-search/src/search-core.ts index b73c31bd1f..a47adfa7c3 100644 --- a/packages/fs/tool-fs-search/src/search-core.ts +++ b/packages/fs/tool-fs-search/src/search-core.ts @@ -158,7 +158,11 @@ async function completeStdout(toolName: string, result: BashRunResult, rawOutput * success with zero results (`noMatches`), anything else throws a * {@link SearchError} (abort/timeout → `SEARCH_ABORTED`, invalid pattern → * `SEARCH_INVALID_PATTERN`, the rest → `SEARCH_FAILED` / - * `SEARCH_RAW_OUTPUT_OVERFLOW`). + * `SEARCH_RAW_OUTPUT_OVERFLOW`). A `run()` REJECTION — the seam's + * infrastructure failures (pre-aborted signal, unusable workdir, missing + * shell) — is translated into the same taxonomy: a pre-aborted signal becomes + * `SEARCH_ABORTED`, everything else `SEARCH_FAILED`, with the original as + * `cause`. * * @param ctx - the plugin context; execution uses its `bash` service. * @param exec - the tool-execution context; supplies the session cwd and the abort signal. @@ -180,7 +184,18 @@ export async function runRipgrep( ...cwd !== undefined ? { workdir: cwd } : {}, ...exec.signal ? { signal: exec.signal } : {}, }) - const result = await ctx.bash.run(spec) + let result: BashRunResult + try { + result = await ctx.bash.run(spec) + } catch (error: unknown) { + // The seam contract: run() REJECTS only for infrastructure failures — a + // pre-aborted signal, an unusable workdir, a missing shell. Translate them + // so these failures stay machine-routable under the SEARCH_* taxonomy. + if (spec.signal?.aborted === true) { + throw new SearchError(`${toolName} was aborted before completion (tool timeout or caller cancellation)`, 'SEARCH_ABORTED', { cause: error }) + } + throw new SearchError(`${toolName} could not start its search command (unusable working directory or missing shell)`, 'SEARCH_FAILED', { cause: error }) + } if (result.aborted) { throw new SearchError(`${toolName} was aborted before completion (tool timeout or caller cancellation)`, 'SEARCH_ABORTED') } diff --git a/packages/fs/tool-fs-search/tests/integration.spec.ts b/packages/fs/tool-fs-search/tests/integration.spec.ts index 74a418238b..481d3af620 100644 --- a/packages/fs/tool-fs-search/tests/integration.spec.ts +++ b/packages/fs/tool-fs-search/tests/integration.spec.ts @@ -158,4 +158,27 @@ describe.skipIf(!hasRg)('search tools over the real bash executor + real rg', () } }) }) + + describe('bash-start infrastructure failures stay in the SEARCH_* taxonomy', () => { + it('a pre-aborted exec.signal (real executor rejects before spawn) is SEARCH_ABORTED', async () => { + const controller = new AbortController() + controller.abort() + const result = await ctx.tools.execute({ + callId: CallId(`it-${++callCounter}`), + name: 'grep', + arguments: { pattern: 'x' }, + signal: controller.signal, + }) + expect(result.isError).toBe(true) + expect(result.error).toMatchObject({ name: 'SearchError', code: 'SEARCH_ABORTED' }) + }) + + it('an unusable session cwd (spawn failure) is SEARCH_FAILED', async () => { + const gone = join(dir, 'deleted-session-dir') + const result = await call('glob', { pattern: '*' }, { session: { header: { id: 'session-int', cwd: gone } } }) + expect(result.isError).toBe(true) + expect(result.error).toMatchObject({ name: 'SearchError', code: 'SEARCH_FAILED' }) + expect(text(result)).toContain('could not start') + }) + }) }) diff --git a/packages/fs/tool-fs-search/tests/tools.spec.ts b/packages/fs/tool-fs-search/tests/tools.spec.ts index 55b54555ae..6afca54efd 100644 --- a/packages/fs/tool-fs-search/tests/tools.spec.ts +++ b/packages/fs/tool-fs-search/tests/tools.spec.ts @@ -279,6 +279,27 @@ describe('workdir derivation and signal forwarding', () => { expect(result.error).toMatchObject({ code: 'SEARCH_ABORTED' }) expect(text(result)).toContain('timed out after 1234ms') }) + + it('translates a run() rejection under a pre-aborted signal into SEARCH_ABORTED', async () => { + // The seam contract: run() REJECTS for a pre-aborted signal (it never + // spawns). The plain rejection must not escape the SEARCH_* taxonomy. + const { ctx, bash } = await setup() + const controller = new AbortController() + controller.abort() + bash.handler = () => { throw new Error('aborted before spawn') } + const result = await call(ctx, 'grep', { pattern: 'x' }, { signal: controller.signal }) + expect(result.isError).toBe(true) + expect(result.error).toMatchObject({ name: 'SearchError', code: 'SEARCH_ABORTED' }) + }) + + it('translates a run() rejection without an abort (unusable workdir) into SEARCH_FAILED', async () => { + const { ctx, bash } = await setup() + bash.handler = () => { throw new Error('spawn bash ENOENT') } + const result = await call(ctx, 'glob', { pattern: '*' }) + expect(result.isError).toBe(true) + expect(result.error).toMatchObject({ name: 'SearchError', code: 'SEARCH_FAILED' }) + expect(text(result)).toContain('could not start') + }) }) describe('exit semantics and failure classification', () => { From 460a58639ae5aba84f97c24f8ba957d25b64db94 Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Thu, 9 Jul 2026 21:47:39 +0800 Subject: [PATCH 17/24] fix: address codex review round 3 glob leaked VCS internals when the model rooted the search AT a VCS directory (path: '.git' or 'sub/.git'): the prune glob !**/.git is matched against root-prefixed candidate paths, which never end in the directory name when the walk starts inside it. Pair each VCS exclude with a contents glob (!**//**), verified empirically to exclude relative, nested, and absolute VCS roots while leaving broad searches untouched. Pinned by the command-construction test and a real-rg integration case rooting at .git. --- packages/fs/tool-fs-search/src/glob.ts | 18 ++++++++++++++---- .../tool-fs-search/tests/integration.spec.ts | 6 ++++++ packages/fs/tool-fs-search/tests/tools.spec.ts | 6 ++++-- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/packages/fs/tool-fs-search/src/glob.ts b/packages/fs/tool-fs-search/src/glob.ts index e565699e2c..09a3e1d9ce 100644 --- a/packages/fs/tool-fs-search/src/glob.ts +++ b/packages/fs/tool-fs-search/src/glob.ts @@ -29,9 +29,12 @@ export const GLOB_MAX_RESULTS = 100 /** * Directory names ripgrep must never descend into for a discovery listing: VCS * metadata stores. `--no-ignore --hidden` would otherwise surface them in every - * broad search. Each is excluded with a negated any-depth `--glob` (see - * {@link buildGlobCommand}), which matches — and prunes — the directory - * wherever it appears. + * broad search. Each name is excluded with TWO negated `--glob`s (see + * {@link buildGlobCommand}): an any-depth directory glob that matches — and + * prunes — the directory during traversal, and a contents glob that still + * excludes the internals when the search root itself is at or inside the + * directory (an explicit `path` of `.git` or `sub/.git`), where the prune glob + * alone never matches. */ export const GLOB_VCS_EXCLUDES: readonly string[] = ['.git', '.svn', '.hg', '.bzr', '.jj', '.sl'] @@ -81,7 +84,14 @@ export function buildGlobCommand(input: GlobInput): string { 'rg --files', `--glob=${singleQuote(input.pattern)}`, '--sort=modified --no-ignore --hidden', - ...GLOB_VCS_EXCLUDES.map(name => `--glob=${singleQuote(`!**/${name}`)}`), + // Two negated globs per VCS name: the bare form prunes the directory + // during traversal; the /** form still excludes the contents when the + // search root is AT or INSIDE the directory (where the bare form, + // matched against root-prefixed paths, never fires). + ...GLOB_VCS_EXCLUDES.flatMap(name => [ + `--glob=${singleQuote(`!**/${name}`)}`, + `--glob=${singleQuote(`!**/${name}/**`)}`, + ]), ] if (input.path !== undefined) parts.push('--', singleQuote(input.path)) return parts.join(' ') diff --git a/packages/fs/tool-fs-search/tests/integration.spec.ts b/packages/fs/tool-fs-search/tests/integration.spec.ts index 481d3af620..36fb2c28e6 100644 --- a/packages/fs/tool-fs-search/tests/integration.spec.ts +++ b/packages/fs/tool-fs-search/tests/integration.spec.ts @@ -87,6 +87,12 @@ describe.skipIf(!hasRg)('search tools over the real bash executor + real rg', () expect(text(await call('glob', { pattern: '*.nomatch' }))).toBe('No files found') }) + it('excludes VCS internals even when the search root IS the VCS directory', async () => { + // The prune glob alone never matches root-prefixed paths when rg is + // rooted at .git; the paired contents glob keeps the exclusion airtight. + expect(text(await call('glob', { pattern: '*', path: '.git' }))).toBe('No files found') + }) + it('classifies an invalid glob as SEARCH_INVALID_PATTERN', async () => { const result = await call('glob', { pattern: '[' }) expect(result.isError).toBe(true) diff --git a/packages/fs/tool-fs-search/tests/tools.spec.ts b/packages/fs/tool-fs-search/tests/tools.spec.ts index 6afca54efd..a3a3e89cce 100644 --- a/packages/fs/tool-fs-search/tests/tools.spec.ts +++ b/packages/fs/tool-fs-search/tests/tools.spec.ts @@ -204,11 +204,13 @@ describe('config validation', () => { }) describe('command construction (shell-safe)', () => { - it('glob: fixed rg --files template with quoted pattern and VCS excludes', () => { + it('glob: fixed rg --files template with quoted pattern and paired VCS excludes', () => { const command = buildGlobCommand({ pattern: '**/*.ts' }) expect(command).toBe( "rg --files --glob='**/*.ts' --sort=modified --no-ignore --hidden " - + "--glob='!**/.git' --glob='!**/.svn' --glob='!**/.hg' --glob='!**/.bzr' --glob='!**/.jj' --glob='!**/.sl'", + + "--glob='!**/.git' --glob='!**/.git/**' --glob='!**/.svn' --glob='!**/.svn/**' " + + "--glob='!**/.hg' --glob='!**/.hg/**' --glob='!**/.bzr' --glob='!**/.bzr/**' " + + "--glob='!**/.jj' --glob='!**/.jj/**' --glob='!**/.sl' --glob='!**/.sl/**'", ) }) From 1df9f3a84ade29f3393afbe4737f014b5ee4dde1 Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Thu, 9 Jul 2026 22:00:43 +0800 Subject: [PATCH 18/24] test: cover the glob path arg in the fake-executor tier CI has no rg, so the integration suite self-skips there and the fake-executor suite must carry the per-file 100% coverage gate alone. parseGlobArgs's valid-path branch was only exercised by integration (node 24 / coverage failed at 95.45% branches on glob.ts); a fake-tier test now threads a valid path through to the quoted `-- 'sub'` root. --- packages/fs/tool-fs-search/tests/tools.spec.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/fs/tool-fs-search/tests/tools.spec.ts b/packages/fs/tool-fs-search/tests/tools.spec.ts index a3a3e89cce..7abbe20638 100644 --- a/packages/fs/tool-fs-search/tests/tools.spec.ts +++ b/packages/fs/tool-fs-search/tests/tools.spec.ts @@ -459,6 +459,14 @@ describe('glob results', () => { expect(text(await call(ctx, 'glob', { pattern: '*', path: ' ' }))).toContain('path must be a non-empty string') }) + it('threads a valid path through to the command as the quoted search root', async () => { + const { ctx, bash } = await setup() + bash.handler = () => runResult('sub/a.ts\n') + const result = await call(ctx, 'glob', { pattern: '*.ts', path: 'sub' }) + expect(result.isError).toBe(false) + expect(bash.specs[0]?.command).toContain("-- 'sub'") + }) + it('caps at globMaxResults and saves the FULL sorted list through spillFiles', async () => { const { ctx, bash, spill } = await setup({ config: { globMaxResults: 2 }, spill: true }) bash.handler = () => runResult('a.ts\nb.ts\nc.ts\nd.ts\n') From 31be42e92d1792c74466a0f7afe6e2a2d7f18106 Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Fri, 10 Jul 2026 10:07:32 +0800 Subject: [PATCH 19/24] test: add ACP spill snapshot coverage --- ...026-07-06-tool-result-retention-library.md | 2 +- examples/acp-agent/README.md | 2 +- examples/acp-agent/composition.md | 6 ++++ examples/acp-agent/cordis.yml | 15 +++++++++ examples/acp-agent/tests/acp.snapshot.ts | 1 + .../tests/snapshots/bash-spill/input.json | 7 +++++ .../tests/snapshots/bash-spill/session.jsonl | 23 ++++++++++++++ .../snapshots/bash-spill/stdout.golden.jsonl | 6 ++++ .../support/acp-snapshot/src/normalize.ts | 7 +++++ .../acp-snapshot/tests/normalize.spec.ts | 31 +++++++++++++++++++ 10 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 examples/acp-agent/tests/snapshots/bash-spill/input.json create mode 100644 examples/acp-agent/tests/snapshots/bash-spill/session.jsonl create mode 100644 examples/acp-agent/tests/snapshots/bash-spill/stdout.golden.jsonl diff --git a/docs/rfc/implemented/architecture/2026-07-06-tool-result-retention-library.md b/docs/rfc/implemented/architecture/2026-07-06-tool-result-retention-library.md index 344b50753a..eec1f9ebd1 100644 --- a/docs/rfc/implemented/architecture/2026-07-06-tool-result-retention-library.md +++ b/docs/rfc/implemented/architecture/2026-07-06-tool-result-retention-library.md @@ -14,7 +14,7 @@ The shared abstraction the tools need is **retention**, not generic collection. The library has two independent retainers: -- `ItemRetainer` handles ordered logical units such as paths, grep matches, or search sources. It supports `head` retention only in v1. +- `ItemRetainer` handles ordered logical units such as paths, grep matches, or search sources. It supports `head` retention only in v1, while keeping the retainer shape open to additional retention strategies later. - `TextRetainer` handles byte-oriented text streams such as bash stdout/stderr or web response bodies. It supports `head`, `tail`, and `headTail` retention while preserving UTF-8 boundaries at `finish()`. Both retainers return a small `PushDecision` after each `push()` so callers can tell whether that unit/chunk was fully retained and whether the accumulated result is now truncated. Omission counts are exact because callers keep feeding every observed item/chunk. diff --git a/examples/acp-agent/README.md b/examples/acp-agent/README.md index a1f6e818ab..bdefd2ab71 100644 --- a/examples/acp-agent/README.md +++ b/examples/acp-agent/README.md @@ -6,7 +6,7 @@ The DeepSeek Harness SDK agent demo exposed as an **Agent Client Protocol (ACP)* pnpm run demo:acp # needs DEEPSEEK_API_KEY (repo-root .env or env) ``` -This example is just a leaf `cordis.yml`: it loads the [`@deepseek-ai/dsh-acp-agent`](../../packages/ui/acp-agent) app (which bundles the [`@deepseek-ai/dsh-agent-core`](../../packages/core/agent-core) spine, JSONL session persistence, and the `@deepseek-ai/dsh-acp` bridge — with **no pre-created agents**, since ACP `session/new` creates them on demand), the swappable DeepSeek, bash, and filesystem backends, the model-facing `read`/`write`/`edit`/`subagent`/`subagent_fork`/`todo_write` tool entries, and the advisory `repeat-tool-guard` loop-hygiene plugin. The app package bakes in the no-stdout-logger cluster, so a leaf has no logger entry to get wrong by default — keeping stdout pure for JSON-RPC. +This example is just a leaf `cordis.yml`: it loads the [`@deepseek-ai/dsh-acp-agent`](../../packages/ui/acp-agent) app (which bundles the [`@deepseek-ai/dsh-agent-core`](../../packages/core/agent-core) spine, JSONL session persistence, and the `@deepseek-ai/dsh-acp` bridge — with **no pre-created agents**, since ACP `session/new` creates them on demand), the swappable DeepSeek, bash, filesystem, and spill backends, the model-facing `read`/`write`/`edit`/`subagent`/`subagent_fork`/`todo_write` tool entries, and the advisory `repeat-tool-guard` loop-hygiene plugin. The app package bakes in the no-stdout-logger cluster, so a leaf has no logger entry to get wrong by default — keeping stdout pure for JSON-RPC. ## stdout is the protocol diff --git a/examples/acp-agent/composition.md b/examples/acp-agent/composition.md index 7fd5e25875..b1a3b736ef 100644 --- a/examples/acp-agent/composition.md +++ b/examples/acp-agent/composition.md @@ -41,6 +41,10 @@ flowchart LR cfg --> plugin_acp_fs_policy plugin_acp_tool_fs["tool-fs
@deepseek-ai/dsh-tool-fs"] cfg --> plugin_acp_tool_fs + plugin_acp_spill_local["spill-local
@deepseek-ai/dsh-spill-local"] + cfg --> plugin_acp_spill_local + plugin_acp_spill_policy["spill-policy
@deepseek-ai/dsh-spill-policy"] + cfg --> plugin_acp_spill_policy plugin_acp_hooks_claude["hooks-claude
@deepseek-ai/dsh-hooks-claude"] cfg --> plugin_acp_hooks_claude plugin_acp_hooks_codex["hooks-codex
@deepseek-ai/dsh-hooks-codex"] @@ -62,6 +66,8 @@ flowchart LR | `fs-local` | `@deepseek-ai/dsh-fs-local` | | `fs-policy` | `@deepseek-ai/dsh-fs-policy` | | `tool-fs` | `@deepseek-ai/dsh-tool-fs` | +| `spill-local` | `@deepseek-ai/dsh-spill-local` | +| `spill-policy` | `@deepseek-ai/dsh-spill-policy` | | `hooks-claude` | `@deepseek-ai/dsh-hooks-claude` | | `hooks-codex` | `@deepseek-ai/dsh-hooks-codex` | diff --git a/examples/acp-agent/cordis.yml b/examples/acp-agent/cordis.yml index af2da25b86..48760828e2 100644 --- a/examples/acp-agent/cordis.yml +++ b/examples/acp-agent/cordis.yml @@ -109,6 +109,21 @@ - id: tool-fs name: '@deepseek-ai/dsh-tool-fs' +# Tool-output spill stack: a local backend that saves oversized plain-text tool +# results under the session cwd, and the post-execute policy that replaces the +# model-facing result with a bounded preview + read path. Snapshots lower the +# cap so a deterministic bash result exercises this transcript surface without a +# real model call; normal demo runs keep the coding-agent cap. +- id: spill-local + name: '@deepseek-ai/dsh-spill-local' + config: + root: ./.spill + +- id: spill-policy + name: '@deepseek-ai/dsh-spill-policy' + config: + maxInlineBytes: !!js process.env.DSH_SNAPSHOT && 800 || 50000 + # The Claude Code hook bridge. `configPath` is PROCESS-LEVEL: it is read ONCE at # load and the relative `./hooks.json` resolves against the ACP server's launch # cwd, NOT each `session/new.cwd`. So a single `hooks.json` next to where the diff --git a/examples/acp-agent/tests/acp.snapshot.ts b/examples/acp-agent/tests/acp.snapshot.ts index 647a37e9df..1254281b06 100644 --- a/examples/acp-agent/tests/acp.snapshot.ts +++ b/examples/acp-agent/tests/acp.snapshot.ts @@ -29,6 +29,7 @@ const SCENARIOS: Scenario[] = [ // committed and compared verbatim. { name: 'text-turn', hasModelTurn: true, recorded: true, pinsHeader: true }, { name: 'tool-call-turn', hasModelTurn: true, recorded: true }, + { name: 'bash-spill', hasModelTurn: true, recorded: false }, { name: 'fs-terminal-card', hasModelTurn: true, recorded: true }, { name: 'todo-plan', hasModelTurn: true, recorded: true }, { name: 'workspace-edit', hasModelTurn: true, recorded: true }, diff --git a/examples/acp-agent/tests/snapshots/bash-spill/input.json b/examples/acp-agent/tests/snapshots/bash-spill/input.json new file mode 100644 index 0000000000..de9b769cf5 --- /dev/null +++ b/examples/acp-agent/tests/snapshots/bash-spill/input.json @@ -0,0 +1,7 @@ +{ + "steps": [ + { "op": "initialize" }, + { "op": "newSession" }, + { "op": "prompt", "text": "Use the bash tool to print a large deterministic output, then reply DONE." } + ] +} diff --git a/examples/acp-agent/tests/snapshots/bash-spill/session.jsonl b/examples/acp-agent/tests/snapshots/bash-spill/session.jsonl new file mode 100644 index 0000000000..7c7fe12630 --- /dev/null +++ b/examples/acp-agent/tests/snapshots/bash-spill/session.jsonl @@ -0,0 +1,23 @@ +{"type":"session","version":0,"id":"{{sessionId}}","createdAt":0,"cwd":"{{cwd}}"} +{"type":"turn/start","seq":0,"time":0,"data":{"turn":1,"trigger":{"kind":"message","source":{"kind":"user"}}}} +{"type":"user/message","seq":1,"time":0,"data":{"content":[{"type":"text","text":"Use the bash tool to print a large deterministic output, then reply DONE."}],"source":{"kind":"user"}},"surfaceOp":"append"} +{"type":"step/start","seq":2,"time":0,"data":{"turn":1,"step":1}} +{"type":"request/header","seq":3,"time":0,"data":{"header":{"config":{"model":"deepseek-v4-flash"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}} +{"type":"assistant/chunk","seq":4,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"tool-call"}}} +{"type":"assistant/chunk","seq":5,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":0,"id":"call_spill","name":"bash","argumentsDelta":"{\"command\":\"node -e \\\"process.stdout.write('SPILL_START-' + 'x'.repeat(2000) + '-SPILL_END')\\\"\",\"description\":\"Print large deterministic output\"}"}}} +{"type":"assistant/chunk","seq":6,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":0,"block":{"type":"tool-call","id":"call_spill","name":"bash","arguments":"{\"command\":\"node -e \\\"process.stdout.write('SPILL_START-' + 'x'.repeat(2000) + '-SPILL_END')\\\"\",\"description\":\"Print large deterministic output\"}"}}}} +{"type":"assistant/chunk","seq":7,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"usage","usage":{"inputTokens":10,"outputTokens":5}}}} +{"type":"assistant/chunk","seq":8,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}} +{"type":"assistant/message","seq":9,"time":0,"data":{"turn":1,"step":1,"content":[{"type":"tool-call","id":"call_spill","name":"bash","arguments":"{\"command\":\"node -e \\\"process.stdout.write('SPILL_START-' + 'x'.repeat(2000) + '-SPILL_END')\\\"\",\"description\":\"Print large deterministic output\"}"}],"usage":{"inputTokens":10,"outputTokens":5}},"sourceEventSeqs":[4,5,6,7,8],"surfaceOp":"append"} +{"type":"tool/call","seq":10,"time":0,"data":{"turn":1,"step":1,"callId":"call_spill","name":"bash","arguments":"{\"command\":\"node -e \\\"process.stdout.write('SPILL_START-' + 'x'.repeat(2000) + '-SPILL_END')\\\"\",\"description\":\"Print large deterministic output\"}"}} +{"type":"tool/result","seq":11,"time":0,"data":{"turn":1,"step":1,"callId":"call_spill","content":[{"type":"text","text":"SPILL_START-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-SPILL_END\n\n(Omitted 1447 bytes. Full formatted result saved to: {{spillPath:bash.txt}}. Use read with offset/limit to inspect it.)"}],"isError":false},"sourceEventSeqs":[10],"surfaceOp":"append"} +{"type":"step/end","seq":12,"time":0,"data":{"turn":1,"step":1}} +{"type":"step/start","seq":13,"time":0,"data":{"turn":1,"step":2}} +{"type":"assistant/chunk","seq":14,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"block-start","index":0,"blockType":"text"}}} +{"type":"assistant/chunk","seq":15,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"text-delta","index":0,"text":"DONE"}}} +{"type":"assistant/chunk","seq":16,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"block-end","index":0,"block":{"type":"text","text":"DONE"}}}} +{"type":"assistant/chunk","seq":17,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"usage","usage":{"inputTokens":10,"outputTokens":2}}}} +{"type":"assistant/chunk","seq":18,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"finish","reason":{"kind":"stop"}}}} +{"type":"assistant/message","seq":19,"time":0,"data":{"turn":1,"step":2,"content":[{"type":"text","text":"DONE"}],"usage":{"inputTokens":10,"outputTokens":2}},"sourceEventSeqs":[14,15,16,17,18],"surfaceOp":"append"} +{"type":"step/end","seq":20,"time":0,"data":{"turn":1,"step":2}} +{"type":"turn/end","seq":21,"time":0,"data":{"turn":1,"reason":{"kind":"completed"}}} diff --git a/examples/acp-agent/tests/snapshots/bash-spill/stdout.golden.jsonl b/examples/acp-agent/tests/snapshots/bash-spill/stdout.golden.jsonl new file mode 100644 index 0000000000..dfd9c4fb2b --- /dev/null +++ b/examples/acp-agent/tests/snapshots/bash-spill/stdout.golden.jsonl @@ -0,0 +1,6 @@ +{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentInfo":{"name":"deepseek-harness-acp","version":"0.0.1"},"agentCapabilities":{"loadSession":true,"promptCapabilities":{"image":false,"audio":false,"embeddedContext":false}},"authMethods":[]}} +{"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}"}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call","toolCallId":"call_spill","title":"node -e \"process.stdout.write('SPILL_START-' + 'x'.repeat(2000) + '-SPILL_END')\"","kind":"execute","status":"in_progress","rawInput":"node -e \"process.stdout.write('SPILL_START-' + 'x'.repeat(2000) + '-SPILL_END')\"","content":[{"type":"content","content":{"type":"text","text":"Print large deterministic output"}}]}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call_update","toolCallId":"call_spill","status":"completed","content":[{"type":"content","content":{"type":"text","text":"```console\nSPILL_START-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-SPILL_END\n\n(Omitted 1447 bytes. Full formatted result saved to: {{spillPath:bash.txt}}. Use read with offset/limit to inspect it.)\n```"}}]}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"DONE"}}}} +{"jsonrpc":"2.0","id":3,"result":{"stopReason":"end_turn"}} diff --git a/packages/support/acp-snapshot/src/normalize.ts b/packages/support/acp-snapshot/src/normalize.ts index 2cbe914b42..a4ce886f8b 100644 --- a/packages/support/acp-snapshot/src/normalize.ts +++ b/packages/support/acp-snapshot/src/normalize.ts @@ -33,6 +33,11 @@ const TOOLS = '{{tools}}' /** A UUID v4 string, the shape `randomUUID()` produces for session ids. */ const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi +const LOCAL_SPILL_PATH_RE = new RegExp( + String.raw`\{\{cwd\}\}/\.spill/session-[0-9a-f]{12}/[0-9a-f]{12}-([A-Za-z0-9._~-]+?)` + + String.raw`(?=\. Use read with offset/limit|[\s)]|$)`, + 'g', +) /** Inputs the normalizers need to recognize a run's volatile values. */ export interface NormalizeContext { @@ -48,6 +53,8 @@ function scrubString(value: string, ctx: NormalizeContext): string { // cwd first (longest, most specific), then explicit session ids, then any // residual UUID (covers ids that appear in places we didn't enumerate). out = out.split(ctx.cwd).join(CWD) + out = out.split(`/private${CWD}`).join(CWD) + out = out.replace(LOCAL_SPILL_PATH_RE, (_match, name: string) => `{{spillPath:${name}}}`) for (const id of ctx.sessionIds) out = out.split(id).join(SESSION_ID) out = out.replace(UUID_RE, SESSION_ID) return out diff --git a/packages/support/acp-snapshot/tests/normalize.spec.ts b/packages/support/acp-snapshot/tests/normalize.spec.ts index 8ebd1412b9..a33436a164 100644 --- a/packages/support/acp-snapshot/tests/normalize.spec.ts +++ b/packages/support/acp-snapshot/tests/normalize.spec.ts @@ -86,6 +86,37 @@ describe('normalizeSessionLog', () => { expect(out).not.toContain(ctx.cwd) }) + it('scrubs random local spill paths under the snapshot cwd', () => { + const ev = JSON.stringify({ + type: 'tool/result', seq: 2, time: 5, + data: { + content: [{ + type: 'text', + text: `Full formatted result saved to: ${ctx.cwd}/.spill/session-c22bc3f1d2af/8a7b6c5d4e3f-bash.txt. Use read with offset/limit to inspect it.`, + }], + }, + }) + const out = normalizeSessionLog(`${header({ cwd: ctx.cwd })}\n${ev}\n`, ctx) + expect(out).toContain('{{spillPath:bash.txt}}') + expect(out).not.toContain('session-c22bc3f1d2af') + expect(out).not.toContain('8a7b6c5d4e3f') + }) + + it('scrubs macOS /private aliases for local spill paths', () => { + const ev = JSON.stringify({ + type: 'tool/result', seq: 2, time: 5, + data: { + content: [{ + type: 'text', + text: `Full formatted result saved to: /private${ctx.cwd}/.spill/session-c22bc3f1d2af/8a7b6c5d4e3f-bash.txt. Use read with offset/limit to inspect it.`, + }], + }, + }) + const out = normalizeSessionLog(`${header({ cwd: ctx.cwd })}\n${ev}\n`, ctx) + expect(out).toContain('{{spillPath:bash.txt}}') + expect(out).not.toContain('/private{{spillPath') + }) + it('scrubs the session id in the header', () => { const out = normalizeSessionLog(`${header({ id: ctx.sessionIds[0] })}\n`, ctx) expect(out).toContain('{{sessionId}}') From 3bb90bd4b6d3aa2bdae23509136f585e56912bb8 Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Fri, 10 Jul 2026 11:53:02 +0800 Subject: [PATCH 20/24] fix: make search raw output recovery backend-neutral --- docs/core-data-structures/bash.md | 18 ++++++- ...6-07-09-bash-backed-grep-glob-discovery.md | 16 +++--- packages/bash/bash-local/README.md | 2 +- packages/bash/bash-local/src/index.ts | 9 +++- packages/bash/bash-local/src/run.ts | 10 ++-- .../bash/bash-local/tests/executor.spec.ts | 17 ++++++ packages/bash/bash-local/tests/run.spec.ts | 27 +++++++--- packages/bash/bash/README.md | 2 +- packages/bash/bash/src/types.ts | 12 +++++ packages/bash/bash/tests/service.spec.ts | 1 + packages/bash/tool-bash/README.md | 2 +- packages/bash/tool-bash/tests/tools.spec.ts | 23 +++++--- packages/fs/tool-fs-search/README.md | 4 +- packages/fs/tool-fs-search/src/search-core.ts | 52 ++++++------------- .../fs/tool-fs-search/tests/tools.spec.ts | 47 +++++------------ .../hooks/hook-protocol/tests/runner.spec.ts | 1 + 16 files changed, 136 insertions(+), 107 deletions(-) diff --git a/docs/core-data-structures/bash.md b/docs/core-data-structures/bash.md index 273ba5ebe8..81ddcd06b7 100644 --- a/docs/core-data-structures/bash.md +++ b/docs/core-data-structures/bash.md @@ -6,7 +6,7 @@ Source: [`packages/bash/bash/src/types.ts`](../../packages/bash/bash/src/types.t ## Request vs. spec: the `resolve()` split -The seam separates the **model-/plugin-facing request** (optional `workdir`/`timeoutMs`, filled from config) from the **fully-resolved spec** the executor acts on (those fields required). The tool layer calls `ctx.bash.resolve(request)` between them — this is the repo's "explicit > implicit at package seams" rule made concrete: the reader of a `BashExecSpec` never wonders where the working directory came from. +The seam separates the **model-/plugin-facing request** (optional `workdir`/`timeoutMs`/`stdoutMaxBytes`, filled from config or request policy) from the **fully-resolved spec** the executor acts on (those fields required). The tool layer calls `ctx.bash.resolve(request)` between them — this is the repo's "explicit > implicit at package seams" rule made concrete: the reader of a `BashExecSpec` never wonders where the working directory or output budget came from. ```ts type-equiv interface BashExecRequest { @@ -15,6 +15,13 @@ interface BashExecRequest { workdir?: string | undefined /** Timeout override in milliseconds (implementations cap it). */ timeoutMs?: number | undefined + /** + * Foreground stdout capture budget in bytes. Absent uses the executor's + * default output cap. Trusted in-process consumers use this when they must + * parse complete stdout up to their own bounded limit; the model-facing bash + * tool does not expose it as a parameter. + */ + stdoutMaxBytes?: number | undefined /** Abort signal — implementations kill the command when it fires. */ signal?: AbortSignal | undefined /** @@ -52,6 +59,11 @@ interface BashExecSpec { command: string workdir: string timeoutMs: number + /** + * Resolved foreground stdout capture budget in bytes. `run()` uses it for + * stdout; background tasks and stderr keep the executor's own output cap. + */ + stdoutMaxBytes: number /** Abort signal — implementations kill the command when it fires. */ signal?: AbortSignal | undefined /** @@ -84,7 +96,9 @@ interface BashExecSpec { The `owner` token is the isolation key: the executor stores it but never interprets it (access policy is the consumer's job), so a background task started by one agent isn't readable cross-session. A required-but-nullable field makes a forgotten owner a visible `undefined` rather than a silently-unowned task. -`stdin` and `env` are set by in-process plugins (the hooks bridges, native plugins) to feed a hook command its JSON payload on stdin and its `CLAUDE_PROJECT_DIR`/`CLAUDE_PLUGIN_ROOT` env. The model-facing `dsh-tool-bash` tool does not expose them as parameters — its request is built from `command`/`workdir`/`timeoutMs`/`signal`/`owner` only — because a model already has equivalent power through shell syntax (`FOO=bar cmd`, a heredoc), so duplicating them as tool params would be redundant. This is NOT a security boundary: the credential scrub in `dsh-bash-local` is what stops the harness's ambient secrets reaching a spawned command, and it works regardless of these fields (a model cannot read a value the scrub removed, and tool-call args are static JSON, never shell-evaluated). A guard test asserts the tool doesn't forward model `env`/`stdin` — to catch a future `...args` spread, not to defend a trust wall. `env` is merged AFTER the scrub so an explicit caller entry (a value it already holds) wins even on a credential-shaped name. See [the bash-stdin-env RFC](../rfc/implemented/architecture/2026-06-30-bash-stdin-env-trusted-plugin-surface.md). +`stdin` and `env` are set by in-process plugins (the hooks bridges, native plugins) to feed a hook command its JSON payload on stdin and its `CLAUDE_PROJECT_DIR`/`CLAUDE_PLUGIN_ROOT` env. The model-facing `dsh-tool-bash` tool does not expose them as parameters — its request is built from `command`/`workdir`/`timeoutMs`/`signal`/`owner` only — because a model already has equivalent power through shell syntax (`FOO=bar cmd`, a heredoc), so duplicating them as tool params would be redundant. This is NOT a security boundary: the credential scrub in `dsh-bash-local` is what stops the harness's ambient secrets reaching a spawned command, and it works regardless of these fields (a model cannot read a value the scrub removed, and tool-call args are static JSON, never shell-evaluated). A guard test asserts the tool doesn't forward model `env`/`stdin`/`stdoutMaxBytes` — to catch a future `...args` spread, not to defend a trust wall. `env` is merged AFTER the scrub so an explicit caller entry (a value it already holds) wins even on a credential-shaped name. See [the bash-stdin-env RFC](../rfc/implemented/architecture/2026-06-30-bash-stdin-env-trusted-plugin-surface.md). + +`stdoutMaxBytes` is also trusted-plugin-only. It lets a foreground consumer ask the executor to retain complete stdout up to a bounded parser budget without changing stderr, background tasks, or the model-facing bash tool's ordinary `maxOutputBytes` behavior. Both ids the seam handles are [branded](core.md) (zero-cost `string` brands, the same machinery as `SessionId`/`AgentId`): `BashTaskId` (a tracked background task, generated `bash-N` by the local executor) and `OwnerToken` (the opaque isolation key). `OwnerToken` is deliberately a DISTINCT brand from `SessionId`, not an alias: the bash seam is a capability seam that must not know what an owner token *means*, so it never imports `dsh-session`'s vocabulary — the `dsh-tool-bash` consumer is the single boundary that casts the owning agent's `SessionId` into an `OwnerToken`. Branding both stops a raw `string` (or a `BashTaskId` where an `OwnerToken` is expected, or vice versa) from slipping through the type checker on the model-facing `task_id` path. diff --git a/docs/rfc/implemented/feature/2026-07-09-bash-backed-grep-glob-discovery.md b/docs/rfc/implemented/feature/2026-07-09-bash-backed-grep-glob-discovery.md index 185444872f..74648ab40d 100644 --- a/docs/rfc/implemented/feature/2026-07-09-bash-backed-grep-glob-discovery.md +++ b/docs/rfc/implemented/feature/2026-07-09-bash-backed-grep-glob-discovery.md @@ -77,7 +77,7 @@ The `path` field follows the same split as Claude Code: `grep.path` is a file-or `grep` builds a fixed line-oriented `rg --json` command against the supplied file/directory target (`path` when supplied, else the bash workdir) so file path, line number, and line text are parsed without colon-splitting ambiguity. It consumes `match` records, treats malformed JSON or malformed match records as `SEARCH_FAILED`, maps result paths relative to the bash workdir when possible, applies per-line preview retention with `grepMaxLineBytes`, pushes each match into `ItemRetainer({ kind: 'head', maxItems: grepMaxMatches })`, then groups only the retained preview matches by file for inline output. The spill file stores the full formatted match list, not only the omitted tail, so `read offset/limit` works against the same logical result the model saw. -Raw `rg` stdout is an internal transport detail. If `ctx.bash.run()` returns untruncated stdout, the tool parses `stdout.text`. If stdout is truncated and `stdout.spillPath` is present, the tool reads that local raw spill file up to `rawOutputMaxBytes + 1` bytes and parses it only when the complete file fits within `rawOutputMaxBytes`. If the raw spill file is larger than `rawOutputMaxBytes`, or stdout is truncated without a spill path, the tool fails with a clear search error telling the model to narrow `pattern`, `path`, or `include`. The tool never exposes raw `rg` output or bash raw spill paths to the model. +Raw `rg` stdout is an internal transport detail. The tool requests `stdoutMaxBytes: rawOutputMaxBytes` through `ctx.bash.resolve()` and parses `stdout.text` only when the executor returns untruncated stdout within that cap. If stdout is larger than `rawOutputMaxBytes`, or the executor still returns `stdout.truncated`, the tool fails with a clear search error telling the model to narrow `pattern`, `path`, or `include`. The tool never exposes raw `rg` output or bash raw spill paths to the model. Only stdout is a parse source. Stderr is diagnostic text for invalid patterns, missing `rg`, and search failures; if bash truncates stderr, the tool uses the retained stderr tail with a truncation note and does not read `stderr.spillPath`. @@ -93,7 +93,7 @@ When a search produces more logical results than the inline cap and `ctx.spillFi When spill storage is absent, the call has no session owner, or saving fails, the tool still returns the inline page and a footer explaining that the complete result could not be saved. Search success must not turn into an `isError` result solely because formatted-result spill storage is unavailable. -The bash raw spill file and the formatted search spill file are different artifacts. The raw bash spill file is a local executor implementation detail used only so the search tool can parse complete `rg` stdout. The formatted spill file is the stable model-facing recovery path produced by `ctx.spillFiles.saveText()`. +The bash raw output stream and the formatted search spill file are different artifacts. Raw `rg` stdout is parsed only in memory within the requested bash stdout cap; the formatted spill file is the stable model-facing recovery path produced by `ctx.spillFiles.saveText()`. ### Result shape @@ -122,13 +122,13 @@ If the complete logical result fits under the inline cap, no formatted spill fil **Put `glob` / `grep` on `ctx.fs`.** Rejected for v1: it forces every filesystem backend to grow a search API and makes local ripgrep behavior part of the provider seam. Search is useful product behavior, but it is not a universal text-storage primitive like `readText` or `writeText`. -**Directly spawn ripgrep from `dsh-fs-local`.** Rejected for this RFC's v1: direct spawn gives the cleanest argv boundary, stdout/stderr control, and early-stop control, but it duplicates process execution concerns that the bash seam already owns: environment scrubbing, process-group kill, timeout propagation, sandbox/remote executor substitution, and raw output spill. It remains a reasonable optimization if bash-backed search proves too shell-string-sensitive or if raw bash spill recovery is not portable enough. +**Directly spawn ripgrep from `dsh-fs-local`.** Rejected for this RFC's v1: direct spawn gives the cleanest argv boundary, stdout/stderr control, and early-stop control, but it duplicates process execution concerns that the bash seam already owns: environment scrubbing, process-group kill, timeout propagation, sandbox/remote executor substitution, and bounded output capture. It remains a reasonable optimization if bash-backed search proves too shell-string-sensitive or if foreground streaming becomes necessary. **Use `ctx.bash.start()` for streaming early stop.** Rejected: `start()` creates model-visible background task semantics: task ids, owner tokens, `bash_output`, `bash_kill`, completion notifications, and no built-in timeout. `grep` needs a foreground tool result, not a background bash workflow. If streaming search becomes necessary, the right abstraction is a foreground streaming process handle on the bash/process seam, not borrowing the public background-task API. -**Expose bash raw spill paths to the model.** Rejected: a bash raw spill path contains raw `rg` stdout (`rg --json` records for grep), not the stable formatted search result. The search tool may read raw spill internally, but model recovery uses a formatted result saved through `ctx.spillFiles.saveText()`. +**Expose bash raw spill paths to the model.** Rejected: a bash raw spill path contains raw `rg` stdout (`rg --json` records for grep), not the stable formatted search result. Search parses raw stdout only as an internal transport; model recovery uses a formatted result saved through `ctx.spillFiles.saveText()`. -**Add `spillFiles.saveFile()` for bash output normalization first.** Rejected for this RFC's v1: `saveFile()` would help a future bash normalization pass move existing executor spill files into session-scoped spill storage, but search still has to parse raw `rg` output before producing the model-facing artifact. `saveText()` is sufficient for the formatted search result, and raw bash spill remains an executor-local recovery detail. +**Add `spillFiles.saveFile()` for bash output normalization first.** Rejected for this RFC's v1: `saveFile()` would help a future bash normalization pass move existing executor spill files into session-scoped spill storage, but search only needs bounded in-memory raw `rg` stdout before producing the model-facing artifact. `saveText()` is sufficient for the formatted search result. **Rely on the generic `dsh-spill-policy`.** Rejected: generic post-execute spill sees only the final tool result. If `grep` / `glob` return the first page inline, the generic policy cannot recover omitted results. The search tools must save the complete formatted result themselves before returning the bounded model-facing text. @@ -136,7 +136,7 @@ If the complete logical result fits under the inline cap, no formatted spill fil **Keep early-stop search and skip formatted spill files.** Rejected for this proposal: early stop is more efficient but gives the model no path to inspect later results. The chosen v1 optimizes result recoverability and implementation simplicity, with `timeoutMs`, `rawOutputMaxBytes`, bash backend caps, and formatted spill files as safety backstops. -**Expand the bash seam with a raw-output reader first.** Deferred: a remote bash backend may eventually need a portable `readRawOutput(ref, maxBytes)` style API instead of local `spillPath` reads. v1 uses the existing local-readable `stdout.spillPath` to avoid widening the bash seam for one consumer. +**Expand the bash seam with a raw-output reader first.** Rejected: a portable `readRawOutput(ref, maxBytes)` API would add reference lifetime, permission, and backend storage semantics. A per-run `stdoutMaxBytes` request is the narrower seam: search either receives complete stdout within `rawOutputMaxBytes` or fails clearly. ## Testing @@ -151,7 +151,7 @@ If the complete logical result fits under the inline cap, no formatted spill fil - `glob` and `grep` are model-facing tools in `@deepseek-ai/dsh-tool-fs-search`, not `ctx.fs` provider methods and not part of the existing `@deepseek-ai/dsh-tool-fs` root plugin. The package injects `tools`, `systemPrompt`, and `bash`; it does not inject `fs`, and `ctx.spillFiles` stays optional via `ctx.get('spillFiles')`. - The schemas are exactly `glob(pattern, path?)` and `grep(pattern, path?, include?)`; search caps and timeout are defaulted, validated Config fields (`globMaxResults`, `grepMaxMatches`, `grepMaxLineBytes`, `rawOutputMaxBytes`, `timeoutMs`). - The tools execute through `ctx.bash.resolve(request)` → `ctx.bash.run(spec)`, forward `exec.signal`, never call `ctx.bash.start()`, and never expose a bash task id. The bash request workdir comes from `exec.agent?.session.header.cwd` when available; the resolved `spec.workdir` drives execution and relative-path display. -- When bash stdout is truncated, the tools parse the full raw stdout only through a local `stdout.spillPath` that fits within `rawOutputMaxBytes`; missing spill paths or over-cap raw output are clear search failures, and raw `rg` output is never exposed to the model. +- The tools request `stdoutMaxBytes: rawOutputMaxBytes` from the bash seam, parse only untruncated stdout within that cap, and treat over-cap or still-truncated raw output as a clear search failure; raw `rg` output is never exposed to the model. - Oversized complete formatted results are saved through `ctx.spillFiles.saveText()` when available while inline results stay bounded; spill failure, a missing backend, or a missing owner preserves the inline result and reports the unsaved remainder — never an `isError`. - The package README, the generated config catalog, and exported JSDoc document the Config fields and `SEARCH_*` codes; the coding-agent example ships the tools (the acp-agent tree waits on the snapshot re-record above); the fs group README records the co-located bash/filesystem deployment requirement. @@ -163,6 +163,4 @@ Shell command construction is the sharpest safety edge. Because `ctx.bash` accep The v1 assumes a co-located bash/filesystem deployment. If bash searches one workspace and the `read` tool resolves paths against another, returned paths may not be follow-up-readable. The package documents this requirement but does not verify it at runtime. -Raw bash spill recovery is local-path-shaped in v1. A remote or sandboxed bash backend may return no readable `spillPath` or may require a future raw-output read API. In that case broad searches fail clearly instead of pretending a truncated raw result is complete. - Spill paths are local filesystem paths in v1. The formatted-result design works for local deployments where `read` can open spill files; remote or workspace-confined deployments need either an allowlist for spill paths or a future virtual spill URI bridge. diff --git a/packages/bash/bash-local/README.md b/packages/bash/bash-local/README.md index dec29ce93b..bdf4ec5f7f 100644 --- a/packages/bash/bash-local/README.md +++ b/packages/bash/bash-local/README.md @@ -21,7 +21,7 @@ Design surveyed against the bash tools of Claude Code, OpenCode, Codex, and pi; - **Spawn per call, no shell state** — every call is a fresh non-login `bash -c` (deterministic; no rc files). All four surveyed tools spawn per call. `XXX(stateful-shell)` in `src/run.ts` records the two proven stateful designs (Claude Code's cwd-only persistence; Codex's PTY exec sessions) for when real workflows demand them. - **Process-group kills with escalation** — children are spawned `detached` (own process group); kills send SIGTERM to the group, then SIGKILL after the `graceMs` grace (default 3s — OpenCode's escalation; pipelines and subshells die with the parent). ESRCH is tolerated; daemons that re-parent away from the group can still survive — same caveat as the surveyed tools. -- **Tail-keep truncation + spill files** — output beyond `maxOutputBytes` keeps the in-memory TAIL (errors/results cluster at the end — pi/OpenCode rationale) while the FULL stream is appended to a temp file whose path is reported when available. If the final spill close reports a delayed writeback failure, the executor still returns the tail but withholds the path rather than advertising a possibly incomplete file. +- **Tail-keep truncation + spill files** — output beyond `maxOutputBytes` keeps the in-memory TAIL (errors/results cluster at the end — pi/OpenCode rationale) while the FULL stream is appended to a temp file whose path is reported when available. A foreground `BashExecRequest.stdoutMaxBytes` can raise stdout's capture budget for one trusted caller; stderr and background tasks still use `maxOutputBytes`. If the final spill close reports a delayed writeback failure, the executor still returns the tail but withholds the path rather than advertising a possibly incomplete file. - **Model-friendly env + credential scrub** — `process.env` minus credential-shaped vars (`*KEY*`/`*SECRET*`/`*TOKEN*`), then `NO_COLOR=1 TERM=dumb PAGER=cat GIT_PAGER=cat` (Codex's hardcoded set) so pagers and ANSI color don't garble results. This scrub is the security control that keeps the harness's *ambient* credentials out of a spawned command. A spec's `env` is merged LAST (after the scrub), so a caller's explicit entry — a value it already holds — wins even on a credential-shaped name. The spec's `stdin`, when supplied, is written to the child and closed; with none supplied, fd 0 is `/dev/null` — the exact pre-seam default, so a command that probes stdin's file type is unaffected. Both `env`/`stdin` are set by in-process plugins (the hooks bridges); the model-facing tool doesn't expose them. See [the bash-stdin-env RFC](../../../docs/rfc/implemented/architecture/2026-06-30-bash-stdin-env-trusted-plugin-surface.md). - **Background tasks** — `start()` returns immediately, no timeout applies (Claude Code detaches timeouts when backgrounding), `readOutput()` is incremental with whole-stream byte offsets, and disposal kills everything. The spec's opaque `owner` token is stored on the tracked task and returned by `ownerOf(id)` — the executor never interprets it (the consumer's access policy does), and because it lives with the task here it survives a `tool-bash` HMR reload. diff --git a/packages/bash/bash-local/src/index.ts b/packages/bash/bash-local/src/index.ts index 3e09d7e35b..e8efbabc71 100644 --- a/packages/bash/bash-local/src/index.ts +++ b/packages/bash/bash-local/src/index.ts @@ -121,10 +121,13 @@ export class LocalBashExecutor extends BashExecutor { this.config.maxTimeoutMs, 'bash-local: request.timeoutMs', ) + const stdoutMaxBytes = request.stdoutMaxBytes ?? this.config.maxOutputBytes + assertPositiveFinite('request.stdoutMaxBytes', stdoutMaxBytes) return { command: request.command, workdir: request.workdir ?? this.config.cwd ?? process.cwd(), timeoutMs, + stdoutMaxBytes, ...request.signal ? { signal: request.signal } : {}, // Carry stdin/env through verbatim — optional, no config default (absent // means none). env merges AFTER the scrub in run.ts. @@ -144,7 +147,8 @@ export class LocalBashExecutor extends BashExecutor { const outcome = await runBash({ command: spec.command, cwd: spec.workdir, - maxOutputBytes: this.config.maxOutputBytes, + stdoutMaxBytes: spec.stdoutMaxBytes, + stderrMaxBytes: this.config.maxOutputBytes, graceMs: this.config.graceMs, signal: d.signal, stdin: spec.stdin, @@ -170,7 +174,8 @@ export class LocalBashExecutor extends BashExecutor { const running = runBash({ command: spec.command, cwd: spec.workdir, - maxOutputBytes: this.config.maxOutputBytes, + stdoutMaxBytes: this.config.maxOutputBytes, + stderrMaxBytes: this.config.maxOutputBytes, graceMs: this.config.graceMs, signal: spec.signal, stdin: spec.stdin, diff --git a/packages/bash/bash-local/src/run.ts b/packages/bash/bash-local/src/run.ts index bc4a017dea..bc29da60f5 100644 --- a/packages/bash/bash-local/src/run.ts +++ b/packages/bash/bash-local/src/run.ts @@ -77,8 +77,10 @@ export function childEnv(extra?: Record): NodeJS.ProcessEnv { export interface SpawnSpec { command: string cwd: string - /** Per-stream in-memory cap; overflow spills to disk (tail kept in memory). */ - maxOutputBytes: number + /** Stdout in-memory cap; overflow spills to disk (tail kept in memory). */ + stdoutMaxBytes: number + /** Stderr in-memory cap; overflow spills to disk (tail kept in memory). */ + stderrMaxBytes: number /** Grace period between the SIGTERM and the SIGKILL escalation on a kill. */ graceMs: number /** @@ -351,8 +353,8 @@ export function runBash(spec: SpawnSpec, internals: RunInternals = {}): RunningB ? spawn('bash', ['-c', spec.command], { cwd: spec.cwd, env, stdio: ['pipe', 'pipe', 'pipe'], detached: true }) : spawn('bash', ['-c', spec.command], { cwd: spec.cwd, env, stdio: ['ignore', 'pipe', 'pipe'], detached: true }) - const stdout = new OutputCollector(spec.maxOutputBytes, 'stdout', spillDir) - const stderr = new OutputCollector(spec.maxOutputBytes, 'stderr', spillDir) + const stdout = new OutputCollector(spec.stdoutMaxBytes, 'stdout', spillDir) + const stderr = new OutputCollector(spec.stderrMaxBytes, 'stderr', spillDir) child.stdout.on('data', (chunk: Buffer) => { stdout.push(chunk) }) child.stderr.on('data', (chunk: Buffer) => { stderr.push(chunk) }) diff --git a/packages/bash/bash-local/tests/executor.spec.ts b/packages/bash/bash-local/tests/executor.spec.ts index 98095b8d47..3d845a88c3 100644 --- a/packages/bash/bash-local/tests/executor.spec.ts +++ b/packages/bash/bash-local/tests/executor.spec.ts @@ -86,6 +86,23 @@ describe('LocalBashExecutor.run', () => { const { bash } = await setup() expect(() => bash.resolve({ command: 'true', timeoutMs: Number.NaN })).toThrow(/request\.timeoutMs/) expect(() => bash.resolve({ command: 'true', timeoutMs: -1 })).toThrow(/request\.timeoutMs/) + expect(() => bash.resolve({ command: 'true', stdoutMaxBytes: Number.NaN })).toThrow(/request\.stdoutMaxBytes/) + expect(() => bash.resolve({ command: 'true', stdoutMaxBytes: -1 })).toThrow(/request\.stdoutMaxBytes/) + }) + + it('defaults stdoutMaxBytes to maxOutputBytes and lets foreground callers raise stdout only', async () => { + const { bash } = await setup({ maxOutputBytes: 100 }) + expect(bash.resolve({ command: 'true' }).stdoutMaxBytes).toBe(100) + + const result = await bash.run(bash.resolve({ + command: 'printf "%.0sx" $(seq 1 500); printf "%.0se" $(seq 1 500) >&2', + stdoutMaxBytes: 500, + })) + + expect(result.stdout.truncated).toBe(false) + expect(result.stdout.text).toBe('x'.repeat(500)) + expect(result.stderr.truncated).toBe(true) + expect(result.stderr.text.length).toBeLessThanOrEqual(100) }) it('kill escalation uses the configured graceMs (a TERM-trapping task dies by SIGKILL)', async () => { diff --git a/packages/bash/bash-local/tests/run.spec.ts b/packages/bash/bash-local/tests/run.spec.ts index 1d6e93afe3..dc4146f9de 100644 --- a/packages/bash/bash-local/tests/run.spec.ts +++ b/packages/bash/bash-local/tests/run.spec.ts @@ -26,7 +26,8 @@ function spec(command: string, overrides: Partial[0]> return { command, cwd: process.cwd(), - maxOutputBytes: 64_000, + stdoutMaxBytes: 64_000, + stderrMaxBytes: 64_000, graceMs: 3_000, ...overrides, } @@ -229,10 +230,24 @@ describe('stdin and extra env (set by in-process plugins)', () => { }) describe('output truncation and spill', () => { + it('applies stdout and stderr caps independently', async () => { + const result = await runBash( + spec('printf "%.0sx" $(seq 1 500); printf "%.0se" $(seq 1 500) >&2', { + stdoutMaxBytes: 500, + stderrMaxBytes: 100, + }), + { spillDir }, + ).done + expect(result.stdout.truncated).toBe(false) + expect(result.stdout.text).toBe('x'.repeat(500)) + expect(result.stderr.truncated).toBe(true) + expect(result.stderr.text.length).toBeLessThanOrEqual(100) + }) + it('keeps the tail and spills the full stream to disk', async () => { // 200 numbered lines of ~10 bytes; cap at 500 bytes keeps a late tail. const result = await runBash( - spec('for i in $(seq 1 200); do printf "line-%04d\\n" $i; done', { maxOutputBytes: 500 }), + spec('for i in $(seq 1 200); do printf "line-%04d\\n" $i; done', { stdoutMaxBytes: 500, stderrMaxBytes: 500 }), { spillDir }, ).done expect(result.stdout.truncated).toBe(true) @@ -247,7 +262,7 @@ describe('output truncation and spill', () => { it('does not truncate output exactly at the cap', async () => { const result = await runBash( - spec('printf "%.0sx" $(seq 1 500)', { maxOutputBytes: 500 }), + spec('printf "%.0sx" $(seq 1 500)', { stdoutMaxBytes: 500, stderrMaxBytes: 500 }), { spillDir }, ).done expect(result.stdout.truncated).toBe(false) @@ -258,7 +273,7 @@ describe('output truncation and spill', () => { it('settles with the tail and no spill path when final spill close fails', async () => { failNextClose.value = true const result = await runBash( - spec('for i in $(seq 1 200); do printf "line-%04d\\n" $i; done', { maxOutputBytes: 500 }), + spec('for i in $(seq 1 200); do printf "line-%04d\\n" $i; done', { stdoutMaxBytes: 500, stderrMaxBytes: 500 }), { spillDir }, ).done expect(failNextClose.value).toBe(false) @@ -377,7 +392,7 @@ describe('review fixes: env scrubbing and spill hardening', () => { it('creates spill files with owner-only permissions and random names', async () => { const result = await runBash( - spec('for i in $(seq 1 200); do printf "line-%04d\\n" $i; done', { maxOutputBytes: 500 }), + spec('for i in $(seq 1 200); do printf "line-%04d\\n" $i; done', { stdoutMaxBytes: 500, stderrMaxBytes: 500 }), { spillDir }, ).done const path = result.stdout.spillPath! @@ -388,7 +403,7 @@ describe('review fixes: env scrubbing and spill hardening', () => { it('defaults spills into a private per-process directory', async () => { const result = await runBash( - spec('for i in $(seq 1 200); do printf "line-%04d\\n" $i; done', { maxOutputBytes: 500 }), + spec('for i in $(seq 1 200); do printf "line-%04d\\n" $i; done', { stdoutMaxBytes: 500, stderrMaxBytes: 500 }), ).done const dir = dirname(result.stdout.spillPath!) expect(dir).toMatch(/dsh-bash-/) diff --git a/packages/bash/bash/README.md b/packages/bash/bash/README.md index 39318ae371..dc46ad78f1 100644 --- a/packages/bash/bash/README.md +++ b/packages/bash/bash/README.md @@ -28,6 +28,6 @@ Implementations subclass `BashExecutor`, implement the abstract methods, and cal ## Vocabulary -`BashExecRequest` (command, workdir?, timeoutMs?, signal?, stdin?, env?, owner?) resolves to `BashExecSpec` (command, workdir, timeoutMs, signal?, stdin?, env?, owner) before execution; `owner` is optional on the request and **required-but-nullable** (`OwnerToken | undefined`) on the resolved spec, so a forgotten owner is a visible `undefined` rather than a silently-absent property. The task id (`BashTaskId`) and the `owner` token (`OwnerToken`) are [branded](../../util/brand) — `OwnerToken` is a DISTINCT brand from `SessionId` (the seam never imports `dsh-session`; the `dsh-tool-bash` consumer is the single boundary that casts its `SessionId` into one). `run()` returns `BashRunResult` (exitCode, signal, timedOut, aborted, timeoutMs, stdout/stderr as `CollectedOutput`) and `start()`/`readOutput()` use `BashTask`/`BashTaskRead` for the background side. See `src/types.ts` for the full contracts. +`BashExecRequest` (command, workdir?, timeoutMs?, stdoutMaxBytes?, signal?, stdin?, env?, owner?) resolves to `BashExecSpec` (command, workdir, timeoutMs, stdoutMaxBytes, signal?, stdin?, env?, owner) before execution; `owner` is optional on the request and **required-but-nullable** (`OwnerToken | undefined`) on the resolved spec, so a forgotten owner is a visible `undefined` rather than a silently-absent property. The task id (`BashTaskId`) and the `owner` token (`OwnerToken`) are [branded](../../util/brand) — `OwnerToken` is a DISTINCT brand from `SessionId` (the seam never imports `dsh-session`; the `dsh-tool-bash` consumer is the single boundary that casts its `SessionId` into one). `stdoutMaxBytes` is a trusted foreground-run capture budget for consumers that must parse complete stdout up to their own limit; the model-facing bash tool does not expose it. `run()` returns `BashRunResult` (exitCode, signal, timedOut, aborted, timeoutMs, stdout/stderr as `CollectedOutput`) and `start()`/`readOutput()` use `BashTask`/`BashTaskRead` for the background side. See `src/types.ts` for the full contracts. `stdin` and `env` are set by in-process plugins (the hooks bridges, native plugins) to feed a hook command its JSON payload on stdin and its `CLAUDE_PROJECT_DIR`/`CLAUDE_PLUGIN_ROOT` env. The model-facing `dsh-tool-bash` tool does not expose them as parameters — a model already has equivalent power through shell syntax (`FOO=bar cmd`, a heredoc), so they would be redundant tool params. This is not a security boundary: the implementation's credential scrub (not these fields) is what keeps the harness's ambient secrets out of a spawned command. They are plain optionals on the resolved spec (unlike `owner`'s required-but-nullable): a missing one means "none", the safe default. See [the bash-stdin-env RFC](../../../docs/rfc/implemented/architecture/2026-06-30-bash-stdin-env-trusted-plugin-surface.md). diff --git a/packages/bash/bash/src/types.ts b/packages/bash/bash/src/types.ts index 7b27231312..11ff7a0124 100644 --- a/packages/bash/bash/src/types.ts +++ b/packages/bash/bash/src/types.ts @@ -52,6 +52,13 @@ export interface BashExecRequest { workdir?: string | undefined /** Timeout override in milliseconds (implementations cap it). */ timeoutMs?: number | undefined + /** + * Foreground stdout capture budget in bytes. Absent uses the executor's + * default output cap. Trusted in-process consumers use this when they must + * parse complete stdout up to their own bounded limit; the model-facing bash + * tool does not expose it as a parameter. + */ + stdoutMaxBytes?: number | undefined /** Abort signal — implementations kill the command when it fires. */ signal?: AbortSignal | undefined /** @@ -95,6 +102,11 @@ export interface BashExecSpec { command: string workdir: string timeoutMs: number + /** + * Resolved foreground stdout capture budget in bytes. `run()` uses it for + * stdout; background tasks and stderr keep the executor's own output cap. + */ + stdoutMaxBytes: number /** Abort signal — implementations kill the command when it fires. */ signal?: AbortSignal | undefined /** diff --git a/packages/bash/bash/tests/service.spec.ts b/packages/bash/bash/tests/service.spec.ts index 81530843ed..38b79031d2 100644 --- a/packages/bash/bash/tests/service.spec.ts +++ b/packages/bash/bash/tests/service.spec.ts @@ -13,6 +13,7 @@ class StubExecutor extends BashExecutor { command: request.command, workdir: request.workdir ?? '/stub', timeoutMs: request.timeoutMs ?? 1000, + stdoutMaxBytes: request.stdoutMaxBytes ?? 64_000, ...request.signal ? { signal: request.signal } : {}, owner: request.owner, } diff --git a/packages/bash/tool-bash/README.md b/packages/bash/tool-bash/README.md index eabb9298aa..d827e7c383 100644 --- a/packages/bash/tool-bash/README.md +++ b/packages/bash/tool-bash/README.md @@ -44,7 +44,7 @@ When a background task finishes, a short notice is injected into the owning agen ## The tool builds its request from named args only -The `BashExecRequest` seam carries optional `stdin` and `env`, used by the hooks bridges to feed a hook command its JSON payload and `CLAUDE_*` env. This tool does **not** expose them as parameters: its request is built from `command`/`workdir`/`timeoutMs`/`signal`/`owner` only, so a model that includes `env` or `stdin` keys in its tool arguments has them ignored. This is not a trust boundary — a model already has equivalent power through shell syntax (`FOO=bar cmd`, a heredoc), and the real defense against leaking the harness's ambient secrets is `dsh-bash-local`'s credential scrub, which works regardless. A regression guard drives the real tool with those extra args and asserts the resulting request carries neither field — its job is to catch a future refactor that blindly spreads `...args` into the request (which would silently forward model input into the post-scrub `env` merge), not to defend a wall. See [the bash-stdin-env RFC](../../../docs/rfc/implemented/architecture/2026-06-30-bash-stdin-env-trusted-plugin-surface.md). +The `BashExecRequest` seam carries optional trusted-plugin fields (`stdoutMaxBytes`, `stdin`, and `env`); hooks use `stdin`/`env` to feed a hook command its JSON payload and `CLAUDE_*` env. This tool does **not** expose them as parameters: its request is built from `command`/`workdir`/`timeoutMs`/`signal`/`owner` only, so a model that includes `env`, `stdin`, or `stdoutMaxBytes` keys in its tool arguments has them ignored. This is not a trust boundary — a model already has equivalent power through shell syntax (`FOO=bar cmd`, a heredoc), and the real defense against leaking the harness's ambient secrets is `dsh-bash-local`'s credential scrub, which works regardless. A regression guard drives the real tool with those extra args and asserts the resulting request carries none of those fields — its job is to catch a future refactor that blindly spreads `...args` into the request (which would silently forward model input into the post-scrub `env` merge), not to defend a wall. See [the bash-stdin-env RFC](../../../docs/rfc/implemented/architecture/2026-06-30-bash-stdin-env-trusted-plugin-surface.md). ## Permissions diff --git a/packages/bash/tool-bash/tests/tools.spec.ts b/packages/bash/tool-bash/tests/tools.spec.ts index 7d4b34f74f..a6f1bac5b1 100644 --- a/packages/bash/tool-bash/tests/tools.spec.ts +++ b/packages/bash/tool-bash/tests/tools.spec.ts @@ -98,6 +98,7 @@ class LossyReadBashExecutor extends BashExecutor { command: request.command, workdir: request.workdir ?? process.cwd(), timeoutMs: request.timeoutMs ?? 0, + stdoutMaxBytes: request.stdoutMaxBytes ?? 64_000, ...request.signal ? { signal: request.signal } : {}, owner: request.owner, } @@ -873,11 +874,12 @@ describe('the model-facing bash tool builds its request from named args only (no /** * Records every {@link BashExecRequest} the consumer hands to `resolve()`, so a * test can assert what the model-facing tool DID and DID NOT forward. The `bash` - * tool does not expose `stdin`/`env` as parameters (bash syntax already gives a - * model that power), so it must build its request from named args only and + * tool does not expose trusted-plugin fields (`stdoutMaxBytes`, `stdin`, or + * `env`) as parameters, so it must build its request from named args only and * never spread unknown tool-call keys into it. This guard's job is to catch a * future refactor that blindly forwards `...args` — which would silently thread - * model input into the post-scrub `env` merge — NOT to defend a trust boundary + * model input into the post-scrub `env` merge or per-run capture budget — NOT + * to defend a trust boundary * (the credential scrub in dsh-bash-local is the security control; see the * bash-stdin-env RFC). Foreground `run()` returns a canned result; `start()` is * unused here. @@ -890,6 +892,7 @@ describe('the model-facing bash tool builds its request from named args only (no command: request.command, workdir: request.workdir ?? process.cwd(), timeoutMs: request.timeoutMs ?? 0, + stdoutMaxBytes: request.stdoutMaxBytes ?? 64_000, ...request.signal ? { signal: request.signal } : {}, ...request.stdin !== undefined ? { stdin: request.stdin } : {}, ...request.env !== undefined ? { env: request.env } : {}, @@ -920,12 +923,12 @@ describe('the model-facing bash tool builds its request from named args only (no return { ctx, bash: ctx.bash as RecordingBashExecutor } } - it('does not forward env/stdin even when the model includes them as extra arguments', async () => { + it('does not forward trusted-only fields even when the model includes them as extra arguments', async () => { const { ctx, bash } = await setupRecording() - // Extra args: the model includes `env` and `stdin` keys hoping they reach the + // Extra args: the model includes trusted-plugin keys hoping they reach the // executor. The bash tool's schema ignores unknown keys, and execute() builds // the request from only command/workdir/timeoutMs/signal — so the recorded - // request carries NEITHER. (Not a security wall — the model could set an env + // request carries NONE. (Not a security wall — the model could set an env // var or feed stdin via shell syntax anyway; this just keeps the request // shape honest so a future `...args` spread can't silently forward input.) await ctx.tools.execute({ @@ -936,6 +939,7 @@ describe('the model-facing bash tool builds its request from named args only (no description: 'echo', env: { SNEAKY_API_KEY: 'leak' }, stdin: 'malicious payload', + stdoutMaxBytes: 999_999, }, }) expect(bash.requests).toHaveLength(1) @@ -943,9 +947,10 @@ describe('the model-facing bash tool builds its request from named args only (no expect(request.command).toBe('echo hi') expect('env' in request).toBe(false) expect('stdin' in request).toBe(false) + expect('stdoutMaxBytes' in request).toBe(false) }) - it('a background bash call likewise carries no env/stdin', async () => { + it('a background bash call likewise carries no trusted-only fields', async () => { const { ctx, bash } = await setupRecording() // start() throws in this recorder, but resolve() runs first and records the // request — which is all this no-forward assertion needs. @@ -958,15 +963,17 @@ describe('the model-facing bash tool builds its request from named args only (no run_in_background: true, env: { TOKEN: 'leak' }, stdin: 'x', + stdoutMaxBytes: 999_999, }, }) expect(bash.requests).toHaveLength(1) const request = bash.requests[0]! expect('env' in request).toBe(false) expect('stdin' in request).toBe(false) + expect('stdoutMaxBytes' in request).toBe(false) // The owner token IS set on a background call (the isolation fence) — proving // the recorder sees the real request the consumer built, so the absent - // env/stdin above is a real negative, not a recorder that drops everything. + // trusted-only fields above are a real negative, not a recorder that drops everything. expect('owner' in request).toBe(true) }) }) diff --git a/packages/fs/tool-fs-search/README.md b/packages/fs/tool-fs-search/README.md index e1ed680060..469830683d 100644 --- a/packages/fs/tool-fs-search/README.md +++ b/packages/fs/tool-fs-search/README.md @@ -39,8 +39,8 @@ Routine budgets stay out of the model-facing schema (no `head_limit`/`offset`/`c ## Two budgets, two artifacts -Raw `rg` stdout is an internal transport detail. When the executor truncates it, the tool recovers the complete stream from the executor's **raw bash spill file** — read locally, capped at `rawOutputMaxBytes`, never shown to the model. The model-facing recovery artifact is different: when a search yields more logical results than the inline cap, the tool saves the COMPLETE formatted result through `ctx.spillFiles.saveText()` (suggested names `glob-results.txt` / `grep-results.txt`, owner = the calling session, source = the tool execution identity) and appends a footer naming the saved path. This is the first tool-owned spill call in the codebase — deliberate, because retention here is item-level: the generic `@deepseek-ai/dsh-spill-policy` only sees the final text on `tools/post-execute`, by which point a capped search has already omitted later paths/matches. A missing spill backend, a call with no session owner, or a `saveText()` failure keeps the inline page and reports that the complete result could not be saved — never an `isError`. +Raw `rg` stdout is an internal transport detail. Each search requests `stdoutMaxBytes: rawOutputMaxBytes` from the bash seam and parses only complete retained stdout; if the executor still returns `stdout.truncated`, the search fails with `SEARCH_RAW_OUTPUT_OVERFLOW` and tells the model to narrow the query. The model-facing recovery artifact is different: when a search yields more logical results than the inline cap, the tool saves the COMPLETE formatted result through `ctx.spillFiles.saveText()` (suggested names `glob-results.txt` / `grep-results.txt`, owner = the calling session, source = the tool execution identity) and appends a footer naming the saved path. This is the first tool-owned spill call in the codebase — deliberate, because retention here is item-level: the generic `@deepseek-ai/dsh-spill-policy` only sees the final text on `tools/post-execute`, by which point a capped search has already omitted later paths/matches. A missing spill backend, a call with no session owner, or a `saveText()` failure keeps the inline page and reports that the complete result could not be saved — never an `isError`. ## Errors -Search failures carry the package-owned `SearchError` (a `HarnessError` subclass), surfaced as `{ name, code }` on `isError` results: `SEARCH_INVALID_PATTERN` (ripgrep rejected the regex/glob), `SEARCH_FAILED` (missing `rg`, inaccessible target, signal kill, malformed `--json` output), `SEARCH_RAW_OUTPUT_OVERFLOW` (raw output over `rawOutputMaxBytes`, or truncated with no recovery file), and `SEARCH_ABORTED` (tool timeout, caller cancellation, or the bash executor's own timeout). ripgrep exit semantics are tool-owned: exit 0 is success with results, exit 1 is a successful empty search (`No files found` / `No matches found`), and only other exits are failures. Model argument mistakes (blank pattern, a list-valued `include`) stay ordinary tool argument errors. +Search failures carry the package-owned `SearchError` (a `HarnessError` subclass), surfaced as `{ name, code }` on `isError` results: `SEARCH_INVALID_PATTERN` (ripgrep rejected the regex/glob), `SEARCH_FAILED` (missing `rg`, inaccessible target, signal kill, malformed `--json` output), `SEARCH_RAW_OUTPUT_OVERFLOW` (raw output over `rawOutputMaxBytes`, or still truncated after the requested stdout capture budget), and `SEARCH_ABORTED` (tool timeout, caller cancellation, or the bash executor's own timeout). ripgrep exit semantics are tool-owned: exit 0 is success with results, exit 1 is a successful empty search (`No files found` / `No matches found`), and only other exits are failures. Model argument mistakes (blank pattern, a list-valued `include`) stay ordinary tool argument errors. diff --git a/packages/fs/tool-fs-search/src/search-core.ts b/packages/fs/tool-fs-search/src/search-core.ts index a47adfa7c3..233c1e78d4 100644 --- a/packages/fs/tool-fs-search/src/search-core.ts +++ b/packages/fs/tool-fs-search/src/search-core.ts @@ -7,17 +7,15 @@ * Both tools execute through `ctx.bash.resolve(request)` → `ctx.bash.run(spec)` * as ordinary foreground tool calls — never `ctx.bash.start()`, never a * model-visible background task. Raw `rg` stdout is an internal transport - * detail: when the executor truncates it, the ONLY recovery source is the - * executor's local raw spill file, read here up to `rawOutputMaxBytes` and - * never exposed to the model. The model-facing recovery artifact is the + * detail: the tools request a per-run stdout capture budget from the bash seam, + * parse only complete in-memory stdout within `rawOutputMaxBytes`, and never + * read executor spill files. The model-facing recovery artifact is the * formatted result saved through `ctx.spillFiles.saveText()` - * ({@link trySaveFormattedResult}) — a different artifact from the bash raw - * spill file. + * ({@link trySaveFormattedResult}). * * @module @deepseek-ai/dsh-tool-fs-search/search-core */ -import { readFile, stat } from 'node:fs/promises' import { isAbsolute, relative, sep } from 'node:path' import type { Context } from 'cordis' import { HarnessError } from '@deepseek-ai/dsh-llm' @@ -45,7 +43,7 @@ export const SEARCH_TIMEOUT_MS = 30_000 * glob; `SEARCH_FAILED` — the search could not run or its output could not be * parsed (missing `rg`, inaccessible target, signal kill, malformed `--json`); * `SEARCH_RAW_OUTPUT_OVERFLOW` — raw `rg` output exceeded `rawOutputMaxBytes` - * (or was truncated with no recovery file); `SEARCH_ABORTED` — the tool + * or stayed truncated after that requested stdout budget; `SEARCH_ABORTED` — the tool * timeout, caller cancellation, or the bash executor's own timeout cut the * search short. */ @@ -72,7 +70,7 @@ export class SearchError extends HarnessError { /** The completed acquisition of one `rg` run: complete stdout plus the resolved workdir. */ export interface RipgrepRun { - /** Complete raw stdout — inline executor text, or the raw spill file's content. */ + /** Complete raw stdout retained by the bash executor within the requested cap. */ stdout: string /** True when ripgrep exited 1: a successful search with zero results. */ noMatches: boolean @@ -104,14 +102,11 @@ function classifyRunFailure(toolName: string, result: BashRunResult): SearchErro /** * Acquire the COMPLETE raw stdout of a finished run, enforcing - * `rawOutputMaxBytes` on BOTH transports: inline executor text (an executor - * retaining more than this package's cap must not smuggle an over-cap parse - * through the untruncated path) and the executor's local raw spill file, read - * only when the complete file fits the cap. A missing spill path or over-cap - * output is a clear failure telling the model to narrow the search — never a - * silently-partial parse. + * `rawOutputMaxBytes` on the in-memory transport. A truncated result means the + * bash backend could not retain complete stdout within the requested budget, so + * the tool fails clearly instead of parsing a silently-partial stream. */ -async function completeStdout(toolName: string, result: BashRunResult, rawOutputMaxBytes: number): Promise { +function completeStdout(toolName: string, result: BashRunResult, rawOutputMaxBytes: number): string { const narrow = 'narrow pattern, path, or include and retry' if (!result.stdout.truncated) { const inlineBytes = Buffer.byteLength(result.stdout.text, 'utf8') @@ -123,26 +118,10 @@ async function completeStdout(toolName: string, result: BashRunResult, rawOutput } return result.stdout.text } - const spillPath = result.stdout.spillPath - if (spillPath === undefined) { - throw new SearchError( - `${toolName} produced more raw output than the bash executor retained and no raw spill file is available; ${narrow}`, - 'SEARCH_RAW_OUTPUT_OVERFLOW', - ) - } - try { - const { size } = await stat(spillPath) - if (size > rawOutputMaxBytes) { - throw new SearchError( - `${toolName} produced ${size} bytes of raw output, over the ${rawOutputMaxBytes}-byte cap; ${narrow}`, - 'SEARCH_RAW_OUTPUT_OVERFLOW', - ) - } - return await readFile(spillPath, 'utf8') - } catch (error: unknown) { - if (error instanceof SearchError) throw error - throw new SearchError(`${toolName} could not read the executor's raw output spill file`, 'SEARCH_FAILED', { cause: error }) - } + throw new SearchError( + `${toolName} produced more raw output than the bash executor retained within the ${rawOutputMaxBytes}-byte cap; ${narrow}`, + 'SEARCH_RAW_OUTPUT_OVERFLOW', + ) } /** @@ -181,6 +160,7 @@ export async function runRipgrep( const cwd = exec.agent?.session.header.cwd const spec = ctx.bash.resolve({ command, + stdoutMaxBytes: rawOutputMaxBytes, ...cwd !== undefined ? { workdir: cwd } : {}, ...exec.signal ? { signal: exec.signal } : {}, }) @@ -208,7 +188,7 @@ export async function runRipgrep( if (result.exitCode !== 0 && result.exitCode !== 1) { throw classifyRunFailure(toolName, result) } - const stdout = await completeStdout(toolName, result, rawOutputMaxBytes) + const stdout = completeStdout(toolName, result, rawOutputMaxBytes) return { stdout, noMatches: result.exitCode === 1, workdir: spec.workdir } } diff --git a/packages/fs/tool-fs-search/tests/tools.spec.ts b/packages/fs/tool-fs-search/tests/tools.spec.ts index 7abbe20638..b3afa8ce3e 100644 --- a/packages/fs/tool-fs-search/tests/tools.spec.ts +++ b/packages/fs/tool-fs-search/tests/tools.spec.ts @@ -2,7 +2,7 @@ * Consumer-surface tests for the search tools over a FAKE bash executor and a * FAKE spill backend, exercised through `ctx.tools.execute()` so nothing * bypasses the tool registry. The fake executor makes every seam outcome - * scriptable — truncated stdout with/without a raw spill file, abort/timeout, + * scriptable — truncated stdout with/without a raw spill path, abort/timeout, * signal kills, ripgrep exit codes — so these tests verify schemas, argument * validation, shell-safe command construction, workdir derivation, signal * forwarding, `SEARCH_*` error classification, retention, formatted-result @@ -10,10 +10,7 @@ * pinned separately in integration.spec.ts. */ -import { afterEach, describe, expect, it } from 'vitest' -import { mkdtemp, rm, writeFile } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' +import { describe, expect, it } from 'vitest' import { Context } from 'cordis' import { CallId } from '@deepseek-ai/dsh-llm' import SystemPrompt, { renderPrompt } from '@deepseek-ai/dsh-system-prompt' @@ -66,6 +63,7 @@ class FakeBash extends BashExecutor { command: request.command, workdir: request.workdir ?? '/work', timeoutMs: request.timeoutMs ?? 60_000, + stdoutMaxBytes: request.stdoutMaxBytes ?? 64_000, signal: request.signal, owner: request.owner, } @@ -388,28 +386,18 @@ describe('exit semantics and failure classification', () => { }) describe('raw output acquisition', () => { - let dir: string - afterEach(async () => { - await rm(dir, { recursive: true, force: true }) + it('passes rawOutputMaxBytes to bash as the stdout capture budget', async () => { + const { ctx, bash } = await setup({ config: { rawOutputMaxBytes: 1234 } }) + bash.handler = () => runResult('', { exitCode: 1 }) + await call(ctx, 'glob', { pattern: '*.ts' }) + await call(ctx, 'grep', { pattern: 'needle' }) + expect(bash.requests.map(request => request.stdoutMaxBytes)).toEqual([1234, 1234]) + expect(bash.specs.map(spec => spec.stdoutMaxBytes)).toEqual([1234, 1234]) }) - it('parses the complete raw spill file when stdout is truncated', async () => { - dir = await mkdtemp(join(tmpdir(), 'dsh-search-raw-')) - const spillPath = join(dir, 'raw.txt') - await writeFile(spillPath, 'one.ts\ntwo.ts\nthree.ts\n') - const { ctx, bash } = await setup() - bash.handler = () => runResult('', { stdout: { text: 'one.ts\n', truncated: true, spillPath } }) - const result = await call(ctx, 'glob', { pattern: '*.ts' }) - expect(result.isError).toBe(false) - expect(text(result)).toBe('one.ts\ntwo.ts\nthree.ts') - }) - - it('fails with SEARCH_RAW_OUTPUT_OVERFLOW when the raw spill file exceeds the cap', async () => { - dir = await mkdtemp(join(tmpdir(), 'dsh-search-raw-')) - const spillPath = join(dir, 'raw.txt') - await writeFile(spillPath, 'x'.repeat(64)) + it('fails with SEARCH_RAW_OUTPUT_OVERFLOW when truncated stdout has a raw spill path', async () => { const { ctx, bash } = await setup({ config: { rawOutputMaxBytes: 16 } }) - bash.handler = () => runResult('', { stdout: { text: 'x', truncated: true, spillPath } }) + bash.handler = () => runResult('', { stdout: { text: 'x', truncated: true, spillPath: '/does/not/get-read' } }) const result = await call(ctx, 'glob', { pattern: '*' }) expect(result.error).toMatchObject({ code: 'SEARCH_RAW_OUTPUT_OVERFLOW' }) expect(text(result)).toContain('narrow pattern, path, or include') @@ -419,7 +407,6 @@ describe('raw output acquisition', () => { // An executor retaining more inline than this package's cap (or a // deployment lowering rawOutputMaxBytes below the bash retention) must not // smuggle an over-cap parse through the untruncated path. - dir = await mkdtemp(join(tmpdir(), 'dsh-search-raw-')) const { ctx, bash } = await setup({ config: { rawOutputMaxBytes: 16 } }) bash.handler = () => runResult(`${'x'.repeat(64)}\n`) const result = await call(ctx, 'grep', { pattern: 'x' }) @@ -428,21 +415,11 @@ describe('raw output acquisition', () => { }) it('fails with SEARCH_RAW_OUTPUT_OVERFLOW when truncated stdout has no spill path', async () => { - dir = await mkdtemp(join(tmpdir(), 'dsh-search-raw-')) const { ctx, bash } = await setup() bash.handler = () => runResult('', { stdout: { text: 'partial', truncated: true } }) const result = await call(ctx, 'grep', { pattern: 'x' }) expect(result.error).toMatchObject({ code: 'SEARCH_RAW_OUTPUT_OVERFLOW' }) }) - - it('fails with SEARCH_FAILED when the raw spill file cannot be read', async () => { - dir = await mkdtemp(join(tmpdir(), 'dsh-search-raw-')) - const { ctx, bash } = await setup() - bash.handler = () => runResult('', { stdout: { text: 'partial', truncated: true, spillPath: join(dir, 'gone.txt') } }) - const result = await call(ctx, 'glob', { pattern: '*' }) - expect(result.error).toMatchObject({ code: 'SEARCH_FAILED' }) - expect(text(result)).toContain('raw output spill file') - }) }) describe('glob results', () => { diff --git a/packages/hooks/hook-protocol/tests/runner.spec.ts b/packages/hooks/hook-protocol/tests/runner.spec.ts index 1972a39c99..45e6598eeb 100644 --- a/packages/hooks/hook-protocol/tests/runner.spec.ts +++ b/packages/hooks/hook-protocol/tests/runner.spec.ts @@ -22,6 +22,7 @@ function recordingBash(run: (spec: BashExecSpec) => Promise): { command: request.command, workdir: request.workdir ?? '/stub', timeoutMs: request.timeoutMs ?? 0, + stdoutMaxBytes: request.stdoutMaxBytes ?? 64_000, ...request.signal ? { signal: request.signal } : {}, ...request.stdin !== undefined ? { stdin: request.stdin } : {}, ...request.env !== undefined ? { env: request.env } : {}, From 8b86c3febc03d4416b1e2868f04511ff9d1720cc Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Fri, 10 Jul 2026 13:15:15 +0800 Subject: [PATCH 21/24] fix: stabilize spill snapshot path budget --- examples/acp-agent/cordis.yml | 6 ++++-- .../tests/snapshots/bash-spill/session.jsonl | 2 +- .../snapshots/bash-spill/stdout.golden.jsonl | 2 +- packages/support/acp-snapshot/src/harness.ts | 5 +++++ packages/support/acp-snapshot/src/normalize.ts | 6 ++++++ .../support/acp-snapshot/tests/normalize.spec.ts | 15 +++++++++++++++ 6 files changed, 32 insertions(+), 4 deletions(-) diff --git a/examples/acp-agent/cordis.yml b/examples/acp-agent/cordis.yml index 48760828e2..a6889114fd 100644 --- a/examples/acp-agent/cordis.yml +++ b/examples/acp-agent/cordis.yml @@ -113,11 +113,13 @@ # results under the session cwd, and the post-execute policy that replaces the # model-facing result with a bounded preview + read path. Snapshots lower the # cap so a deterministic bash result exercises this transcript surface without a -# real model call; normal demo runs keep the coding-agent cap. +# real model call. The snapshot harness supplies a fixed spill root so the +# spill-policy preview budget is stable across macOS/Linux path lengths; normal +# demo runs keep the session-local `.spill` root and the coding-agent cap. - id: spill-local name: '@deepseek-ai/dsh-spill-local' config: - root: ./.spill + root: !!js process.env.DSH_SNAPSHOT_SPILL_ROOT ?? './.spill' - id: spill-policy name: '@deepseek-ai/dsh-spill-policy' diff --git a/examples/acp-agent/tests/snapshots/bash-spill/session.jsonl b/examples/acp-agent/tests/snapshots/bash-spill/session.jsonl index 7c7fe12630..aa60d5143e 100644 --- a/examples/acp-agent/tests/snapshots/bash-spill/session.jsonl +++ b/examples/acp-agent/tests/snapshots/bash-spill/session.jsonl @@ -10,7 +10,7 @@ {"type":"assistant/chunk","seq":8,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}} {"type":"assistant/message","seq":9,"time":0,"data":{"turn":1,"step":1,"content":[{"type":"tool-call","id":"call_spill","name":"bash","arguments":"{\"command\":\"node -e \\\"process.stdout.write('SPILL_START-' + 'x'.repeat(2000) + '-SPILL_END')\\\"\",\"description\":\"Print large deterministic output\"}"}],"usage":{"inputTokens":10,"outputTokens":5}},"sourceEventSeqs":[4,5,6,7,8],"surfaceOp":"append"} {"type":"tool/call","seq":10,"time":0,"data":{"turn":1,"step":1,"callId":"call_spill","name":"bash","arguments":"{\"command\":\"node -e \\\"process.stdout.write('SPILL_START-' + 'x'.repeat(2000) + '-SPILL_END')\\\"\",\"description\":\"Print large deterministic output\"}"}} -{"type":"tool/result","seq":11,"time":0,"data":{"turn":1,"step":1,"callId":"call_spill","content":[{"type":"text","text":"SPILL_START-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-SPILL_END\n\n(Omitted 1447 bytes. Full formatted result saved to: {{spillPath:bash.txt}}. Use read with offset/limit to inspect it.)"}],"isError":false},"sourceEventSeqs":[10],"surfaceOp":"append"} +{"type":"tool/result","seq":11,"time":0,"data":{"turn":1,"step":1,"callId":"call_spill","content":[{"type":"text","text":"SPILL_START-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-SPILL_END\n\n(Omitted 1391 bytes. Full formatted result saved to: {{spillPath:bash.txt}}. Use read with offset/limit to inspect it.)"}],"isError":false},"sourceEventSeqs":[10],"surfaceOp":"append"} {"type":"step/end","seq":12,"time":0,"data":{"turn":1,"step":1}} {"type":"step/start","seq":13,"time":0,"data":{"turn":1,"step":2}} {"type":"assistant/chunk","seq":14,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"block-start","index":0,"blockType":"text"}}} diff --git a/examples/acp-agent/tests/snapshots/bash-spill/stdout.golden.jsonl b/examples/acp-agent/tests/snapshots/bash-spill/stdout.golden.jsonl index dfd9c4fb2b..df47558232 100644 --- a/examples/acp-agent/tests/snapshots/bash-spill/stdout.golden.jsonl +++ b/examples/acp-agent/tests/snapshots/bash-spill/stdout.golden.jsonl @@ -1,6 +1,6 @@ {"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentInfo":{"name":"deepseek-harness-acp","version":"0.0.1"},"agentCapabilities":{"loadSession":true,"promptCapabilities":{"image":false,"audio":false,"embeddedContext":false}},"authMethods":[]}} {"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}"}} {"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call","toolCallId":"call_spill","title":"node -e \"process.stdout.write('SPILL_START-' + 'x'.repeat(2000) + '-SPILL_END')\"","kind":"execute","status":"in_progress","rawInput":"node -e \"process.stdout.write('SPILL_START-' + 'x'.repeat(2000) + '-SPILL_END')\"","content":[{"type":"content","content":{"type":"text","text":"Print large deterministic output"}}]}}} -{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call_update","toolCallId":"call_spill","status":"completed","content":[{"type":"content","content":{"type":"text","text":"```console\nSPILL_START-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-SPILL_END\n\n(Omitted 1447 bytes. Full formatted result saved to: {{spillPath:bash.txt}}. Use read with offset/limit to inspect it.)\n```"}}]}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call_update","toolCallId":"call_spill","status":"completed","content":[{"type":"content","content":{"type":"text","text":"```console\nSPILL_START-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-SPILL_END\n\n(Omitted 1391 bytes. Full formatted result saved to: {{spillPath:bash.txt}}. Use read with offset/limit to inspect it.)\n```"}}]}}} {"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"DONE"}}}} {"jsonrpc":"2.0","id":3,"result":{"stopReason":"end_turn"}} diff --git a/packages/support/acp-snapshot/src/harness.ts b/packages/support/acp-snapshot/src/harness.ts index 538b81d57e..befcf32dca 100644 --- a/packages/support/acp-snapshot/src/harness.ts +++ b/packages/support/acp-snapshot/src/harness.ts @@ -180,6 +180,9 @@ export interface RunOptions { export async function runScenario(input: InputScript, opts: RunOptions): Promise { const cwd = await mkdtemp(join(tmpdir(), 'acp-snap-cwd-')) const sessionsRoot = await mkdtemp(join(tmpdir(), 'acp-snap-sessions-')) + // Fixed path length: spill-policy budgets the preview against the REAL path + // before stdout normalization, so tmpdir() length differences churn goldens. + const spillRoot = '/tmp/dsh-acp-snapshot-spill' // Everything past the temp-dir creation runs under a try/finally that always // removes both dirs — so a failure in workspace seeding, spawn, or any step // never leaks them (the "e2e tests own their resources" rule). @@ -201,6 +204,7 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise DSH_SNAPSHOT: opts.mode, DSH_SNAPSHOT_FILE: opts.fixtureFile, DSH_SNAPSHOT_SESSIONS_ROOT: sessionsRoot, + DSH_SNAPSHOT_SPILL_ROOT: spillRoot, ...opts.overrideFile !== undefined ? { DSH_SNAPSHOT_OVERRIDE: opts.overrideFile } : {}, ...opts.childFiles !== undefined && opts.childFiles.length > 0 ? { DSH_SNAPSHOT_CHILD_FILES: opts.childFiles.join(delimiter) } @@ -310,6 +314,7 @@ export async function runScenario(input: InputScript, opts: RunOptions): Promise } await rm(cwd, { recursive: true, force: true }) await rm(sessionsRoot, { recursive: true, force: true }) + await rm(spillRoot, { recursive: true, force: true }) } return { diff --git a/packages/support/acp-snapshot/src/normalize.ts b/packages/support/acp-snapshot/src/normalize.ts index a4ce886f8b..e5b15a4abb 100644 --- a/packages/support/acp-snapshot/src/normalize.ts +++ b/packages/support/acp-snapshot/src/normalize.ts @@ -38,6 +38,11 @@ const LOCAL_SPILL_PATH_RE = new RegExp( + String.raw`(?=\. Use read with offset/limit|[\s)]|$)`, 'g', ) +const SNAPSHOT_SPILL_PATH_RE = new RegExp( + String.raw`/tmp/dsh-acp-snapshot-spill/session-[0-9a-f]{12}/[0-9a-f]{12}-([A-Za-z0-9._~-]+?)` + + String.raw`(?=\. Use read with offset/limit|[\s)]|$)`, + 'g', +) /** Inputs the normalizers need to recognize a run's volatile values. */ export interface NormalizeContext { @@ -55,6 +60,7 @@ function scrubString(value: string, ctx: NormalizeContext): string { out = out.split(ctx.cwd).join(CWD) out = out.split(`/private${CWD}`).join(CWD) out = out.replace(LOCAL_SPILL_PATH_RE, (_match, name: string) => `{{spillPath:${name}}}`) + out = out.replace(SNAPSHOT_SPILL_PATH_RE, (_match, name: string) => `{{spillPath:${name}}}`) for (const id of ctx.sessionIds) out = out.split(id).join(SESSION_ID) out = out.replace(UUID_RE, SESSION_ID) return out diff --git a/packages/support/acp-snapshot/tests/normalize.spec.ts b/packages/support/acp-snapshot/tests/normalize.spec.ts index a33436a164..ec388294bf 100644 --- a/packages/support/acp-snapshot/tests/normalize.spec.ts +++ b/packages/support/acp-snapshot/tests/normalize.spec.ts @@ -117,6 +117,21 @@ describe('normalizeSessionLog', () => { expect(out).not.toContain('/private{{spillPath') }) + it('scrubs fixed snapshot spill paths', () => { + const ev = JSON.stringify({ + type: 'tool/result', seq: 2, time: 5, + data: { + content: [{ + type: 'text', + text: 'Full formatted result saved to: /tmp/dsh-acp-snapshot-spill/session-c22bc3f1d2af/8a7b6c5d4e3f-bash.txt. Use read with offset/limit to inspect it.', + }], + }, + }) + const out = normalizeSessionLog(`${header({ cwd: ctx.cwd })}\n${ev}\n`, ctx) + expect(out).toContain('{{spillPath:bash.txt}}') + expect(out).not.toContain('/tmp/dsh-acp-snapshot-spill') + }) + it('scrubs the session id in the header', () => { const out = normalizeSessionLog(`${header({ id: ctx.sessionIds[0] })}\n`, ctx) expect(out).toContain('{{sessionId}}') From 5ac03dde3fb2bc125263088d1f6015770d3878c0 Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Mon, 13 Jul 2026 11:07:27 +0800 Subject: [PATCH 22/24] fix(review): generalize spill storage locators --- docs/capability-seams.md | 10 ++--- docs/config-catalog.md | 2 +- docs/cordis-catalog/services.md | 8 ++-- docs/core-data-structures/core.md | 2 +- docs/core-data-structures/spill.md | 15 ++++--- .../2026-07-08-tool-output-spill-files.md | 44 +++++++++---------- ...6-07-09-bash-backed-grep-glob-discovery.md | 36 +++++++-------- docs/tool-catalog.md | 4 +- .../tests/snapshots/bash-spill/session.jsonl | 2 +- .../snapshots/bash-spill/stdout.golden.jsonl | 2 +- examples/coding-agent/cordis.yml | 8 ++-- .../cordis/tool-cordis/src/api-catalog.ts | 12 ++--- packages/fs/tool-fs-search/README.md | 12 ++--- packages/fs/tool-fs-search/src/glob.ts | 15 ++++--- packages/fs/tool-fs-search/src/grep.ts | 15 ++++--- packages/fs/tool-fs-search/src/index.ts | 4 +- packages/fs/tool-fs-search/src/search-core.ts | 23 +++++----- .../fs/tool-fs-search/tests/tools.spec.ts | 18 +++++--- packages/spill/README.md | 6 +-- packages/spill/spill-local/README.md | 2 +- packages/spill/spill-local/src/index.ts | 18 +++++--- .../spill-local/tests/spill-local.spec.ts | 35 ++++++++------- packages/spill/spill-policy/README.md | 14 +++--- packages/spill/spill-policy/src/index.ts | 36 +++++++-------- packages/spill/spill-policy/src/types.ts | 2 +- .../spill-policy/tests/spill-policy.spec.ts | 32 ++++++++------ packages/spill/spill/README.md | 12 ++--- packages/spill/spill/package.json | 2 +- packages/spill/spill/src/index.ts | 36 +++++++-------- packages/spill/spill/src/types.ts | 27 ++++++------ packages/spill/spill/tests/service.spec.ts | 32 ++++++++------ .../support/acp-snapshot/src/normalize.ts | 4 +- .../acp-snapshot/tests/normalize.spec.ts | 14 +++--- packages/web/tool-web/tests/spill.spec.ts | 20 ++++----- scripts/gen-doc-graphs.ts | 4 +- scripts/gen-tool-catalog.ts | 4 +- scripts/type-equiv.manifest.json | 2 +- 37 files changed, 274 insertions(+), 260 deletions(-) diff --git a/docs/capability-seams.md b/docs/capability-seams.md index 51a14b9162..0e18f44c01 100644 --- a/docs/capability-seams.md +++ b/docs/capability-seams.md @@ -78,7 +78,7 @@ flowchart LR pkg_web_search_deepseek["web-search-deepseek"] pkg_web_fetch_local["web-fetch-local"] pkg_spill["spill"] - svc_spillFiles["ctx.spillFiles
Spill storage seam"] + svc_spillStore["ctx.spillStore
Spill storage seam"] pkg_spill_local["spill-local"] pkg_spill_policy["spill-policy"] pkg_workflow["workflow"] @@ -111,8 +111,8 @@ flowchart LR pkg_session_persistence_sqlite --> svc_sessionPersistence pkg_skill --> svc_skills pkg_skill_local --> svc_skills - pkg_spill --> svc_spillFiles - pkg_spill_local --> svc_spillFiles + pkg_spill --> svc_spillStore + pkg_spill_local --> svc_spillStore pkg_stdio_agent --> svc_userInteraction pkg_subagent --> svc_subagents pkg_subagent_acp --> svc_subagents @@ -154,7 +154,7 @@ flowchart LR svc_sessions --> pkg_session_persistence svc_sessions --> pkg_subagent_inprocess svc_skills --> pkg_tool_skill - svc_spillFiles --> pkg_spill_policy + svc_spillStore --> pkg_spill_policy svc_subagents --> pkg_tool_subagent svc_systemPrompt --> pkg_agent_loop svc_systemPrompt --> pkg_tool_fs @@ -197,7 +197,7 @@ flowchart LR | `ctx.compact` | `seam` | [`compact`](../packages/compact/compact) | [`compact-basic`](../packages/compact/compact-basic) | [`compact-basic`](../packages/compact/compact-basic) | - | The basic backend currently consumes the pre-step event directly; a model-facing compact tool remains deferred. | | `ctx.subagents` | `seam` | [`subagent`](../packages/subagent/subagent) | [`subagent-spawn`](../packages/subagent/subagent-spawn), [`subagent-fork`](../packages/subagent/subagent-fork), [`subagent-acp`](../packages/subagent/subagent-acp), [`subagent-mock`](../packages/support/subagent-mock) | [`tool-subagent`](../packages/subagent/tool-subagent) | - | Providers implement transports; tool-subagent exposes one configured provider as a model-facing tool name. | | `ctx.web` | `seam` | [`web`](../packages/web/web) | [`web-search-exa`](../packages/web/web-search-exa), [`web-search-perplexity`](../packages/web/web-search-perplexity), [`web-search-deepseek`](../packages/web/web-search-deepseek), [`web-fetch-local`](../packages/web/web-fetch-local) | [`tool-web`](../packages/web/tool-web) | - | Search and fetch providers register into one ctx.web seam; tool-web owns the stable model-facing names. | -| `ctx.spillFiles` | `seam` | [`spill`](../packages/spill/spill) | [`spill-local`](../packages/spill/spill-local) | [`spill-policy`](../packages/spill/spill-policy) | - | The backend saves oversized tool text to a session-scoped path; spill-policy is the tools/post-execute consumer that decides when to spill. | +| `ctx.spillStore` | `seam` | [`spill`](../packages/spill/spill) | [`spill-local`](../packages/spill/spill-local) | [`spill-policy`](../packages/spill/spill-policy) | - | The backend saves oversized tool text and returns a model-facing locator plus retrieval hint; spill-policy is the tools/post-execute consumer that decides when to spill. | | `ctx.workflows` | `seam` | [`workflow`](../packages/workflow/workflow) | [`workflow-workerthread`](../packages/workflow/workflow-workerthread) | [`tool-workflow`](../packages/workflow/tool-workflow) | - | One engine per context (bash shape, no named-provider registry); the worker-thread engine fans agent() calls out through ctx.subagents. | Maintenance mode: hybrid: services are discovered from Cordis declarations; interface/implementation/consumer roles are classified in `scripts/gen-doc-graphs.ts` with a completeness guard. diff --git a/docs/config-catalog.md b/docs/config-catalog.md index 808779dd8f..f57dcb147f 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -1211,7 +1211,7 @@ Abstract service classes — a deployment loads a concrete implementation packag - `@deepseek-ai/dsh-fs` — abstract `FileSystem` ([`packages/fs/fs/src/index.ts`](../packages/fs/fs/src/index.ts)) - `@deepseek-ai/dsh-sandbox` — abstract `SandboxProvider` ([`packages/sandbox/sandbox/src/index.ts`](../packages/sandbox/sandbox/src/index.ts)) - `@deepseek-ai/dsh-session-persistence` — abstract `SessionPersistence` ([`packages/session-persistence/session-persistence/src/index.ts`](../packages/session-persistence/session-persistence/src/index.ts)) -- `@deepseek-ai/dsh-spill` — abstract `SpillFiles` ([`packages/spill/spill/src/index.ts`](../packages/spill/spill/src/index.ts)) +- `@deepseek-ai/dsh-spill` — abstract `SpillStore` ([`packages/spill/spill/src/index.ts`](../packages/spill/spill/src/index.ts)) - `@deepseek-ai/dsh-workflow` — abstract `WorkflowService` ([`packages/workflow/workflow/src/index.ts`](../packages/workflow/workflow/src/index.ts)) ## Library packages (no plugin entry) diff --git a/docs/cordis-catalog/services.md b/docs/cordis-catalog/services.md index d2ff829538..5328a8c86a 100644 --- a/docs/cordis-catalog/services.md +++ b/docs/cordis-catalog/services.md @@ -232,13 +232,13 @@ async get(name: string, options: SkillLookupOptions = {}): Promise ``` -Source: [`packages/spill/spill/src/index.ts:46`](../../packages/spill/spill/src/index.ts) +Source: [`packages/spill/spill/src/index.ts:45`](../../packages/spill/spill/src/index.ts) ## `ctx.subagents` — `SubagentService` diff --git a/docs/core-data-structures/core.md b/docs/core-data-structures/core.md index 151ba22deb..93038cc33a 100644 --- a/docs/core-data-structures/core.md +++ b/docs/core-data-structures/core.md @@ -29,7 +29,7 @@ Everything else is documented on a **sub-page**, not here. The rule that draws t | [compaction.md](compaction.md) | the compaction seam: the `compact/*` session events, `CompactionResult`, the `CompactService` interface | | [subagent.md](subagent.md) | the subagent seam: the named-provider registry, `SubagentStartRequest`/`Result`/`Run`, the start-time-vs-runtime capability split | | [web.md](web.md) | the web access seam: `WebSearchRequest`/`Result`, `WebFetchRequest`/`Result`, `WebFetchBody`, provider/capability status, `WebError` | -| [spill.md](spill.md) | the spill storage seam: `SaveTextSpill`, `SpillOwner`/`SpillSource`, `SpillRef`, the branded `SpillPath` | +| [spill.md](spill.md) | the spill storage seam: `SaveTextSpill`, `SpillOwner`/`SpillSource`, `SpillRef`, the branded `SpillLocator` | | [workflow.md](workflow.md) | the workflow seam: `WorkflowStartRequest`, `WorkflowMeta`, `WorkflowRun`/`Result`, the `workflow/*` event payloads, `WorkflowError` fatality | > Type definitions on this page are pasted **verbatim** from source and drift-checked by `pnpm run verify-type-equiv` (see [development.md](../development.md#documenting-types-verbatim-ts-type-equiv)). Inline JSDoc is omitted for readability; follow the source link for the full contracts. diff --git a/docs/core-data-structures/spill.md b/docs/core-data-structures/spill.md index 7586d70633..afc825f09d 100644 --- a/docs/core-data-structures/spill.md +++ b/docs/core-data-structures/spill.md @@ -1,12 +1,12 @@ # Spill Storage -The spill storage seam — a [capability seam](../rfc/implemented/architecture/2026-07-08-tool-output-spill-files.md) that persists a tool's oversized text to a session-scoped path the model can later `read`, split across packages: interface ([dsh-spill](../../packages/spill/spill), `ctx.spillFiles`), implementation ([dsh-spill-local](../../packages/spill/spill-local), private session-scoped files on the host filesystem), and consumer ([dsh-spill-policy](../../packages/spill/spill-policy), the `tools/post-execute` policy). Spill is **one optional capability**, not part of the agent-loop spine — so its vocabulary lives here, not in [core.md](core.md). Preview mechanics stay in [dsh-retention](../../packages/util/retention); this seam only saves the final text the policy hands it. +The spill storage seam — a [capability seam](../rfc/implemented/architecture/2026-07-08-tool-output-spill-files.md) that persists a tool's oversized text and returns a model-facing locator plus retrieval guidance, split across packages: interface ([dsh-spill](../../packages/spill/spill), `ctx.spillStore`), implementation ([dsh-spill-local](../../packages/spill/spill-local), private session-scoped files on the host filesystem), and consumer ([dsh-spill-policy](../../packages/spill/spill-policy), the `tools/post-execute` policy). Spill is **one optional capability**, not part of the agent-loop spine — so its vocabulary lives here, not in [core.md](core.md). Preview mechanics stay in [dsh-retention](../../packages/util/retention); this seam only saves the final text the policy hands it. Source: [`packages/spill/spill/src/types.ts`](../../packages/spill/spill/src/types.ts) ## The save request -`saveText` is the whole seam: persist `content` verbatim, return a readable path plus the exact byte count. The request carries WHO the artifact belongs to (`owner`), WHERE it came from (`source`, descriptive provenance for the filename and future cleanup — not access control), and a `suggestedName` the backend sanitizes to one safe path segment before use (it is a hint, never a path). +`saveText` is the whole seam: persist `content` verbatim, return an opaque locator, a backend-supplied retrieval hint, and the exact byte count. The request carries WHO the artifact belongs to (`owner`), WHERE it came from (`source`, descriptive provenance for naming and future cleanup — not access control), and a `suggestedName` the backend may use as a naming hint (it is not a path). ```ts type-equiv interface SaveTextSpill { @@ -37,19 +37,20 @@ interface SpillSource { ```ts type-equiv interface SpillRef { - path: SpillPath + locator: SpillLocator bytes: number + retrievalHint: string } ``` -`SpillPath` is a [branded](core.md#branded-ids) local filesystem path returned by the backend and intended for the model's `read` tool. The brand records that the path came from the spill seam (a runtime artifact, not a workspace file the model authored); it is still rendered to the model as an ordinary path string in v1. A future remote or virtual backend may replace it with a `spill://…` URI plus a read-only filesystem bridge, so consumers treat it as opaque. +`SpillLocator` is a [branded](core.md#branded-ids) model-facing handle returned by the backend. The local backend renders it as a filesystem path; a remote or database backend can render a URI, key, or command token. Consumers treat it as opaque and render it with `retrievalHint` instead of assuming `read` is always the right retrieval mechanism. ```ts type-equiv -type SpillPath = Branded<'SpillPath'> +type SpillLocator = Branded<'SpillLocator'> ``` ## The service -`SpillFiles` (`ctx.spillFiles`, defined in [`packages/spill/spill/src/index.ts`](../../packages/spill/spill/src/index.ts)) is a one-method abstract service: `saveText(input) → Promise`. It persists the FULL `content`, chooses a private (not world-readable) location and a collision-free name derived from — never equal to — `suggestedName`, and REJECTS on a real storage failure (permissions, ENOSPC, backend unavailable). The seam owns storage only: no retention policy, no tool-result replacement, no file inspection. +`SpillStore` (`ctx.spillStore`, defined in [`packages/spill/spill/src/index.ts`](../../packages/spill/spill/src/index.ts)) is a one-method abstract service: `saveText(input) → Promise`. It persists the FULL `content` and REJECTS on a real storage failure (permissions, ENOSPC, backend unavailable). The seam owns storage only: no retention policy, no tool-result replacement, no retrieval/search API. -The local backend ([dsh-spill-local](../../packages/spill/spill-local)) writes under `/session-/-` — a configured or lazily-created private (0700) root, a `sha256(sessionId)` session subdir, and an exclusive owner-only (`open(path, 'wx', 0o600)`) write so a planted symlink cannot redirect it. The policy consumer ([dsh-spill-policy](../../packages/spill/spill-policy)) replaces an over-`maxInlineBytes` plain-text final result with a retention-library head/tail preview plus the spill path, best-effort: a save failure keeps the original inline result rather than turning a successful call into an `isError`. +The local backend ([dsh-spill-local](../../packages/spill/spill-local)) writes under `/session-/-` — a configured or lazily-created private (0700) root, a `sha256(sessionId)` session subdir, and an exclusive owner-only (`open(path, 'wx', 0o600)`) write so a planted symlink cannot redirect it. Its `locator` is the local path and its `retrievalHint` tells the model to use `read` or `grep` on that path. The policy consumer ([dsh-spill-policy](../../packages/spill/spill-policy)) replaces an over-`maxInlineBytes` plain-text final result with a retention-library head/tail preview plus the spill reference, best-effort: a save failure keeps the original inline result rather than turning a successful call into an `isError`. diff --git a/docs/rfc/implemented/architecture/2026-07-08-tool-output-spill-files.md b/docs/rfc/implemented/architecture/2026-07-08-tool-output-spill-files.md index f261ebf84e..ac56c151fb 100644 --- a/docs/rfc/implemented/architecture/2026-07-08-tool-output-spill-files.md +++ b/docs/rfc/implemented/architecture/2026-07-08-tool-output-spill-files.md @@ -16,18 +16,18 @@ A thin spill storage seam plus a default spill policy plugin, in a new `packages | Package | Role | |---|---| -| `@deepseek-ai/dsh-spill` | Interface: `ctx.spillFiles`, vocabulary types, no filesystem implementation. | +| `@deepseek-ai/dsh-spill` | Interface: `ctx.spillStore`, vocabulary types, no storage implementation. | | `@deepseek-ai/dsh-spill-local` | Local backend: private, session-scoped file storage on the host filesystem. | -| `@deepseek-ai/dsh-spill-policy` | Tool-result policy plugin: wraps final text results after dispatch and replaces oversized results with a retained preview plus a spill-file path. | +| `@deepseek-ai/dsh-spill-policy` | Tool-result policy plugin: wraps final text results after dispatch and replaces oversized results with a retained preview plus a spill locator. | -There is no dedicated model-facing consumer package. The consumer is the existing `ctx.tools` execution pipeline: `dsh-spill-policy` consumes final tool results through the `tools/post-execute` waterfall, and the model uses the existing `read` tool to inspect the returned path. +There is no dedicated model-facing consumer package. The consumer is the existing `ctx.tools` execution pipeline: `dsh-spill-policy` consumes final tool results through the `tools/post-execute` waterfall, and the model follows the backend-supplied retrieval hint for the returned locator. ### Spill seam -The storage seam is minimal: save text and return a local path. +The storage seam is minimal: save text and return a locator plus retrieval hint. ```ts ignore-check -interface SpillFiles { +interface SpillStore { saveText(input: SaveTextSpill): Promise } @@ -44,19 +44,18 @@ interface SaveTextSpill { content: string } -type SpillPath = Branded<'SpillPath'> +type SpillLocator = Branded<'SpillLocator'> interface SpillRef { - path: SpillPath + locator: SpillLocator bytes: number + retrievalHint: string } ``` -`SpillPath` is a [branded](../../../../packages/util/brand) local filesystem path returned by the backend and intended for `read`. The brand records that the path came from the spill seam (a runtime artifact); it is rendered to the model as an ordinary path string in v1. `SpillOwner` scopes storage to a `SessionId` — spill is inherently session-scoped, so the seam imports `dsh-session`'s `SessionId` directly rather than minting a decoupled token like the bash executor's `OwnerToken`. +`SpillLocator` is a [branded](../../../../packages/util/brand) model-facing handle returned by the backend. The local backend renders it as a filesystem path; a remote or database backend can render a URI, key, or command token. Consumers treat it as opaque and render it with `retrievalHint` instead of assuming `read` is always the right retrieval mechanism. `SpillOwner` scopes storage to a `SessionId` — spill is inherently session-scoped, so the seam imports `dsh-session`'s `SessionId` directly rather than minting a decoupled token like the bash executor's `OwnerToken`. -`dsh-spill-local` owns only storage details: session-scoped directory selection, safe names, path-traversal protection, the write, and returning `{ path, bytes }`. It does not own retention policy, model-facing wording, tool-result replacement, search, or file inspection. Files land at `/session-/-`, where `root` is a configured path or a lazily-created private (0700) per-process temp dir, the session subdir is a short `sha256(sessionId)` prefix, and the leaf is a random hex prefix plus the caller's `suggestedName` sanitized to one path segment (mirrors the JSONL backend's `encodeSegment`). The write is `open(path, 'wx', 0o600)` — exclusive and owner-only, so a planted symlink cannot redirect it. - -The v1 local backend returns a real local `path` readable by the existing `read` tool. A future remote or virtual backend may replace this with a `spill://...` URI plus a read-only filesystem bridge; v1 keeps the interface path-shaped until that backend exists. +`dsh-spill-local` owns only storage details: session-scoped directory selection, safe names, path-traversal protection, the write, and returning `{ locator, bytes, retrievalHint }`. It does not own retention policy, tool-result replacement, search, or file inspection. Files land at `/session-/-`, where `root` is a configured path or a lazily-created private (0700) per-process temp dir, the session subdir is a short `sha256(sessionId)` prefix, and the leaf is a random hex prefix plus the caller's `suggestedName` sanitized to one path segment (mirrors the JSONL backend's `encodeSegment`). The write is `open(path, 'wx', 0o600)` — exclusive and owner-only, so a planted symlink cannot redirect it. The locator is the path, and the retrieval hint tells the model it can use `read` or `grep` on that path. ### Spill policy @@ -74,8 +73,8 @@ When `maxInlineBytes` is omitted the plugin registers nothing (a true no-op). Wh 1. Let the tool run normally, delegating via `next()` so a downstream listener settles the result first. 2. Flatten the accepted final `ContentBlock[]` only when it is entirely plain text; a result with any non-text block is left untouched. 3. If its UTF-8 byte size is at or below `maxInlineBytes`, leave it unchanged. -4. If it is larger, call `ctx.spillFiles.saveText()` with the full final text. -5. Replace the model-facing result with a retained head/tail preview plus the spill path. +4. If it is larger, call `ctx.spillStore.saveText()` with the full final text. +5. Replace the model-facing result with a retained head/tail preview plus the spill reference. The preview is an implementation default owned by the policy: a head/tail split of `maxInlineBytes` via the retention library's `TextRetainer`. Future config can expose preview sizing only after a second deployment needs it. @@ -84,10 +83,10 @@ The replacement text is intentionally generic because the policy only knows the ```text -(Omitted N bytes. Full formatted result saved to: /.../session-.../....txt. Use read with offset/limit to inspect it.) +(Omitted N bytes. Full formatted result stored at: /.../session-.../....txt. Use read with offset/limit, or grep this path to search within it.) ``` -If `ctx.spillFiles.saveText()` fails (permissions, ENOSPC, backend unavailable), or the call has no session owner, or no backend is loaded, the plugin logs the reason and returns the original result unchanged. Spill failure never turns a successful tool call into an `isError` result or hides the inline result. +If `ctx.spillStore.saveText()` fails (permissions, ENOSPC, backend unavailable), or the call has no session owner, or no backend is loaded, the plugin logs the reason and returns the original result unchanged. Spill failure never turns a successful tool call into an `isError` result or hides the inline result. The policy skips `read` to avoid a circular `read -> spill file -> read again` loop. Additional opt-out configuration is deferred until a real second tool needs it. @@ -129,7 +128,7 @@ This separation is important. `web-fetch-local` still owns resource caps (`maxRe Retention is separate from spill storage: - `@deepseek-ai/dsh-retention` owns preview mechanics (`TextRetainer`, `ItemRetainer`, and omitted metadata). -- `@deepseek-ai/dsh-spill` owns saving final text to a session-scoped path. +- `@deepseek-ai/dsh-spill` owns saving final text and returning a locator plus retrieval hint. - `@deepseek-ai/dsh-spill-policy` applies the default final-result policy in the tool pipeline, composing the two. The final-result policy cannot replace tool-owned early spill. Some useful content is not present in final `ToolExecutionResult.content`: @@ -138,7 +137,7 @@ The final-result policy cannot replace tool-owned early spill. Some useful conte - `subagent` final output is the child final answer, not the child rollout. - Future tools may produce runtime artifacts that are never represented by their final `ToolExecutionResult.content`. -Those cases can consume `ctx.spillFiles` directly in later work. They are not part of the first showcase. +Those cases can consume `ctx.spillStore` directly in later work. They are not part of the first showcase. ## Non-goals @@ -154,13 +153,12 @@ Those cases can consume `ctx.spillFiles` directly in later work. They are not pa - `saveFile()` / `linkOrCopy` for existing executor spill files, needed for bash normalization. - Tool-owned spill for subagent rollouts (`await run.result`, read in-process child session before `run.dispose()`, save JSONL). - Per-tool opt-out or per-tool policy declarations if the built-in `read` skip is insufficient. -- A virtual `spill://` URI and read-only filesystem bridge. -- Remote storage backends for ACP or remote environments where a local path is not meaningful. +- Remote or database storage backends for ACP or remote environments where a local path is not meaningful. - Cleanup and retention policy for old spill files, likely tied to session cleanup. ## Testing -- `dsh-spill` unit tests pin the seam contract: registration as `ctx.spillFiles`, one-implementation-per-context, and disposal release. +- `dsh-spill` unit tests pin the seam contract: registration as `ctx.spillStore`, one-implementation-per-context, and disposal release. - `dsh-spill-local` unit tests cover `saveText`, `encodeSegment` sanitization (separators/tilde/whole-segment dots/empty), the session-hash directory, owner-only permissions, distinct paths per save, the configured/private root, and a storage-failure rejection. - `dsh-spill-policy` unit tests drive real tools through `ctx.tools.execute`: disabled-mode no-op, oversized-text replacement, small/non-text passthrough, `read` skip, best-effort fallback (save failure / no backend / no owner), and downstream-composition (bounding a replaced result, preserving `additionalContext`). - `dsh-tool-web` integration drives `web_fetch` through `ctx.tools.execute` with the real `spill-local` backend + policy, proving the model-facing text changes only by the deliberate spill notice while the spill file holds the full formatted result. @@ -170,9 +168,9 @@ Those cases can consume `ctx.spillFiles` directly in later work. They are not pa The default policy only sees final formatted text. It cannot preserve provider-internal content that was already capped or runtime artifacts that were never part of the result. This is acceptable for the first cut because the showcase is final-result spill, not early spill; tool-owned early spill remains deferred work. -Returning real paths from the local backend keeps v1 simple and matches proven agent-tool behavior, but it exposes implementation paths to the model and may not work for remote backends. The interface should be revisited when a virtual or remote spill backend exists. +Returning real paths from the local backend keeps v1 simple and matches proven agent-tool behavior, while the seam itself only promises an opaque locator plus retrieval hint so remote backends can return non-file locators. -The v1 value proposition depends on the existing `read` tool being able to inspect the returned local path, even when the spill directory is outside the session cwd. That holds today because the filesystem policy records observations and write guards but does not confine reads to the workspace. A future workspace-confinement policy must either allow spill paths explicitly or provide a read-only spill bridge, or the spill notice would point at an unreadable path. +The local-backend value proposition depends on the existing `read`/`grep` tools being able to inspect the returned local path, even when the spill directory is outside the session cwd. That holds today because the filesystem policy records observations and write guards but does not confine reads to the workspace. A future workspace-confinement policy must either allow local spill paths explicitly or use a non-file spill backend whose retrieval hint points at a supported reader. **Snapshot gap.** No ACP snapshot scenario covers the transcript-visible `web_fetch` spill notice yet. The ACP snapshot harness replays keyless and cannot hit the live web, and a `web_fetch` spill requires a real over-cap HTTP body; a deterministic scenario would need a seeded loopback fetch target the replay tree does not currently wire (the examples do not load `tool-web` at all). The behavior is covered instead by the `dsh-tool-web` integration test against a loopback server. Closing the gap is follow-up work: wire `tool-web` + a seeded fetch target into the ACP example, then record a `web-fetch-spill` scenario. @@ -182,7 +180,7 @@ The policy can become too large if it starts owning tool-specific semantics. It **Require each tool to opt in with a retention declaration.** Rejected for v1: the goal is a default behavior similar to Claude Code's generic tool-result persistence. A single `maxInlineBytes` deployment knob is enough to prove the shape. -**Make `tool-results` a broad tool-result platform.** Rejected: a broad package name invites retention policy, result replacement, preview wording, search, and early spill into one seam. The shared storage part is smaller: save text and return a path. +**Make `tool-results` a broad tool-result platform.** Rejected: a broad package name invites retention policy, result replacement, preview wording, search, and early spill into one seam. The shared storage part is smaller: save text and return a locator plus retrieval hint. **Use `ctx.fs.writeText` or the model-facing `write` tool.** Rejected: workspace filesystem writes carry project-file semantics, write/edit policy, observation state, and user-facing side effects. Spill files are runtime artifacts, not model-authored workspace edits. The existing `read` tool may inspect them later, but creation belongs to the runtime spill seam. diff --git a/docs/rfc/implemented/feature/2026-07-09-bash-backed-grep-glob-discovery.md b/docs/rfc/implemented/feature/2026-07-09-bash-backed-grep-glob-discovery.md index 74648ab40d..bc8a452c91 100644 --- a/docs/rfc/implemented/feature/2026-07-09-bash-backed-grep-glob-discovery.md +++ b/docs/rfc/implemented/feature/2026-07-09-bash-backed-grep-glob-discovery.md @@ -16,7 +16,7 @@ The tools do not use `ctx.bash.start()` and do not create model-visible backgrou The tools align `path` with Claude Code's search tools while binding resolution to the bash workdir, not to `ctx.fs`. The tool derives the bash request workdir from `exec.agent?.session.header.cwd`, mirroring `dsh-tool-bash` and `dsh-tool-fs`; when no session cwd exists, it omits `request.workdir` so the bash implementation applies its configured cwd or process cwd through `resolve()`. For `grep`, `path` is an optional ripgrep target and may be a file or directory; omitted means the resolved bash workdir. For `glob`, `path` is an optional directory search root; omitted means the resolved bash workdir. Relative `path` values resolve against that workdir. Returned paths are displayed relative to the resolved bash workdir when possible and are intended to be follow-up-readable only in co-located deployments where the bash workdir and filesystem `read` root are the same workspace. v1 documents that deployment requirement but does not perform runtime cross-service validation. Remote or virtual filesystem search is deferred until there is a shared workspace/root contract or a provider-specific search backend. -The package does not inject `fs`. It injects `tools`, `systemPrompt`, and `bash`; it deliberately reads `spillFiles` with `ctx.get('spillFiles')` instead of static inject because formatted-result spill is optional. Existing `@deepseek-ai/dsh-tool-fs` deployments that only want `read` / `write` / `edit` do not need to load bash. +The package does not inject `fs`. It injects `tools`, `systemPrompt`, and `bash`; it deliberately reads `spillStore` with `ctx.get('spillStore')` instead of static inject because formatted-result spill is optional. Existing `@deepseek-ai/dsh-tool-fs` deployments that only want `read` / `write` / `edit` do not need to load bash. ### Package shape @@ -63,9 +63,9 @@ Routine budgets stay out of the model-facing schema. `@deepseek-ai/dsh-tool-fs-s | `rawOutputMaxBytes` | `20000000` | Max complete raw `rg` stdout the tool will parse; matches Claude Code's ripgrep raw buffer. | | `timeoutMs` | `30000` | Tool-call timeout attached to both tool definitions and enforced by `@deepseek-ai/dsh-timeout-policy`. | -`globMaxResults` and `grepMaxMatches` use `ItemRetainer({ kind: 'head' })`. `grepMaxLineBytes` uses `TextRetainer({ kind: 'head', maxBytes: grepMaxLineBytes })` for each matched line so preview cuts preserve UTF-8 boundaries. This follows the [tool result retention library](../../implemented/architecture/2026-07-06-tool-result-retention-library.md) mapping for discovery items: collect the complete result, retain head items inline, and keep path mapping, grouping, and per-line preview outside the retainer. `grep` does not expose `case_insensitive`, `head_limit`, `offset`, `count`, multiline, context lines, output modes, or file type filters in v1. A model that needs surrounding context reads the matched file with `read`; a model that needs later results reads the formatted spill file with `read offset/limit`. +`globMaxResults` and `grepMaxMatches` use `ItemRetainer({ kind: 'head' })`. `grepMaxLineBytes` uses `TextRetainer({ kind: 'head', maxBytes: grepMaxLineBytes })` for each matched line so preview cuts preserve UTF-8 boundaries. This follows the [tool result retention library](../../implemented/architecture/2026-07-06-tool-result-retention-library.md) mapping for discovery items: collect the complete result, retain head items inline, and keep path mapping, grouping, and per-line preview outside the retainer. `grep` does not expose `case_insensitive`, `head_limit`, `offset`, `count`, multiline, context lines, output modes, or file type filters in v1. A model that needs surrounding context reads the matched file with `read`; a model that needs later results follows the returned spill locator's retrieval hint. -The Claude Code values are reference points for the two-layer budget, not model-facing schema precedent. Its dedicated search tools buffer raw ripgrep output up to 20 MB for internal processing, use a 20-second ripgrep timeout on non-WSL platforms (60 seconds on WSL), then apply search-specific caps before the model sees a result: `GrepTool` defaults to `head_limit = 250` and persists formatted results above 20,000 characters, while `GlobTool` defaults to 100 paths and persists formatted results above 100,000 characters. This RFC mirrors the raw-buffer and inline-count defaults, chooses a 30-second default search timeout, and uses this harness's `ctx.spillFiles.saveText()` path for formatted-result recovery. +The Claude Code values are reference points for the two-layer budget, not model-facing schema precedent. Its dedicated search tools buffer raw ripgrep output up to 20 MB for internal processing, use a 20-second ripgrep timeout on non-WSL platforms (60 seconds on WSL), then apply search-specific caps before the model sees a result: `GrepTool` defaults to `head_limit = 250` and persists formatted results above 20,000 characters, while `GlobTool` defaults to 100 paths and persists formatted results above 100,000 characters. This RFC mirrors the raw-buffer and inline-count defaults, chooses a 30-second default search timeout, and uses this harness's `ctx.spillStore.saveText()` path for formatted-result recovery. The `path` field follows the same split as Claude Code: `grep.path` is a file-or-directory ripgrep target, while `glob.path` is a directory search root. v1 does not expose a separate cwd/workdir argument on these tools. @@ -73,9 +73,9 @@ The `path` field follows the same split as Claude Code: `grep.path` is a file-or ### Execution -`glob` builds a fixed `rg --files` command rooted at the resolved directory search root (`path` when supplied, else the bash workdir): `rg --files --glob --sort=modified --no-ignore --hidden`, plus VCS metadata excludes for `.git`, `.svn`, `.hg`, `.bzr`, `.jj`, and `.sl`. This aligns with Claude Code on hidden/ignored-file discovery and modified-time ordering while keeping VCS internals out of broad searches. The tool parses one path per line, maps results back to paths relative to the bash workdir when possible, pushes each path into `ItemRetainer({ kind: 'head', maxItems: globMaxResults })`, and formats the full sorted path list for a spill file when the retained result is capped. +`glob` builds a fixed `rg --files` command rooted at the resolved directory search root (`path` when supplied, else the bash workdir): `rg --files --glob --sort=modified --no-ignore --hidden`, plus VCS metadata excludes for `.git`, `.svn`, `.hg`, `.bzr`, `.jj`, and `.sl`. This aligns with Claude Code on hidden/ignored-file discovery and modified-time ordering while keeping VCS internals out of broad searches. The tool parses one path per line, maps results back to paths relative to the bash workdir when possible, pushes each path into `ItemRetainer({ kind: 'head', maxItems: globMaxResults })`, and formats the full sorted path list for a spill artifact when the retained result is capped. -`grep` builds a fixed line-oriented `rg --json` command against the supplied file/directory target (`path` when supplied, else the bash workdir) so file path, line number, and line text are parsed without colon-splitting ambiguity. It consumes `match` records, treats malformed JSON or malformed match records as `SEARCH_FAILED`, maps result paths relative to the bash workdir when possible, applies per-line preview retention with `grepMaxLineBytes`, pushes each match into `ItemRetainer({ kind: 'head', maxItems: grepMaxMatches })`, then groups only the retained preview matches by file for inline output. The spill file stores the full formatted match list, not only the omitted tail, so `read offset/limit` works against the same logical result the model saw. +`grep` builds a fixed line-oriented `rg --json` command against the supplied file/directory target (`path` when supplied, else the bash workdir) so file path, line number, and line text are parsed without colon-splitting ambiguity. It consumes `match` records, treats malformed JSON or malformed match records as `SEARCH_FAILED`, maps result paths relative to the bash workdir when possible, applies per-line preview retention with `grepMaxLineBytes`, pushes each match into `ItemRetainer({ kind: 'head', maxItems: grepMaxMatches })`, then groups only the retained preview matches by file for inline output. The spill artifact stores the full formatted match list, not only the omitted tail, so the retrieval hint points at the same logical result the model saw. Raw `rg` stdout is an internal transport detail. The tool requests `stdoutMaxBytes: rawOutputMaxBytes` through `ctx.bash.resolve()` and parses `stdout.text` only when the executor returns untruncated stdout within that cap. If stdout is larger than `rawOutputMaxBytes`, or the executor still returns `stdout.truncated`, the tool fails with a clear search error telling the model to narrow `pattern`, `path`, or `include`. The tool never exposes raw `rg` output or bash raw spill paths to the model. @@ -87,13 +87,13 @@ Search failures use a package-owned `HarnessError` subclass with `SEARCH_*` code ### Formatted result spill -`ctx.spillFiles` is optional and used only for model-facing formatted results. This is the first tool-owned spill call pattern in the codebase, and it is intentional because search retention is item-level policy: `globMaxResults` caps paths and `grepMaxMatches` caps matches while the tool still holds the complete logical result. The generic `dsh-spill-policy` caps final text bytes on `tools/post-execute`; by then a search tool would already have omitted later paths or matches, so the policy cannot recover them. +`ctx.spillStore` is optional and used only for model-facing formatted results. This is the first tool-owned spill call pattern in the codebase, and it is intentional because search retention is item-level policy: `globMaxResults` caps paths and `grepMaxMatches` caps matches while the tool still holds the complete logical result. The generic `dsh-spill-policy` caps final text bytes on `tools/post-execute`; by then a search tool would already have omitted later paths or matches, so the policy cannot recover them. -When a search produces more logical results than the inline cap and `ctx.spillFiles` is present, the tool saves the complete formatted result with `saveText()`. The spill owner is the calling agent's session header id (`exec.agent?.session.header.id`); without that owner, the search keeps the inline result and reports that the complete result could not be saved. The spill source is the tool execution identity: `{ toolName: exec.name, callId: exec.callId, label: 'result' }`. The suggested filenames are `grep-results.txt` and `glob-results.txt`; the spill backend still sanitizes them as hints, never paths. +When a search produces more logical results than the inline cap and `ctx.spillStore` is present, the tool saves the complete formatted result with `saveText()`. The spill owner is the calling agent's session header id (`exec.agent?.session.header.id`); without that owner, the search keeps the inline result and reports that the complete result could not be saved. The spill source is the tool execution identity: `{ toolName: exec.name, callId: exec.callId, label: 'result' }`. The suggested filenames are `grep-results.txt` and `glob-results.txt`; the spill backend still treats them as hints, never paths. When spill storage is absent, the call has no session owner, or saving fails, the tool still returns the inline page and a footer explaining that the complete result could not be saved. Search success must not turn into an `isError` result solely because formatted-result spill storage is unavailable. -The bash raw output stream and the formatted search spill file are different artifacts. Raw `rg` stdout is parsed only in memory within the requested bash stdout cap; the formatted spill file is the stable model-facing recovery path produced by `ctx.spillFiles.saveText()`. +The bash raw output stream and the formatted search spill artifact are different artifacts. Raw `rg` stdout is parsed only in memory within the requested bash stdout cap; the formatted spill artifact is the stable model-facing recovery locator produced by `ctx.spillStore.saveText()`. ### Result shape @@ -102,7 +102,7 @@ A capped `glob` result with successful formatted spill returns the inline page a ```text -(Showing N of M paths. Full sorted result saved to: /.../session-abc123/9f8e7d-glob-results.txt. Use read with offset/limit to inspect it.) +(Showing N of M paths. Full sorted result stored at: /.../session-abc123/9f8e7d-glob-results.txt. Use read with offset/limit, or grep this path to search within it.) ``` A capped `grep` result with successful formatted spill returns grouped preview matches and a spill notice: @@ -113,10 +113,10 @@ Found N of M matches Line 12: ... -(Full grep result saved to: /.../session-abc123/9f8e7d-grep-results.txt. Use read with offset/limit to inspect it.) +(Full grep result stored at: /.../session-abc123/9f8e7d-grep-results.txt. Use read with offset/limit, or grep this path to search within it.) ``` -If the complete logical result fits under the inline cap, no formatted spill file is created. If the complete logical result is too large but formatted spill is unavailable, the footer says that the result was capped and the complete result could not be saved. The `truncated` / omitted count is a budget fact, not an incomplete-search fact; timeout, invalid regex, missing `rg`, inaccessible workdirs, raw-output overflow, binary skips, and parse failures stay in tool-domain error or incomplete fields. +If the complete logical result fits under the inline cap, no formatted spill artifact is created. If the complete logical result is too large but formatted spill is unavailable, the footer says that the result was capped and the complete result could not be saved. The `truncated` / omitted count is a budget fact, not an incomplete-search fact; timeout, invalid regex, missing `rg`, inaccessible workdirs, raw-output overflow, binary skips, and parse failures stay in tool-domain error or incomplete fields. ## Alternatives considered @@ -126,15 +126,15 @@ If the complete logical result fits under the inline cap, no formatted spill fil **Use `ctx.bash.start()` for streaming early stop.** Rejected: `start()` creates model-visible background task semantics: task ids, owner tokens, `bash_output`, `bash_kill`, completion notifications, and no built-in timeout. `grep` needs a foreground tool result, not a background bash workflow. If streaming search becomes necessary, the right abstraction is a foreground streaming process handle on the bash/process seam, not borrowing the public background-task API. -**Expose bash raw spill paths to the model.** Rejected: a bash raw spill path contains raw `rg` stdout (`rg --json` records for grep), not the stable formatted search result. Search parses raw stdout only as an internal transport; model recovery uses a formatted result saved through `ctx.spillFiles.saveText()`. +**Expose bash raw spill paths to the model.** Rejected: a bash raw spill path contains raw `rg` stdout (`rg --json` records for grep), not the stable formatted search result. Search parses raw stdout only as an internal transport; model recovery uses a formatted result saved through `ctx.spillStore.saveText()`. -**Add `spillFiles.saveFile()` for bash output normalization first.** Rejected for this RFC's v1: `saveFile()` would help a future bash normalization pass move existing executor spill files into session-scoped spill storage, but search only needs bounded in-memory raw `rg` stdout before producing the model-facing artifact. `saveText()` is sufficient for the formatted search result. +**Add `spillStore.saveFile()` for bash output normalization first.** Rejected for this RFC's v1: `saveFile()` would help a future bash normalization pass move existing executor spill files into session-scoped spill storage, but search only needs bounded in-memory raw `rg` stdout before producing the model-facing artifact. `saveText()` is sufficient for the formatted search result. **Rely on the generic `dsh-spill-policy`.** Rejected: generic post-execute spill sees only the final tool result. If `grep` / `glob` return the first page inline, the generic policy cannot recover omitted results. The search tools must save the complete formatted result themselves before returning the bounded model-facing text. -**Expose Claude Code's full `GrepTool` schema.** Rejected for v1: `output_mode`, context flags, multiline, `head_limit`, `offset`, `case_insensitive`, and type filters make the model-facing surface into a ripgrep wrapper. This harness keeps routine budgets and continuation mechanics in deployment policy and spill files. +**Expose Claude Code's full `GrepTool` schema.** Rejected for v1: `output_mode`, context flags, multiline, `head_limit`, `offset`, `case_insensitive`, and type filters make the model-facing surface into a ripgrep wrapper. This harness keeps routine budgets and continuation mechanics in deployment policy and spill artifacts. -**Keep early-stop search and skip formatted spill files.** Rejected for this proposal: early stop is more efficient but gives the model no path to inspect later results. The chosen v1 optimizes result recoverability and implementation simplicity, with `timeoutMs`, `rawOutputMaxBytes`, bash backend caps, and formatted spill files as safety backstops. +**Keep early-stop search and skip formatted spill artifacts.** Rejected for this proposal: early stop is more efficient but gives the model no path to inspect later results. The chosen v1 optimizes result recoverability and implementation simplicity, with `timeoutMs`, `rawOutputMaxBytes`, bash backend caps, and formatted spill artifacts as safety backstops. **Expand the bash seam with a raw-output reader first.** Rejected: a portable `readRawOutput(ref, maxBytes)` API would add reference lifetime, permission, and backend storage semantics. A per-run `stdoutMaxBytes` request is the narrower seam: search either receives complete stdout within `rawOutputMaxBytes` or fails clearly. @@ -148,11 +148,11 @@ If the complete logical result fits under the inline cap, no formatted spill fil ## Consequences -- `glob` and `grep` are model-facing tools in `@deepseek-ai/dsh-tool-fs-search`, not `ctx.fs` provider methods and not part of the existing `@deepseek-ai/dsh-tool-fs` root plugin. The package injects `tools`, `systemPrompt`, and `bash`; it does not inject `fs`, and `ctx.spillFiles` stays optional via `ctx.get('spillFiles')`. +- `glob` and `grep` are model-facing tools in `@deepseek-ai/dsh-tool-fs-search`, not `ctx.fs` provider methods and not part of the existing `@deepseek-ai/dsh-tool-fs` root plugin. The package injects `tools`, `systemPrompt`, and `bash`; it does not inject `fs`, and `ctx.spillStore` stays optional via `ctx.get('spillStore')`. - The schemas are exactly `glob(pattern, path?)` and `grep(pattern, path?, include?)`; search caps and timeout are defaulted, validated Config fields (`globMaxResults`, `grepMaxMatches`, `grepMaxLineBytes`, `rawOutputMaxBytes`, `timeoutMs`). - The tools execute through `ctx.bash.resolve(request)` → `ctx.bash.run(spec)`, forward `exec.signal`, never call `ctx.bash.start()`, and never expose a bash task id. The bash request workdir comes from `exec.agent?.session.header.cwd` when available; the resolved `spec.workdir` drives execution and relative-path display. - The tools request `stdoutMaxBytes: rawOutputMaxBytes` from the bash seam, parse only untruncated stdout within that cap, and treat over-cap or still-truncated raw output as a clear search failure; raw `rg` output is never exposed to the model. -- Oversized complete formatted results are saved through `ctx.spillFiles.saveText()` when available while inline results stay bounded; spill failure, a missing backend, or a missing owner preserves the inline result and reports the unsaved remainder — never an `isError`. +- Oversized complete formatted results are saved through `ctx.spillStore.saveText()` when available while inline results stay bounded; spill failure, a missing backend, or a missing owner preserves the inline result and reports the unsaved remainder — never an `isError`. - The package README, the generated config catalog, and exported JSDoc document the Config fields and `SEARCH_*` codes; the coding-agent example ships the tools (the acp-agent tree waits on the snapshot re-record above); the fs group README records the co-located bash/filesystem deployment requirement. ## Risks @@ -163,4 +163,4 @@ Shell command construction is the sharpest safety edge. Because `ctx.bash` accep The v1 assumes a co-located bash/filesystem deployment. If bash searches one workspace and the `read` tool resolves paths against another, returned paths may not be follow-up-readable. The package documents this requirement but does not verify it at runtime. -Spill paths are local filesystem paths in v1. The formatted-result design works for local deployments where `read` can open spill files; remote or workspace-confined deployments need either an allowlist for spill paths or a future virtual spill URI bridge. +Spill locators are backend-owned. The current local backend returns local filesystem paths and works in deployments where `read`/`grep` can open those files; remote or workspace-confined deployments can use a backend whose locator and retrieval hint point at a supported retrieval mechanism. diff --git a/docs/tool-catalog.md b/docs/tool-catalog.md index bb5abbcbc5..7778ac1e26 100644 --- a/docs/tool-catalog.md +++ b/docs/tool-catalog.md @@ -20,7 +20,7 @@ This table connects model-visible tool names to the plugin package and service s | `@deepseek-ai/dsh-tool-bash` | `bash`, `bash_kill`, `bash_output` | `ctx.tools`, `ctx.bash` | `tool/call`, `tool/result`, `context/message via agent.inject() for background completion notices` | - | The bash/bash_output/bash_kill tools are model-facing consumers of the bash executor seam. | | `@deepseek-ai/dsh-tool-cordis` | `cordis_inspect`, `cordis_mount`, `cordis_unmount` | `ctx.tools` | `tool/call`, `tool/result`, `live plugin-tree mutations (mount/unmount)` | - | Ships in examples/cordis-agent only (a deliberate opt-in — mounted code gets the real ctx, see docs/rfc/implemented/feature/2026-07-08-self-referential-cordis-toolset.md). Plugins the model mounts may register ADDITIONAL model-visible tools at runtime; the request-header ToolsDelta logs those tool-set changes. | | `@deepseek-ai/dsh-tool-fs` | `edit`, `read`, `write` | `ctx.tools`, `ctx.fs`, `ctx.systemPrompt` | `tool/call`, `fs/write-intent or fs/edit-intent for mutations`, `fs/observed after successful file operations`, `tool/result` | - | The read-before-write/edit policy is added by `@deepseek-ai/dsh-fs-policy` (an `fs/*` event-gate plugin, no schema change); a deployment that loads these tools is expected to also load it. The tool schemas above are identical with or without the policy plugin. | -| `@deepseek-ai/dsh-tool-fs-search` | `glob`, `grep` | `ctx.tools`, `ctx.bash`, `ctx.systemPrompt` | `tool/call`, `tool/result` | - | glob and grep are bash-backed discovery tools: they run fixed ripgrep commands through ctx.bash as ordinary foreground calls (never background tasks). Capped results save the complete formatted list through the optional ctx.spillFiles backend; returned paths are follow-up-readable in co-located bash/filesystem deployments. | +| `@deepseek-ai/dsh-tool-fs-search` | `glob`, `grep` | `ctx.tools`, `ctx.bash`, `ctx.systemPrompt` | `tool/call`, `tool/result` | - | glob and grep are bash-backed discovery tools: they run fixed ripgrep commands through ctx.bash as ordinary foreground calls (never background tasks). Capped results save the complete formatted list through the optional ctx.spillStore backend; returned locators are follow-up-readable/searchable when the backend exposes local paths in co-located deployments. | | `@deepseek-ai/dsh-tool-skill` | `skill` | `ctx.tools`, `ctx.skills` | `tool/call`, `tool/result` | - | - | | `@deepseek-ai/dsh-tool-subagent` | `subagent` | `ctx.tools`, `ctx.subagents` | `tool/call`, `tool/result`, `child session events through the chosen provider` | `subagent`, `subagent_fork` | The registered tool name is the load-time `toolName` config (default `subagent`); the schema above is that default. The shipped example agents load this package once per subagent backend, so the model additionally sees `subagent_fork` (bound to the fork backend) with an identical schema — see `examples/coding-agent/cordis.yml` and `examples/acp-agent/cordis.yml`. | | `@deepseek-ai/dsh-tool-todo` | `todo_write` | `ctx.tools`, `owning Agent session` | `tool/call`, `todo/write`, `tool/result` | - | todo_write is session-owned state; UIs render the latest todo/write event as a checklist or ACP plan. | @@ -428,7 +428,7 @@ Search file contents with a ripgrep regular expression. Returns matching lines w Source: [`packages/fs/tool-fs-search/src/index.ts`](../packages/fs/tool-fs-search/src/index.ts) -glob and grep are bash-backed discovery tools: they run fixed ripgrep commands through ctx.bash as ordinary foreground calls (never background tasks). Capped results save the complete formatted list through the optional ctx.spillFiles backend; returned paths are follow-up-readable in co-located bash/filesystem deployments. +glob and grep are bash-backed discovery tools: they run fixed ripgrep commands through ctx.bash as ordinary foreground calls (never background tasks). Capped results save the complete formatted list through the optional ctx.spillStore backend; returned locators are follow-up-readable/searchable when the backend exposes local paths in co-located deployments. ## `@deepseek-ai/dsh-tool-skill` diff --git a/examples/acp-agent/tests/snapshots/bash-spill/session.jsonl b/examples/acp-agent/tests/snapshots/bash-spill/session.jsonl index aa60d5143e..bc839dd9e2 100644 --- a/examples/acp-agent/tests/snapshots/bash-spill/session.jsonl +++ b/examples/acp-agent/tests/snapshots/bash-spill/session.jsonl @@ -10,7 +10,7 @@ {"type":"assistant/chunk","seq":8,"time":0,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}} {"type":"assistant/message","seq":9,"time":0,"data":{"turn":1,"step":1,"content":[{"type":"tool-call","id":"call_spill","name":"bash","arguments":"{\"command\":\"node -e \\\"process.stdout.write('SPILL_START-' + 'x'.repeat(2000) + '-SPILL_END')\\\"\",\"description\":\"Print large deterministic output\"}"}],"usage":{"inputTokens":10,"outputTokens":5}},"sourceEventSeqs":[4,5,6,7,8],"surfaceOp":"append"} {"type":"tool/call","seq":10,"time":0,"data":{"turn":1,"step":1,"callId":"call_spill","name":"bash","arguments":"{\"command\":\"node -e \\\"process.stdout.write('SPILL_START-' + 'x'.repeat(2000) + '-SPILL_END')\\\"\",\"description\":\"Print large deterministic output\"}"}} -{"type":"tool/result","seq":11,"time":0,"data":{"turn":1,"step":1,"callId":"call_spill","content":[{"type":"text","text":"SPILL_START-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-SPILL_END\n\n(Omitted 1391 bytes. Full formatted result saved to: {{spillPath:bash.txt}}. Use read with offset/limit to inspect it.)"}],"isError":false},"sourceEventSeqs":[10],"surfaceOp":"append"} +{"type":"tool/result","seq":11,"time":0,"data":{"turn":1,"step":1,"callId":"call_spill","content":[{"type":"text","text":"SPILL_START-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-SPILL_END\n\n(Omitted 1417 bytes. Full formatted result stored at: {{spillLocator:bash.txt}}. Use read with offset/limit, or grep this path to search within it.)"}],"isError":false},"sourceEventSeqs":[10],"surfaceOp":"append"} {"type":"step/end","seq":12,"time":0,"data":{"turn":1,"step":1}} {"type":"step/start","seq":13,"time":0,"data":{"turn":1,"step":2}} {"type":"assistant/chunk","seq":14,"time":0,"data":{"turn":1,"step":2,"chunk":{"type":"block-start","index":0,"blockType":"text"}}} diff --git a/examples/acp-agent/tests/snapshots/bash-spill/stdout.golden.jsonl b/examples/acp-agent/tests/snapshots/bash-spill/stdout.golden.jsonl index df47558232..b3590d29bb 100644 --- a/examples/acp-agent/tests/snapshots/bash-spill/stdout.golden.jsonl +++ b/examples/acp-agent/tests/snapshots/bash-spill/stdout.golden.jsonl @@ -1,6 +1,6 @@ {"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentInfo":{"name":"deepseek-harness-acp","version":"0.0.1"},"agentCapabilities":{"loadSession":true,"promptCapabilities":{"image":false,"audio":false,"embeddedContext":false}},"authMethods":[]}} {"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}"}} {"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call","toolCallId":"call_spill","title":"node -e \"process.stdout.write('SPILL_START-' + 'x'.repeat(2000) + '-SPILL_END')\"","kind":"execute","status":"in_progress","rawInput":"node -e \"process.stdout.write('SPILL_START-' + 'x'.repeat(2000) + '-SPILL_END')\"","content":[{"type":"content","content":{"type":"text","text":"Print large deterministic output"}}]}}} -{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call_update","toolCallId":"call_spill","status":"completed","content":[{"type":"content","content":{"type":"text","text":"```console\nSPILL_START-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-SPILL_END\n\n(Omitted 1391 bytes. Full formatted result saved to: {{spillPath:bash.txt}}. Use read with offset/limit to inspect it.)\n```"}}]}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call_update","toolCallId":"call_spill","status":"completed","content":[{"type":"content","content":{"type":"text","text":"```console\nSPILL_START-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-SPILL_END\n\n(Omitted 1417 bytes. Full formatted result stored at: {{spillLocator:bash.txt}}. Use read with offset/limit, or grep this path to search within it.)\n```"}}]}}} {"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"DONE"}}}} {"jsonrpc":"2.0","id":3,"result":{"stopReason":"end_turn"}} diff --git a/examples/coding-agent/cordis.yml b/examples/coding-agent/cordis.yml index cdff07ae00..b23d785439 100644 --- a/examples/coding-agent/cordis.yml +++ b/examples/coding-agent/cordis.yml @@ -135,7 +135,7 @@ # Bash-backed discovery tools (glob/grep): fixed ripgrep commands through the # local bash executor above — not ctx.fs. Capped results save the complete -# formatted list through the spill backend below (ctx.spillFiles, optional). +# formatted list through the spill backend below (ctx.spillStore, optional). - id: tool-fs-search name: '@deepseek-ai/dsh-tool-fs-search' @@ -148,9 +148,9 @@ # Tool-output spill stack: a local backend that saves oversized tool text under # a private session-scoped dir, and the tools/post-execute policy that replaces -# an over-budget plain-text result with a preview + the spill path (the model -# reads the full result later). A leaf pair after the app (needs ctx.tools). The -# policy is a no-op until a tool returns more than maxInlineBytes of plain text. +# an over-budget plain-text result with a preview + the spill locator/retrieval +# hint. A leaf pair after the app (needs ctx.tools). The policy is a no-op until +# a tool returns more than maxInlineBytes of plain text. - id: spill-local name: '@deepseek-ai/dsh-spill-local' diff --git a/packages/cordis/tool-cordis/src/api-catalog.ts b/packages/cordis/tool-cordis/src/api-catalog.ts index 18cca1f382..e9f8b76bcc 100644 --- a/packages/cordis/tool-cordis/src/api-catalog.ts +++ b/packages/cordis/tool-cordis/src/api-catalog.ts @@ -173,7 +173,7 @@ export const SERVICE_API: readonly ServiceApiEntry[] = [ ], }, { - key: 'spillFiles', + key: 'spillStore', summary: 'Abstract spill storage service.', methods: [ 'abstract saveText(input: SaveTextSpill): Promise', @@ -818,17 +818,17 @@ export const TYPE_API: readonly TypeApiEntry[] = [ name: 'SkillSummary', declaration: 'export interface SkillSummary {\n name: string;\n description: string;\n whenToUse?: string;\n disableModelInvocation?: boolean;\n source: SkillSource;\n provider: string;\n resourceBase?: SkillResourceBase;\n}', }, + { + name: 'SpillLocator', + declaration: 'export type SpillLocator = Branded<\'SpillLocator\'>;', + }, { name: 'SpillOwner', declaration: 'export interface SpillOwner {\n sessionId: SessionId;\n}', }, - { - name: 'SpillPath', - declaration: 'export type SpillPath = Branded<\'SpillPath\'>;', - }, { name: 'SpillRef', - declaration: 'export interface SpillRef {\n path: SpillPath;\n bytes: number;\n}', + declaration: 'export interface SpillRef {\n locator: SpillLocator;\n bytes: number;\n retrievalHint: string;\n}', }, { name: 'SpillSource', diff --git a/packages/fs/tool-fs-search/README.md b/packages/fs/tool-fs-search/README.md index 469830683d..2f9df200de 100644 --- a/packages/fs/tool-fs-search/README.md +++ b/packages/fs/tool-fs-search/README.md @@ -1,13 +1,13 @@ # @deepseek-ai/dsh-tool-fs-search -The **model-facing filesystem discovery tools** — `glob`, `grep` — backed by the **bash executor seam**, not by `ctx.fs` provider methods. Each call assembles a fixed ripgrep command (every model-controlled value through one package-private shell-quoting helper), runs it via `ctx.bash.resolve(request)` → `ctx.bash.run(spec)` as an ordinary foreground tool call, parses the raw `rg` output, and returns a bounded, workdir-relative result. The package injects `tools`, `systemPrompt`, and `bash` — deliberately **not** `fs`; `ctx.spillFiles` is read opportunistically with `ctx.get()` because formatted-result spill is optional. +The **model-facing filesystem discovery tools** — `glob`, `grep` — backed by the **bash executor seam**, not by `ctx.fs` provider methods. Each call assembles a fixed ripgrep command (every model-controlled value through one package-private shell-quoting helper), runs it via `ctx.bash.resolve(request)` → `ctx.bash.run(spec)` as an ordinary foreground tool call, parses the raw `rg` output, and returns a bounded, workdir-relative result. The package injects `tools`, `systemPrompt`, and `bash` — deliberately **not** `fs`; `ctx.spillStore` is read opportunistically with `ctx.get()` because formatted-result spill is optional. ```ts ignore-check // Default deployment: a bash executor, then the discovery tools. await ctx.plugin(LocalBashExecutor, { cwd: process.cwd() }) // @deepseek-ai/dsh-bash-local await ctx.plugin(ToolFsSearch) // this package — registers glob/grep // Optional: a spill backend makes capped results fully recoverable. -await ctx.plugin(LocalSpillFiles) // @deepseek-ai/dsh-spill-local +await ctx.plugin(LocalSpillStore) // @deepseek-ai/dsh-spill-local ``` Why bash-backed: local workspace discovery is naturally a process-backed `rg` workflow, and putting search on `ctx.fs` would force every filesystem backend to grow a search API. The bash executor owns request defaulting/capping, subprocess execution, process-group termination, environment scrubbing, raw output capture, and backend substitution (local, sandboxed, remote); this package owns schemas, argument validation, shell quoting, parsing, retention, formatted-result spill, and timeout declaration. The tools never call `ctx.bash.start()` and never expose a bash task id — the call returns only after `rg` exits, times out, is aborted, or fails. @@ -22,8 +22,8 @@ All keys are optional; the defaults are the shipped search caps. | Key | Default | Meaning | |---|---|---| -| `globMaxResults` | `100` | Max paths one `glob` call retains inline (matches Claude Code's `GlobTool` limit); later paths go to the formatted spill file. | -| `grepMaxMatches` | `250` | Max flat matches one `grep` call retains inline (matches Claude Code's `GrepTool` `head_limit`); later matches go to the formatted spill file. | +| `globMaxResults` | `100` | Max paths one `glob` call retains inline (matches Claude Code's `GlobTool` limit); later paths go to the formatted spill artifact. | +| `grepMaxMatches` | `250` | Max flat matches one `grep` call retains inline (matches Claude Code's `GrepTool` `head_limit`); later matches go to the formatted spill artifact. | | `grepMaxLineBytes` | `2000` | Byte cap per matched-line preview; the cut preserves UTF-8 boundaries and is marked `(line truncated)`. | | `rawOutputMaxBytes` | `20000000` | Max complete raw `rg` stdout a search will parse (matches Claude Code's ripgrep raw buffer); larger raw output fails with `SEARCH_RAW_OUTPUT_OVERFLOW`. | | `timeoutMs` | `30000` | Cooperative tool-call budget attached to both tool definitions, enforced by `@deepseek-ai/dsh-timeout-policy` through `exec.signal`; the bash backend's own timeout stays a second safety cap. | @@ -35,11 +35,11 @@ All keys are optional; the defaults are the shipped search caps. | `glob` | `pattern`, `path?` | `rg --files --glob --sort=modified --no-ignore --hidden` plus VCS metadata excludes (`.git`, `.svn`, `.hg`, `.bzr`, `.jj`, `.sl`). `path` is an optional **directory** search root; omitted means the resolved bash workdir. Returns one path per line, modification-time ordered. | | `grep` | `pattern`, `path?`, `include?` | Line-oriented `rg --json` parse (no colon-splitting ambiguity). `pattern` is a ripgrep regex; `path` is an optional **file or directory** target; `include` is ONE positive glob filter — a comma-separated list or a negated (`!…`) value is rejected up front (brace alternation like `*.{ts,tsx}` is fine). Returns matches grouped by file as `Line N: `. | -Routine budgets stay out of the model-facing schema (no `head_limit`/`offset`/`case_insensitive`/output modes): a model that needs surrounding context reads the matched file with `read`; one that needs later results reads the formatted spill file with `read offset/limit`. +Routine budgets stay out of the model-facing schema (no `head_limit`/`offset`/`case_insensitive`/output modes): a model that needs surrounding context reads the matched file with `read`; one that needs later results follows the returned spill locator's retrieval hint. ## Two budgets, two artifacts -Raw `rg` stdout is an internal transport detail. Each search requests `stdoutMaxBytes: rawOutputMaxBytes` from the bash seam and parses only complete retained stdout; if the executor still returns `stdout.truncated`, the search fails with `SEARCH_RAW_OUTPUT_OVERFLOW` and tells the model to narrow the query. The model-facing recovery artifact is different: when a search yields more logical results than the inline cap, the tool saves the COMPLETE formatted result through `ctx.spillFiles.saveText()` (suggested names `glob-results.txt` / `grep-results.txt`, owner = the calling session, source = the tool execution identity) and appends a footer naming the saved path. This is the first tool-owned spill call in the codebase — deliberate, because retention here is item-level: the generic `@deepseek-ai/dsh-spill-policy` only sees the final text on `tools/post-execute`, by which point a capped search has already omitted later paths/matches. A missing spill backend, a call with no session owner, or a `saveText()` failure keeps the inline page and reports that the complete result could not be saved — never an `isError`. +Raw `rg` stdout is an internal transport detail. Each search requests `stdoutMaxBytes: rawOutputMaxBytes` from the bash seam and parses only complete retained stdout; if the executor still returns `stdout.truncated`, the search fails with `SEARCH_RAW_OUTPUT_OVERFLOW` and tells the model to narrow the query. The model-facing recovery artifact is different: when a search yields more logical results than the inline cap, the tool saves the COMPLETE formatted result through `ctx.spillStore.saveText()` (suggested names `glob-results.txt` / `grep-results.txt`, owner = the calling session, source = the tool execution identity) and appends a footer naming the returned locator and retrieval hint. This is the first tool-owned spill call in the codebase — deliberate, because retention here is item-level: the generic `@deepseek-ai/dsh-spill-policy` only sees the final text on `tools/post-execute`, by which point a capped search has already omitted later paths/matches. A missing spill backend, a call with no session owner, or a `saveText()` failure keeps the inline page and reports that the complete result could not be saved — never an `isError`. ## Errors diff --git a/packages/fs/tool-fs-search/src/glob.ts b/packages/fs/tool-fs-search/src/glob.ts index 09a3e1d9ce..a3e803fb50 100644 --- a/packages/fs/tool-fs-search/src/glob.ts +++ b/packages/fs/tool-fs-search/src/glob.ts @@ -15,6 +15,7 @@ import type { GenericCallView } from '@deepseek-ai/dsh-tools' import type { ContentBlock } from '@deepseek-ai/dsh-llm' import { ItemRetainer } from '@deepseek-ai/dsh-retention' import type { RetainedItems } from '@deepseek-ai/dsh-retention' +import type { SpillRef } from '@deepseek-ai/dsh-spill' import type {} from '@deepseek-ai/dsh-bash' import type {} from '@deepseek-ai/dsh-system-prompt' import { runRipgrep, toWorkdirRelative, trySaveFormattedResult } from './search-core.ts' @@ -100,18 +101,18 @@ export function buildGlobCommand(input: GlobInput): string { /** * Format the model-facing `glob` result: the retained paths, then — when the * result was capped — a footer carrying either the formatted-spill recovery - * path or the could-not-save explanation. The omitted count is a budget fact: + * locator or the could-not-save explanation. The omitted count is a budget fact: * the search itself completed. * * @param retained - the retention outcome over every discovered path. - * @param spillPath - the saved complete-result path, or `undefined` when unsaved. + * @param spillRef - the saved complete-result reference, or `undefined` when unsaved. * @returns the model-facing text. */ -export function formatGlobOutput(retained: RetainedItems, spillPath: string | undefined): string { +export function formatGlobOutput(retained: RetainedItems, spillRef: SpillRef | undefined): string { const body = retained.items.join('\n') if (!retained.truncated) return body - const recovery = spillPath !== undefined - ? `Full sorted result saved to: ${spillPath}. Use read with offset/limit to inspect it.` + const recovery = spillRef !== undefined + ? `Full sorted result stored at: ${spillRef.locator}. ${spillRef.retrievalHint}` : 'The complete result could not be saved; narrow pattern or path to see more.' return `${body}\n\n(Showing ${retained.kept} of ${retained.seen} paths. ${recovery})` } @@ -168,10 +169,10 @@ export function applyGlobTool(ctx: Context, caps: GlobToolCaps): void { // The complete sorted list is the recovery artifact; save it only when // the inline page omitted paths (an uncapped result needs no spill file). - const spillPath = retained.truncated + const spillRef = retained.truncated ? await trySaveFormattedResult(ctx, exec, 'glob-results.txt', all.join('\n')) : undefined - return [{ type: 'text', text: formatGlobOutput(retained, spillPath) }] + return [{ type: 'text', text: formatGlobOutput(retained, spillRef) }] }, presentCall: presentGlobCall, })) diff --git a/packages/fs/tool-fs-search/src/grep.ts b/packages/fs/tool-fs-search/src/grep.ts index e5e64e5222..3935513b73 100644 --- a/packages/fs/tool-fs-search/src/grep.ts +++ b/packages/fs/tool-fs-search/src/grep.ts @@ -16,6 +16,7 @@ import type { GenericCallView } from '@deepseek-ai/dsh-tools' import type { ContentBlock } from '@deepseek-ai/dsh-llm' import { ItemRetainer, TextRetainer } from '@deepseek-ai/dsh-retention' import type { RetainedItems } from '@deepseek-ai/dsh-retention' +import type { SpillRef } from '@deepseek-ai/dsh-spill' import type {} from '@deepseek-ai/dsh-bash' import type {} from '@deepseek-ai/dsh-system-prompt' import { SearchError, runRipgrep, toWorkdirRelative, trySaveFormattedResult } from './search-core.ts' @@ -221,21 +222,21 @@ export function formatGrepMatches(matches: GrepMatch[]): string { /** * Format the model-facing `grep` result: a found-count header, the retained * matches grouped by file, then — when the result was capped — a footer - * carrying either the formatted-spill recovery path or the could-not-save + * carrying either the formatted-spill recovery locator or the could-not-save * explanation. The omitted count is a budget fact: the search itself completed. * * @param retained - the retention outcome over every parsed match. - * @param spillPath - the saved complete-result path, or `undefined` when unsaved. + * @param spillRef - the saved complete-result reference, or `undefined` when unsaved. * @returns the model-facing text. */ -export function formatGrepOutput(retained: RetainedItems, spillPath: string | undefined): string { +export function formatGrepOutput(retained: RetainedItems, spillRef: SpillRef | undefined): string { const header = retained.truncated ? `Found ${retained.kept} of ${retained.seen} matches` : `Found ${retained.seen} ${matchNoun(retained.seen)}` const body = formatGrepMatches(retained.items) if (!retained.truncated) return `${header}\n\n${body}` - const recovery = spillPath !== undefined - ? `Full grep result saved to: ${spillPath}. Use read with offset/limit to inspect it.` + const recovery = spillRef !== undefined + ? `Full grep result stored at: ${spillRef.locator}. ${spillRef.retrievalHint}` : 'The complete result could not be saved; narrow pattern, path, or include to see more.' return `${header}\n\n${body}\n\n(${recovery})` } @@ -299,7 +300,7 @@ export function applyGrepTool(ctx: Context, caps: GrepToolCaps): void { // The spill file stores the FULL formatted match list (same grouped, // per-line-previewed shape the model saw), so read offset/limit pages the // same logical result; save only when the inline page omitted matches. - const spillPath = retained.truncated + const spillRef = retained.truncated ? await trySaveFormattedResult( ctx, exec, @@ -307,7 +308,7 @@ export function applyGrepTool(ctx: Context, caps: GrepToolCaps): void { `Found ${all.length} ${matchNoun(all.length)}\n\n${formatGrepMatches(all)}`, ) : undefined - return [{ type: 'text', text: formatGrepOutput(retained, spillPath) }] + return [{ type: 'text', text: formatGrepOutput(retained, spillRef) }] }, presentCall: presentGrepCall, })) diff --git a/packages/fs/tool-fs-search/src/index.ts b/packages/fs/tool-fs-search/src/index.ts index 1fec6a999e..8c33d5770a 100644 --- a/packages/fs/tool-fs-search/src/index.ts +++ b/packages/fs/tool-fs-search/src/index.ts @@ -13,7 +13,7 @@ * bash executor owns request defaulting/capping, subprocess execution, * process-group termination, environment scrubbing, raw output capture, and * backend substitution. The package injects `tools`, `systemPrompt`, and - * `bash` — deliberately NOT `fs`, and `ctx.spillFiles` is read opportunistically + * `bash` — deliberately NOT `fs`, and `ctx.spillStore` is read opportunistically * with `ctx.get()` because formatted-result spill is optional. * * Returned paths are displayed relative to the resolved bash workdir and are @@ -52,7 +52,7 @@ export { singleQuote } from './shell-quote.ts' /** Cordis plugin name used by loader diagnostics. */ export const name = 'tool-fs-search' -/** Services required by the search tool suite (`spillFiles` is optional, read via `ctx.get()`). */ +/** Services required by the search tool suite (`spillStore` is optional, read via `ctx.get()`). */ export const inject = ['tools', 'systemPrompt', 'bash'] /** Plugin config (all optional — `Config` supplies the defaults). */ diff --git a/packages/fs/tool-fs-search/src/search-core.ts b/packages/fs/tool-fs-search/src/search-core.ts index 233c1e78d4..0682c86e35 100644 --- a/packages/fs/tool-fs-search/src/search-core.ts +++ b/packages/fs/tool-fs-search/src/search-core.ts @@ -10,7 +10,7 @@ * detail: the tools request a per-run stdout capture budget from the bash seam, * parse only complete in-memory stdout within `rawOutputMaxBytes`, and never * read executor spill files. The model-facing recovery artifact is the - * formatted result saved through `ctx.spillFiles.saveText()` + * formatted result saved through `ctx.spillStore.saveText()` * ({@link trySaveFormattedResult}). * * @module @deepseek-ai/dsh-tool-fs-search/search-core @@ -20,7 +20,7 @@ import { isAbsolute, relative, sep } from 'node:path' import type { Context } from 'cordis' import { HarnessError } from '@deepseek-ai/dsh-llm' import type { BashRunResult, CollectedOutput } from '@deepseek-ai/dsh-bash' -import type { SaveTextSpill } from '@deepseek-ai/dsh-spill' +import type { SaveTextSpill, SpillRef } from '@deepseek-ai/dsh-spill' import type { ToolExecution } from '@deepseek-ai/dsh-tools' /** @@ -214,8 +214,8 @@ export function toWorkdirRelative(path: string, workdir: string): string { /** * Best-effort save of one COMPLETE formatted search result through - * `ctx.spillFiles.saveText()` — the model-facing recovery path for a capped - * result. `spillFiles` is read with `ctx.get()` (not static inject) because + * `ctx.spillStore.saveText()` — the model-facing recovery path for a capped + * result. `spillStore` is read with `ctx.get()` (not static inject) because * formatted-result spill is optional; the spill owner is the calling agent's * session header id and the source is the tool execution identity. A missing * backend, a call with no session owner, or a `saveText()` rejection logs a @@ -223,26 +223,26 @@ export function toWorkdirRelative(path: string, workdir: string): string { * reports that the complete result could not be saved; search success never * turns into `isError` because spill storage is unavailable. * - * @param ctx - the plugin context; `spillFiles` is looked up opportunistically. + * @param ctx - the plugin context; `spillStore` is looked up opportunistically. * @param exec - the tool-execution context; supplies the owning session, tool name, and call id. * @param suggestedName - the backend-sanitized filename hint (e.g. `grep-results.txt`). * @param content - the complete formatted result to persist. - * @returns the saved spill path, or `undefined` when the result could not be saved. + * @returns the saved spill reference, or `undefined` when the result could not be saved. */ export async function trySaveFormattedResult( ctx: Context, exec: ToolExecution, suggestedName: string, content: string, -): Promise { +): Promise { const sessionId = exec.agent?.session.header.id if (sessionId === undefined) { ctx.logger.warn(`tool-fs-search: no session owner for ${exec.name} result; complete result not saved`) return undefined } - const spillFiles = ctx.get('spillFiles') - if (!spillFiles) { - ctx.logger.warn(`tool-fs-search: no ctx.spillFiles backend loaded; complete ${exec.name} result not saved`) + const spillStore = ctx.get('spillStore') + if (!spillStore) { + ctx.logger.warn(`tool-fs-search: no ctx.spillStore backend loaded; complete ${exec.name} result not saved`) return undefined } const save: SaveTextSpill = { @@ -252,8 +252,7 @@ export async function trySaveFormattedResult( content, } try { - const { path } = await spillFiles.saveText(save) - return path + return await spillStore.saveText(save) } catch (error: unknown) { // Best-effort: a storage failure must never fail the search or hide the // inline result — the footer reports the unsaved remainder instead. diff --git a/packages/fs/tool-fs-search/tests/tools.spec.ts b/packages/fs/tool-fs-search/tests/tools.spec.ts index 7e7199c1be..9131940de5 100644 --- a/packages/fs/tool-fs-search/tests/tools.spec.ts +++ b/packages/fs/tool-fs-search/tests/tools.spec.ts @@ -17,7 +17,7 @@ import SystemPrompt, { renderPrompt } from '@deepseek-ai/dsh-system-prompt' import ToolRegistry from '@deepseek-ai/dsh-tools' import { BashExecutor } from '@deepseek-ai/dsh-bash' import type { BashExecRequest, BashExecSpec, BashRunResult, BashTask, BashTaskId, BashTaskRead, OwnerToken } from '@deepseek-ai/dsh-bash' -import { SpillFiles, SpillPath } from '@deepseek-ai/dsh-spill' +import { SpillLocator, SpillStore } from '@deepseek-ai/dsh-spill' import type { SaveTextSpill, SpillRef } from '@deepseek-ai/dsh-spill' import * as ToolFsSearch from '@deepseek-ai/dsh-tool-fs-search' import { @@ -95,14 +95,18 @@ class FakeBash extends BashExecutor { } /** A recording spill backend; arm `failWith` to script a storage failure. */ -class FakeSpill extends SpillFiles { +class FakeSpill extends SpillStore { saves: SaveTextSpill[] = [] failWith?: Error override saveText(input: SaveTextSpill): Promise { if (this.failWith) return Promise.reject(this.failWith) this.saves.push(input) - return Promise.resolve({ path: SpillPath(`/spill/${input.suggestedName}`), bytes: Buffer.byteLength(input.content, 'utf8') }) + return Promise.resolve({ + locator: SpillLocator(`/spill/${input.suggestedName}`), + bytes: Buffer.byteLength(input.content, 'utf8'), + retrievalHint: 'Use the fake retrieval hint.', + }) } } @@ -119,7 +123,7 @@ async function setup(options: SetupOptions = {}) { if (options.spill === true) await ctx.plugin(FakeSpill) const fiber = await ctx.plugin(ToolFsSearch, options.config) const bash = ctx.bash as FakeBash - const spill = options.spill === true ? ctx.get('spillFiles') as FakeSpill : undefined + const spill = options.spill === true ? ctx.get('spillStore') as FakeSpill : undefined return { ctx, bash, spill, fiber } } @@ -445,12 +449,12 @@ describe('glob results', () => { expect(bash.specs[0]?.command).toContain("-- 'sub'") }) - it('caps at globMaxResults and saves the FULL sorted list through spillFiles', async () => { + it('caps at globMaxResults and saves the FULL sorted list through spillStore', async () => { const { ctx, bash, spill } = await setup({ config: { globMaxResults: 2 }, spill: true }) bash.handler = () => runResult('a.ts\nb.ts\nc.ts\nd.ts\n') const result = await call(ctx, 'glob', { pattern: '*.ts' }, { agent: agent('/w') }) expect(result.isError).toBe(false) - expect(text(result)).toBe('a.ts\nb.ts\n\n(Showing 2 of 4 paths. Full sorted result saved to: /spill/glob-results.txt. Use read with offset/limit to inspect it.)') + expect(text(result)).toBe('a.ts\nb.ts\n\n(Showing 2 of 4 paths. Full sorted result stored at: /spill/glob-results.txt. Use the fake retrieval hint.)') expect(spill?.saves).toHaveLength(1) expect(spill?.saves[0]).toMatchObject({ owner: { sessionId: 'session-1' }, @@ -543,7 +547,7 @@ describe('grep results', () => { '', ].join('\n')) const result = await call(ctx, 'grep', { pattern: 'e' }, { agent: agent('/w') }) - expect(text(result)).toBe('Found 2 of 3 matches\n\na.ts\nLine 1: one\nLine 2: two\n\n(Full grep result saved to: /spill/grep-results.txt. Use read with offset/limit to inspect it.)') + expect(text(result)).toBe('Found 2 of 3 matches\n\na.ts\nLine 1: one\nLine 2: two\n\n(Full grep result stored at: /spill/grep-results.txt. Use the fake retrieval hint.)') expect(spill?.saves[0]).toMatchObject({ source: { toolName: 'grep', label: 'result' }, suggestedName: 'grep-results.txt', diff --git a/packages/spill/README.md b/packages/spill/README.md index 35122275a3..7d54c91eb5 100644 --- a/packages/spill/README.md +++ b/packages/spill/README.md @@ -4,9 +4,9 @@ The tool-output spill capability seam: an abstract storage interface, a local fi | Package | Role | ctx key | |---|---|---| -| `spill/` | Abstract spill storage seam (`saveText` — persist oversized tool text to a session-scoped path) | `ctx.spillFiles` | -| `spill-local/` | Local-filesystem backend: private, session-scoped files with traversal-safe names | (registers on `ctx.spillFiles`) | -| `spill-policy/` | `tools/post-execute` policy: replaces oversized plain-text results with a preview + spill path | (no service surface) | +| `spill/` | Abstract spill storage seam (`saveText` — persist oversized tool text and return a locator + retrieval hint) | `ctx.spillStore` | +| `spill-local/` | Local-filesystem backend: private, session-scoped files with traversal-safe names | (registers on `ctx.spillStore`) | +| `spill-policy/` | `tools/post-execute` policy: replaces oversized plain-text results with a preview + spill locator | (no service surface) | The interface lives at `spill/spill/`. The split mirrors bash/fs: the seam owns storage only, `spill-local` owns the filesystem mechanics, and `spill-policy` owns WHEN to spill and the model-facing notice. Preview mechanics stay in [`util/retention`](../util/README.md) — the policy composes the two without either owning the other's job. diff --git a/packages/spill/spill-local/README.md b/packages/spill/spill-local/README.md index 1205b31eef..6860cc7638 100644 --- a/packages/spill/spill-local/README.md +++ b/packages/spill/spill-local/README.md @@ -1,6 +1,6 @@ # @deepseek-ai/dsh-spill-local -The **local-filesystem** implementation of the [`@deepseek-ai/dsh-spill`](../spill) storage seam. Registers as `ctx.spillFiles` and persists a tool's oversized text to a private, session-scoped file the model's `read` tool can open. +The **local-filesystem** implementation of the [`@deepseek-ai/dsh-spill`](../spill) storage seam. Registers as `ctx.spillStore` and persists a tool's oversized text to a private, session-scoped file; its locator is the file path and its retrieval hint tells the model to use `read` or `grep` on that path. ## Storage layout diff --git a/packages/spill/spill-local/src/index.ts b/packages/spill/spill-local/src/index.ts index 8aaf35e0a8..73e2cad851 100644 --- a/packages/spill/spill-local/src/index.ts +++ b/packages/spill/spill-local/src/index.ts @@ -1,9 +1,9 @@ /** - * `LocalSpillFiles`: the host-filesystem implementation of the + * `LocalSpillStore`: the host-filesystem implementation of the * `@deepseek-ai/dsh-spill` storage seam. Persists a tool's oversized text to a * private, session-scoped file (see `./store.ts` for the traversal-safe naming - * and exclusive owner-only write) and returns a path the local `read` tool can - * open. + * and exclusive owner-only write) and returns a path locator plus local + * read/grep retrieval guidance. * * @module @deepseek-ai/dsh-spill-local */ @@ -11,7 +11,7 @@ import { Context } from 'cordis' import { resolve } from 'node:path' import z from 'schemastery' -import { SpillFiles, SpillPath } from '@deepseek-ai/dsh-spill' +import { SpillLocator, SpillStore } from '@deepseek-ai/dsh-spill' import type { SaveTextSpill, SpillRef } from '@deepseek-ai/dsh-spill' import { privateRoot, saveTextFile } from './store.ts' @@ -34,7 +34,7 @@ export interface Config { * (0700) root — a spilled tool result must not be readable by other local users * or redirectable via a planted symlink. */ -export class LocalSpillFiles extends SpillFiles { +export class LocalSpillStore extends SpillStore { static Config: z = z.object({ root: z.string(), }) @@ -54,8 +54,12 @@ export class LocalSpillFiles extends SpillFiles { suggestedName: input.suggestedName, content: input.content, }) - return { path: SpillPath(saved.path), bytes: saved.bytes } + return { + locator: SpillLocator(saved.path), + bytes: saved.bytes, + retrievalHint: 'Use read with offset/limit, or grep this path to search within it.', + } } } -export default LocalSpillFiles +export default LocalSpillStore diff --git a/packages/spill/spill-local/tests/spill-local.spec.ts b/packages/spill/spill-local/tests/spill-local.spec.ts index 7357c1ede5..d73fca9fe3 100644 --- a/packages/spill/spill-local/tests/spill-local.spec.ts +++ b/packages/spill/spill-local/tests/spill-local.spec.ts @@ -1,9 +1,9 @@ /** * Tests for the LOCAL spill backend: `saveText` writes a session-scoped file and - * returns its path + byte length, filename sanitization neutralizes traversal, - * the configured `root` is honored (and the private default when omitted), and a - * storage failure rejects. The Cordis-free `store.ts` helpers are exercised - * directly for the naming/encoding edge cases. + * returns a locator + byte length + retrieval hint, filename sanitization + * neutralizes traversal, the configured `root` is honored (and the private + * default when omitted), and a storage failure rejects. The Cordis-free + * `store.ts` helpers are exercised directly for the naming/encoding edge cases. */ import { describe, expect, it, beforeEach, afterEach } from 'vitest' @@ -14,7 +14,7 @@ import { dirname, isAbsolute, join } from 'node:path' import { CallId } from '@deepseek-ai/dsh-llm' import { SessionId } from '@deepseek-ai/dsh-session' import type { SaveTextSpill } from '@deepseek-ai/dsh-spill' -import LocalSpillFiles, { encodeSegment, privateRoot, saveTextFile, sessionDir } from '@deepseek-ai/dsh-spill-local' +import LocalSpillStore, { encodeSegment, privateRoot, saveTextFile, sessionDir } from '@deepseek-ai/dsh-spill-local' let root: string @@ -106,33 +106,34 @@ describe('privateRoot', () => { }) }) -describe('LocalSpillFiles service', () => { - it('registers as ctx.spillFiles and saves under the configured root', async () => { +describe('LocalSpillStore service', () => { + it('registers as ctx.spillStore and saves under the configured root', async () => { const ctx = new Context() - await ctx.plugin(LocalSpillFiles, { root }) - const ref = await ctx.spillFiles.saveText(request()) - expect(dirname(ref.path)).toBe(sessionDir(root, 'sess-1')) - expect(readFileSync(ref.path, 'utf8')).toBe('the full body') + await ctx.plugin(LocalSpillStore, { root }) + const ref = await ctx.spillStore.saveText(request()) + expect(dirname(ref.locator)).toBe(sessionDir(root, 'sess-1')) + expect(readFileSync(ref.locator, 'utf8')).toBe('the full body') expect(ref.bytes).toBe(Buffer.byteLength('the full body', 'utf8')) + expect(ref.retrievalHint).toBe('Use read with offset/limit, or grep this path to search within it.') }) it('resolves a relative configured root to absolute', async () => { const ctx = new Context() - await ctx.plugin(LocalSpillFiles, { root: '.' }) - expect(isAbsolute((ctx.spillFiles as LocalSpillFiles).root)).toBe(true) + await ctx.plugin(LocalSpillStore, { root: '.' }) + expect(isAbsolute((ctx.spillStore as LocalSpillStore).root)).toBe(true) }) it('falls back to the private root when none is configured', async () => { const ctx = new Context() - await ctx.plugin(LocalSpillFiles, {}) - expect((ctx.spillFiles as LocalSpillFiles).root).toBe(privateRoot()) + await ctx.plugin(LocalSpillStore, {}) + expect((ctx.spillStore as LocalSpillStore).root).toBe(privateRoot()) }) it('rejects when the root is not writable (missing parent, exclusive open)', async () => { const ctx = new Context() // A file (not a dir) as the root makes mkdir under it fail — a real storage error. const filePath = (await saveTextFile({ root, sessionId: 's', suggestedName: 'f', content: 'x' })).path - await ctx.plugin(LocalSpillFiles, { root: filePath }) - await expect(ctx.spillFiles.saveText(request())).rejects.toThrow() + await ctx.plugin(LocalSpillStore, { root: filePath }) + await expect(ctx.spillStore.saveText(request())).rejects.toThrow() }) }) diff --git a/packages/spill/spill-policy/README.md b/packages/spill/spill-policy/README.md index fe128de9a0..c1592e926e 100644 --- a/packages/spill/spill-policy/README.md +++ b/packages/spill/spill-policy/README.md @@ -1,8 +1,8 @@ # @deepseek-ai/dsh-spill-policy -The **tool-result spill policy**: a `tools/post-execute` transformer that keeps oversized plain-text tool results out of the model's context. When a final result exceeds `maxInlineBytes`, it saves the FULL text to a session-scoped spill file via [`ctx.spillFiles`](../spill) and replaces the model-facing result with a bounded head/tail preview plus the spill path — the model reads the complete result later with the existing `read` tool. +The **tool-result spill policy**: a `tools/post-execute` transformer that keeps oversized plain-text tool results out of the model's context. When a final result exceeds `maxInlineBytes`, it saves the FULL text through [`ctx.spillStore`](../spill) and replaces the model-facing result with a bounded head/tail preview plus the backend's locator and retrieval hint. -This plugin registers **no service** and owns no storage or preview mechanics: preview is [`@deepseek-ai/dsh-retention`](../../util/retention) (`TextRetainer`), storage is `ctx.spillFiles`. It only decides WHEN to spill and composes the notice. +This plugin registers **no service** and owns no storage or preview mechanics: preview is [`@deepseek-ai/dsh-retention`](../../util/retention) (`TextRetainer`), storage is `ctx.spillStore`. It only decides WHEN to spill and composes the notice. ## Config @@ -13,7 +13,7 @@ This plugin registers **no service** and owns no storage or preview mechanics: p ## Behavior 1. Let the tool run (delegates via `next()`, so it bounds whatever a downstream hook accepted). -2. Skip `read` (avoids a `read → spill file → read again` loop) and any non-`accept` decision (a `block`'s corrective feedback passes through). +2. Skip `read` (avoids a `read → spill → read again` loop) and any non-`accept` decision (a `block`'s corrective feedback passes through). 3. Flatten the accepted content only when it is **plain text** (all `text` blocks); a result with any non-text block is left untouched. 4. If its UTF-8 size is `≤ maxInlineBytes`, leave it unchanged. 5. Otherwise save the full text and replace the result with a preview + this notice, sized so the whole replacement (preview + blank line + notice) stays within `maxInlineBytes` — the notice's byte cost is reserved out of the budget, so the preview shrinks to fit and the model-facing result never exceeds the cap: @@ -21,13 +21,13 @@ This plugin registers **no service** and owns no storage or preview mechanics: p ```text - (Omitted N bytes. Full formatted result saved to: /…/session-…/…-web_fetch.txt. Use read with offset/limit to inspect it.) + (Omitted N bytes. Full formatted result stored at: /…/session-…/…-web_fetch.txt. Use read with offset/limit, or grep this path to search within it.) ``` - When the notice alone fills the budget (a tiny cap or a long path) the preview is empty and only the notice is returned. If even that notice-only replacement would exceed `maxInlineBytes`, the policy keeps the inline result — it never emits a replacement over the cap (and a within-cap replacement is always smaller than the original, so this also means spilling never adds bytes). + When the notice alone fills the budget (a tiny cap or a long locator) the preview is empty and only the notice is returned. If even that notice-only replacement would exceed `maxInlineBytes`, the policy keeps the inline result — it never emits a replacement over the cap (and a within-cap replacement is always smaller than the original, so this also means spilling never adds bytes). -**Best-effort:** no session owner, no `ctx.spillFiles` backend, or a `saveText` rejection ⇒ the policy logs a warning and returns the original result. A spill failure never turns a successful call into an `isError` or hides the inline result. +**Best-effort:** no session owner, no `ctx.spillStore` backend, or a `saveText` rejection ⇒ the policy logs a warning and returns the original result. A spill failure never turns a successful call into an `isError` or hides the inline result. ## Scope -The policy sees only the FINAL formatted tool result — not a tool's internal resource. If a provider already truncated (e.g. `web-fetch-local.maxBodyChars`), the spill file holds the full formatted result the tool returned, not the full original source. Provider/resource caps stay mandatory and separate. Tool-owned early spill (bash streams, subagent rollouts) is future work — see the [tool output spill RFC](../../../docs/rfc/implemented/architecture/2026-07-08-tool-output-spill-files.md). +The policy sees only the FINAL formatted tool result — not a tool's internal resource. If a provider already truncated (e.g. `web-fetch-local.maxBodyChars`), the spill artifact holds the full formatted result the tool returned, not the full original source. Provider/resource caps stay mandatory and separate. Tool-owned early spill (bash streams, subagent rollouts) is future work — see the [tool output spill RFC](../../../docs/rfc/implemented/architecture/2026-07-08-tool-output-spill-files.md). diff --git a/packages/spill/spill-policy/src/index.ts b/packages/spill/spill-policy/src/index.ts index 0472bd9a8a..b1b16b7452 100644 --- a/packages/spill/spill-policy/src/index.ts +++ b/packages/spill/spill-policy/src/index.ts @@ -2,12 +2,12 @@ * The spill-policy PLUGIN: a `tools/post-execute` result transformer that keeps * oversized plain-text tool results out of the model's context. When a final * result's UTF-8 size exceeds `maxInlineBytes`, it saves the FULL text to a - * session-scoped spill file (`ctx.spillFiles`) and replaces the model-facing - * result with a bounded head/tail preview plus the spill path — the model reads - * the complete result later with the existing `read` tool. + * session-scoped spill artifact (`ctx.spillStore`) and replaces the + * model-facing result with a bounded head/tail preview plus the backend's + * locator and retrieval guidance. * * It registers NO service and owns NO storage or preview mechanics: preview is - * `@deepseek-ai/dsh-retention` (`TextRetainer`), storage is `ctx.spillFiles`. + * `@deepseek-ai/dsh-retention` (`TextRetainer`), storage is `ctx.spillStore`. * The policy only decides WHEN to spill and composes the notice. * * ## Deliberately narrow @@ -16,8 +16,8 @@ * - Plain-text results only: a result carrying any non-text block is left * untouched (the policy knows only the final formatted text, not tool * internals). - * - `read` is skipped to avoid a `read → spill file → read again` loop. - * - Best-effort: no session owner, no `ctx.spillFiles` backend, or a save + * - `read` is skipped to avoid a `read → spill → read again` loop. + * - Best-effort: no session owner, no `ctx.spillStore` backend, or a save * failure ⇒ log and return the original result. A spill failure must NEVER * turn a successful tool call into an `isError` or hide the inline result. * @@ -34,7 +34,7 @@ import z from 'schemastery' import type { ContentBlock } from '@deepseek-ai/dsh-llm' import { TextRetainer, describeOmitted } from '@deepseek-ai/dsh-retention' import type { Omitted } from '@deepseek-ai/dsh-retention' -import type { SaveTextSpill } from '@deepseek-ai/dsh-spill' +import type { SaveTextSpill, SpillRef } from '@deepseek-ai/dsh-spill' import type { SessionId } from '@deepseek-ai/dsh-session' import type { PostToolDecision, ToolExecution } from '@deepseek-ai/dsh-tools' import type { SpillPolicyExec } from './types.ts' @@ -86,10 +86,10 @@ function preview(text: string, budget: number): { text: string; omitted: Omitted return { text: kept.text, omitted: kept.omittedBytes } } -/** The spill-notice line for a given omission + path (no preview, no leading blank line). */ -function spillNotice(omitted: Omitted, spillPath: string): string { +/** The spill-notice line for a given omission + saved reference (no preview, no leading blank line). */ +function spillNotice(omitted: Omitted, ref: SpillRef): string { const omission = describeOmitted(omitted, 'bytes') - return `(${omission} Full formatted result saved to: ${spillPath}. Use read with offset/limit to inspect it.)` + return `(${omission} Full formatted result stored at: ${ref.locator}. ${ref.retrievalHint})` } export function apply(ctx: Context, config: Config): void { @@ -108,7 +108,7 @@ export function apply(ctx: Context, config: Config): void { // we bound whatever it accepted. A block passes through — spill only shapes // accepted plain-text results, never corrective feedback. const decision = await next() - // Skip `read` to avoid a read → spill file → read again loop. + // Skip `read` to avoid a read → spill → read again loop. if (decision.kind !== 'accept' || exec.name === 'read') return decision const content = decision.content ?? result.content @@ -122,9 +122,9 @@ export function apply(ctx: Context, config: Config): void { ctx.logger.warn(`spill-policy: no session owner for ${exec.name} result; keeping the inline result`) return decision } - const spillFiles = ctx.get('spillFiles') - if (!spillFiles) { - ctx.logger.warn('spill-policy: no ctx.spillFiles backend loaded; keeping the inline result') + const spillStore = ctx.get('spillStore') + if (!spillStore) { + ctx.logger.warn('spill-policy: no ctx.spillStore backend loaded; keeping the inline result') return decision } @@ -134,9 +134,9 @@ export function apply(ctx: Context, config: Config): void { suggestedName: `${exec.name}.txt`, content: text, } - let path: string + let ref: SpillRef try { - ({ path } = await spillFiles.saveText(save)) + ref = await spillStore.saveText(save) } catch (error: unknown) { // Best-effort: a storage failure (permissions, ENOSPC, backend down) must // never fail the call or hide the result — keep the original inline. @@ -152,10 +152,10 @@ export function apply(ctx: Context, config: Config): void { // count (the full byte total): its digit count bounds the real count's, so // the reserved size is a safe upper bound and the final notice is never // longer than what we reserved. `\n\n` is the 2-byte join. - const reserve = Buffer.byteLength(spillNotice({ kind: 'exact', count: totalBytes }, path), 'utf8') + 2 + const reserve = Buffer.byteLength(spillNotice({ kind: 'exact', count: totalBytes }, ref), 'utf8') + 2 const previewBudget = Math.max(0, maxInlineBytes - reserve) const { text: previewText, omitted } = preview(text, previewBudget) - const notice = spillNotice(omitted, path) + const notice = spillNotice(omitted, ref) const replacedText = previewText.length > 0 ? `${previewText}\n\n${notice}` : notice // Invariant: the policy NEVER emits a replacement larger than the cap. When // the notice alone exceeds maxInlineBytes (a tiny cap or a long spill root), diff --git a/packages/spill/spill-policy/src/types.ts b/packages/spill/spill-policy/src/types.ts index 032d0af550..3046e3efe5 100644 --- a/packages/spill/spill-policy/src/types.ts +++ b/packages/spill/spill-policy/src/types.ts @@ -1,6 +1,6 @@ /** * Vocabulary for the spill-policy plugin: the minimal structural view of a tool - * execution the policy needs to derive the owning session for a spill file. + * execution the policy needs to derive the owning session for a spill artifact. * * `@deepseek-ai/dsh-tools`' `ToolExecution` satisfies this shape, so the policy * reads `exec` straight through without importing `dsh-tools` or `dsh-agent`. diff --git a/packages/spill/spill-policy/tests/spill-policy.spec.ts b/packages/spill/spill-policy/tests/spill-policy.spec.ts index b0678c27c2..cd1cfb8356 100644 --- a/packages/spill/spill-policy/tests/spill-policy.spec.ts +++ b/packages/spill/spill-policy/tests/spill-policy.spec.ts @@ -2,7 +2,7 @@ * Tests for the spill-policy PLUGIN. It registers no service, only the * `tools/post-execute` transformer. We drive real tools through * `ctx.tools.execute(...)` and assert: disabled mode is a true no-op, an - * oversized plain-text result is spilled and replaced with a preview + path, + * oversized plain-text result is spilled and replaced with a preview + locator, * a small result and a non-text result pass through, `read` is skipped, and a * `saveText` failure / missing backend / missing owner all preserve the original * result without an `isError`. @@ -17,19 +17,23 @@ import { SessionId } from '@deepseek-ai/dsh-session' import SystemPrompt from '@deepseek-ai/dsh-system-prompt' import ToolRegistry, { defineTool } from '@deepseek-ai/dsh-tools' import type { ToolExecution } from '@deepseek-ai/dsh-tools' -import { SpillFiles, SpillPath } from '@deepseek-ai/dsh-spill' +import { SpillLocator, SpillStore } from '@deepseek-ai/dsh-spill' import type { SaveTextSpill, SpillRef } from '@deepseek-ai/dsh-spill' import * as SpillPolicy from '@deepseek-ai/dsh-spill-policy' /** A stub spill backend recording its saves; `fail` exercises the best-effort fallback. */ -class StubSpill extends SpillFiles { +class StubStore extends SpillStore { saves: SaveTextSpill[] = [] fail = false async saveText(input: SaveTextSpill): Promise { if (this.fail) throw new Error('disk full') this.saves.push(input) - return { path: SpillPath(`/spill/${input.suggestedName}`), bytes: Buffer.byteLength(input.content, 'utf8') } + return { + locator: SpillLocator(`/spill/${input.suggestedName}`), + bytes: Buffer.byteLength(input.content, 'utf8'), + retrievalHint: 'Use the stub retrieval path.', + } } } @@ -54,14 +58,14 @@ function exec(name: string, session = 's1'): ToolExecution { * Build a context with tools + the policy, and optionally a spill backend. * Returns the context and the backend handle (undefined when `withSpill` false). */ -async function setup(config: SpillPolicy.Config, withSpill = true): Promise<{ ctx: Context; spill?: StubSpill; fiber: Awaited> }> { +async function setup(config: SpillPolicy.Config, withSpill = true): Promise<{ ctx: Context; spill?: StubStore; fiber: Awaited> }> { const ctx = new Context() await ctx.plugin(SystemPrompt) await ctx.plugin(ToolRegistry) - let spill: StubSpill | undefined + let spill: StubStore | undefined if (withSpill) { - await ctx.plugin(StubSpill) - spill = ctx.spillFiles as StubSpill + await ctx.plugin(StubStore) + spill = ctx.spillStore as StubStore } const fiber = await ctx.plugin(SpillPolicy, config) return { ctx, fiber, ...spill ? { spill } : {} } @@ -108,7 +112,7 @@ describe('config validation', () => { }) describe('oversized plain-text replacement', () => { - it('spills the full text and replaces the result with a preview + path within the cap', async () => { + it('spills the full text and replaces the result with a preview + locator within the cap', async () => { const { ctx, spill } = await setup({ maxInlineBytes: 200 }) const body = 'HEAD'.repeat(200) + 'TAIL'.repeat(200) // 1600 bytes > 200 ctx.tools.register(textTool('big', body)) @@ -124,8 +128,8 @@ describe('oversized plain-text replacement', () => { const text = textOf(result.content) expect(text).not.toBe(body) expect(text.startsWith('HEAD')).toBe(true) - expect(text).toContain('Full formatted result saved to: /spill/big.txt') - expect(text).toContain('Use read with offset/limit') + expect(text).toContain('Full formatted result stored at: /spill/big.txt') + expect(text).toContain('Use the stub retrieval path.') expect(text).toContain('Omitted') // The replacement (preview + blank line + notice) stays within the cap and // is smaller than the original — the whole point of spilling. @@ -221,7 +225,7 @@ describe('composition', () => { ctx.tools.register(textTool('small', 'tiny')) const result = await ctx.tools.execute(exec('small')) expect(spill?.saves[0]?.content).toBe('z'.repeat(500)) - expect(textOf(result.content)).toContain('Full formatted result saved to') + expect(textOf(result.content)).toContain('Full formatted result stored at') }) it('preserves a downstream accept decision additionalContext when spilling', async () => { @@ -231,7 +235,7 @@ describe('composition', () => { ({ kind: 'accept', additionalContext: context })) ctx.tools.register(textTool('big', 'x'.repeat(1000))) const result = await ctx.tools.execute(exec('big')) - expect(textOf(result.content)).toContain('Full formatted result saved to') + expect(textOf(result.content)).toContain('Full formatted result stored at') expect(result.additionalContext).toEqual(context) }) }) @@ -259,7 +263,7 @@ describe('disposal (HMR safety)', () => { // Live: the listener spills and replaces. const before = await ctx.tools.execute(exec('big')) - expect(textOf(before.content)).toContain('Full formatted result saved to') + expect(textOf(before.content)).toContain('Full formatted result stored at') expect(spill?.saves).toHaveLength(1) // After disposal the listener is gone — the result passes through untouched diff --git a/packages/spill/spill/README.md b/packages/spill/spill/README.md index 50f115573b..f550e31d84 100644 --- a/packages/spill/spill/README.md +++ b/packages/spill/spill/README.md @@ -1,6 +1,6 @@ # @deepseek-ai/dsh-spill -The **spill storage seam**: an abstract `SpillFiles` service (`ctx.spillFiles`) defining WHAT a spill backend does — persist a tool's oversized text to a session-scoped path the model can later `read` — without saying HOW. +The **spill storage seam**: an abstract `SpillStore` service (`ctx.spillStore`) defining WHAT a spill backend does — persist a tool's oversized text and return a model-facing locator plus retrieval guidance — without saying HOW. This package is one third of the spill capability, split so each concern evolves (and swaps) independently: @@ -10,18 +10,18 @@ This package is one third of the spill capability, split so each concern evolves | `@deepseek-ai/dsh-spill-local` | an implementation: private session-scoped files on the host filesystem | | `@deepseek-ai/dsh-spill-policy` | the tool-result policy that spills oversized final results | -The split mirrors the bash/fs seams. A future remote or virtual backend (e.g. a `spill://…` URI plus a read-only bridge for ACP or remote environments) implements this interface without touching the policy plugin. +The split mirrors the bash/fs seams. A future remote or virtual backend (e.g. a `spill://…` URI, a database key, or a backend-specific retrieval tool) implements this interface without touching the policy plugin. -## Service API (`ctx.spillFiles`) +## Service API (`ctx.spillStore`) | Member | Semantics | |---|---| -| `saveText(input)` | Persist `input.content` verbatim to a session-scoped file; resolves with a `SpillRef` (path readable by the local `read` tool + exact bytes written). **Rejects on a real storage failure** (permissions, ENOSPC, backend unavailable) — the caller decides how to degrade. | +| `saveText(input)` | Persist `input.content` verbatim; resolves with a `SpillRef` (opaque locator, exact bytes written, and retrieval hint). **Rejects on a real storage failure** (permissions, ENOSPC, backend unavailable) — the caller decides how to degrade. | -Storage is scoped by the request's `owner` session; the backend chooses a private (not world-readable) location and a collision-free name derived from — never equal to — the caller's `suggestedName`. The seam owns storage only: NO retention policy (that is [`@deepseek-ai/dsh-retention`](../../util/retention)), NO tool-result replacement (that is `@deepseek-ai/dsh-spill-policy`), NO file inspection (the model uses the existing `read` tool on the returned path). +Storage is scoped by the request's `owner` session; the backend chooses its own private representation and may derive names from — never trust as a path — the caller's `suggestedName`. The seam owns storage only: NO retention policy (that is [`@deepseek-ai/dsh-retention`](../../util/retention)), NO tool-result replacement (that is `@deepseek-ai/dsh-spill-policy`), NO retrieval/search API (the backend's `retrievalHint` tells the model what to do with the locator). ## Vocabulary -`SaveTextSpill` (owner, source, suggestedName, content) is the request; `SpillRef` (path, bytes) is the result. `SpillPath` is [branded](../../util/brand) and rendered to the model as an ordinary path string in v1 — the brand records provenance (a runtime artifact, not a workspace file) so a future virtual backend can swap the path shape without a consumer change. `SpillOwner` scopes storage to a `SessionId`; unlike the bash executor's decoupled `OwnerToken`, spill is inherently session-scoped, so the seam imports `dsh-session`'s `SessionId` directly. `SpillSource` (toolName, callId, label) is descriptive provenance for the filename and future cleanup, not access control. See `src/types.ts` for the full contracts. +`SaveTextSpill` (owner, source, suggestedName, content) is the request; `SpillRef` (locator, bytes, retrievalHint) is the result. `SpillLocator` is [branded](../../util/brand) and rendered to the model as an opaque string — a local path for `dsh-spill-local`, but a future backend may return a URI, key, or command token without changing policy/tool consumers. `SpillOwner` scopes storage to a `SessionId`; unlike the bash executor's decoupled `OwnerToken`, spill is inherently session-scoped, so the seam imports `dsh-session`'s `SessionId` directly. `SpillSource` (toolName, callId, label) is descriptive provenance for backend naming and future cleanup, not access control. See `src/types.ts` for the full contracts. See the [tool output spill RFC](../../../docs/rfc/implemented/architecture/2026-07-08-tool-output-spill-files.md) for the design rationale, including why creation belongs to the runtime spill seam rather than the model-facing `write` tool. diff --git a/packages/spill/spill/package.json b/packages/spill/spill/package.json index 167c67183c..3103c9cd11 100644 --- a/packages/spill/spill/package.json +++ b/packages/spill/spill/package.json @@ -1,6 +1,6 @@ { "name": "@deepseek-ai/dsh-spill", - "description": "Abstract spill storage seam (ctx.spillFiles) for the DeepSeek Harness — save oversized tool text to a session-scoped path", + "description": "Abstract spill storage seam (ctx.spillStore) for the DeepSeek Harness — save oversized tool text and return a retrieval locator", "version": "0.0.1", "private": true, "type": "module", diff --git a/packages/spill/spill/src/index.ts b/packages/spill/spill/src/index.ts index 4c8fa37030..4c8826defb 100644 --- a/packages/spill/spill/src/index.ts +++ b/packages/spill/spill/src/index.ts @@ -1,16 +1,15 @@ /** - * The spill storage seam (`ctx.spillFiles`): an abstract service defining WHAT a - * spill backend does — persist a tool's oversized text to a session-scoped path - * the model can later `read` — without saying HOW. Implementations subclass - * {@link SpillFiles} and register as the `spillFiles` service; + * The spill storage seam (`ctx.spillStore`): an abstract service defining WHAT a + * spill backend does — persist a tool's oversized text and return a model-facing + * locator plus retrieval guidance — without saying HOW. Implementations + * subclass {@link SpillStore} and register as the `spillStore` service; * `@deepseek-ai/dsh-spill-local` (host filesystem) is the first. * * The seam is deliberately minimal: `saveText` and nothing else. It owns NO * retention policy (that is `@deepseek-ai/dsh-retention`), NO tool-result - * replacement (that is `@deepseek-ai/dsh-spill-policy`), and NO file inspection - * (the model uses the existing `read` tool on the returned path). A future - * remote/virtual backend may return a `spill://…` URI plus a read-only bridge; - * v1 keeps the path filesystem-shaped until such a backend exists. + * replacement (that is `@deepseek-ai/dsh-spill-policy`), and NO retrieval or + * search API. The backend supplies the locator and retrieval hint appropriate + * for its storage substrate. * * @module @deepseek-ai/dsh-spill */ @@ -18,24 +17,24 @@ import { Context, Service } from 'cordis' import type { SaveTextSpill, SpillRef } from './types.ts' -export { SpillPath } from './types.ts' +export { SpillLocator } from './types.ts' export type { SaveTextSpill, SpillOwner, SpillRef, SpillSource } from './types.ts' declare module 'cordis' { interface Context { - spillFiles: SpillFiles + spillStore: SpillStore } } /** * Abstract spill storage service. Subclass, implement {@link saveText}, and load - * the subclass as a plugin — it registers as `ctx.spillFiles` (one + * the subclass as a plugin — it registers as `ctx.spillStore` (one * implementation per context; loading a second throws, cordis' standard * duplicate-service behavior). * * Semantics every implementation must honor: - * - {@link saveText} persists the FULL `content` verbatim and returns a path - * the local `read` tool can open, plus the exact byte length written. + * - {@link saveText} persists the FULL `content` verbatim and returns an opaque + * locator, exact byte length, and model-facing retrieval guidance. * - Storage is scoped by the request's {@link SaveTextSpill.owner} session; the * backend chooses a private (not world-readable) location and a collision-free * name derived from — never equal to — the caller's `suggestedName`. @@ -43,18 +42,17 @@ declare module 'cordis' { * unavailable); the caller decides how to degrade (the spill policy treats a * rejection as best-effort and keeps the inline result). */ -export abstract class SpillFiles extends Service { +export abstract class SpillStore extends Service { constructor(ctx: Context) { - super(ctx, 'spillFiles') + super(ctx, 'spillStore') } /** - * Persist `input.content` to a session-scoped spill file. + * Persist `input.content` to a session-scoped spill artifact. * @param input - the owner, provenance, suggested name, and full text to save. - * @returns the saved file's {@link SpillRef} (path + bytes written); rejects on - * a storage failure. + * @returns the saved artifact's {@link SpillRef}; rejects on a storage failure. */ abstract saveText(input: SaveTextSpill): Promise } -export default SpillFiles +export default SpillStore diff --git a/packages/spill/spill/src/types.ts b/packages/spill/spill/src/types.ts index 28be96c738..5290a9738e 100644 --- a/packages/spill/spill/src/types.ts +++ b/packages/spill/spill/src/types.ts @@ -11,22 +11,20 @@ import type { CallId } from '@deepseek-ai/dsh-llm' import type { SessionId } from '@deepseek-ai/dsh-session' /** - * A local filesystem path produced by the spill seam, intended for the model's - * `read` tool. The brand records that the path came from {@link SpillFiles.saveText} - * (a runtime artifact, not a workspace file); it is still rendered to the model - * as an ordinary path string in v1. A future remote/virtual backend may replace - * this with a `spill://…` URI, so consumers treat it as opaque. + * Opaque model-facing handle for one spilled artifact. A local backend may use a + * filesystem path; a remote or database backend may use a URI or key. Consumers + * render it with {@link SpillRef.retrievalHint}, but do not parse it. */ -export type SpillPath = Branded<'SpillPath'> +export type SpillLocator = Branded<'SpillLocator'> /** - * Brand a string as a {@link SpillPath}. + * Brand a string as a {@link SpillLocator}. * - * @param path The backend-produced path string to brand. - * @returns The branded spill path. + * @param locator The backend-produced locator string to brand. + * @returns The branded spill locator. */ -export function SpillPath(path: string): SpillPath { - return path as SpillPath +export function SpillLocator(locator: string): SpillLocator { + return locator as SpillLocator } /** @@ -53,7 +51,7 @@ export interface SpillSource { label: string } -/** One request to persist text to a spill file. */ +/** One request to persist text to a spill artifact. */ export interface SaveTextSpill { owner: SpillOwner source: SpillSource @@ -66,8 +64,9 @@ export interface SaveTextSpill { content: string } -/** A saved spill file: its path plus the byte length written. */ +/** A saved spill artifact: its locator, byte length, and backend-specific retrieval guidance. */ export interface SpillRef { - path: SpillPath + locator: SpillLocator bytes: number + retrievalHint: string } diff --git a/packages/spill/spill/tests/service.spec.ts b/packages/spill/spill/tests/service.spec.ts index 271725442b..ddbc4086e1 100644 --- a/packages/spill/spill/tests/service.spec.ts +++ b/packages/spill/spill/tests/service.spec.ts @@ -1,6 +1,6 @@ /** * Tests for the spill seam INTERFACE: a minimal concrete subclass registers as - * `ctx.spillFiles`, a second load throws (duplicate service), and disposal + * `ctx.spillStore`, a second load throws (duplicate service), and disposal * releases the service. The storage behavior is the implementation's concern * (`@deepseek-ai/dsh-spill-local`); here we only pin the seam contract. */ @@ -9,16 +9,20 @@ import { describe, expect, it } from 'vitest' import { Context } from 'cordis' import { CallId } from '@deepseek-ai/dsh-llm' import { SessionId } from '@deepseek-ai/dsh-session' -import { SpillFiles, SpillPath } from '@deepseek-ai/dsh-spill' +import { SpillLocator, SpillStore } from '@deepseek-ai/dsh-spill' import type { SaveTextSpill, SpillRef } from '@deepseek-ai/dsh-spill' /** Minimal concrete backend: records the last request, returns a fixed ref. */ -class StubSpill extends SpillFiles { +class StubStore extends SpillStore { last: SaveTextSpill | undefined async saveText(input: SaveTextSpill): Promise { this.last = input - return { path: SpillPath(`/stub/${input.suggestedName}`), bytes: Buffer.byteLength(input.content, 'utf8') } + return { + locator: SpillLocator(`/stub/${input.suggestedName}`), + bytes: Buffer.byteLength(input.content, 'utf8'), + retrievalHint: 'Use the stub reader.', + } } } @@ -32,25 +36,25 @@ function request(content: string): SaveTextSpill { } describe('spill seam', () => { - it('registers as ctx.spillFiles and saves text', async () => { + it('registers as ctx.spillStore and saves text', async () => { const ctx = new Context() - await ctx.plugin(StubSpill) - const ref = await ctx.spillFiles.saveText(request('hello')) - expect(ref).toEqual({ path: '/stub/web_fetch.txt', bytes: 5 }) - expect((ctx.spillFiles as StubSpill).last?.content).toBe('hello') + await ctx.plugin(StubStore) + const ref = await ctx.spillStore.saveText(request('hello')) + expect(ref).toEqual({ locator: '/stub/web_fetch.txt', bytes: 5, retrievalHint: 'Use the stub reader.' }) + expect((ctx.spillStore as StubStore).last?.content).toBe('hello') }) it('rejects a second implementation (one per context)', async () => { const ctx = new Context() - await ctx.plugin(StubSpill) - await expect(ctx.plugin(StubSpill)).rejects.toThrow() + await ctx.plugin(StubStore) + await expect(ctx.plugin(StubStore)).rejects.toThrow() }) it('releases the service on disposal', async () => { const ctx = new Context() - const fiber = await ctx.plugin(StubSpill) - expect(ctx.spillFiles).toBeInstanceOf(StubSpill) + const fiber = await ctx.plugin(StubStore) + expect(ctx.spillStore).toBeInstanceOf(StubStore) await fiber.dispose() - expect((ctx as Context & { spillFiles?: unknown }).spillFiles).toBeUndefined() + expect((ctx as Context & { spillStore?: unknown }).spillStore).toBeUndefined() }) }) diff --git a/packages/support/acp-snapshot/src/normalize.ts b/packages/support/acp-snapshot/src/normalize.ts index 5669eb6f84..ccebaaaae6 100644 --- a/packages/support/acp-snapshot/src/normalize.ts +++ b/packages/support/acp-snapshot/src/normalize.ts @@ -61,8 +61,8 @@ function scrubString(value: string, ctx: NormalizeContext): string { // residual UUID (covers ids that appear in places we didn't enumerate). out = out.split(ctx.cwd).join(CWD) out = out.split(`/private${CWD}`).join(CWD) - out = out.replace(LOCAL_SPILL_PATH_RE, (_match, name: string) => `{{spillPath:${name}}}`) - out = out.replace(SNAPSHOT_SPILL_PATH_RE, (_match, name: string) => `{{spillPath:${name}}}`) + out = out.replace(LOCAL_SPILL_PATH_RE, (_match, name: string) => `{{spillLocator:${name}}}`) + out = out.replace(SNAPSHOT_SPILL_PATH_RE, (_match, name: string) => `{{spillLocator:${name}}}`) for (const id of ctx.sessionIds) out = out.split(id).join(SESSION_ID) out = out.replace(UUID_RE, SESSION_ID) return out diff --git a/packages/support/acp-snapshot/tests/normalize.spec.ts b/packages/support/acp-snapshot/tests/normalize.spec.ts index b104a3ad2b..f9df56c180 100644 --- a/packages/support/acp-snapshot/tests/normalize.spec.ts +++ b/packages/support/acp-snapshot/tests/normalize.spec.ts @@ -98,12 +98,12 @@ describe('normalizeSessionLog', () => { data: { content: [{ type: 'text', - text: `Full formatted result saved to: ${ctx.cwd}/.spill/session-c22bc3f1d2af/8a7b6c5d4e3f-bash.txt. Use read with offset/limit to inspect it.`, + text: `Full formatted result stored at: ${ctx.cwd}/.spill/session-c22bc3f1d2af/8a7b6c5d4e3f-bash.txt. Use read with offset/limit, or grep this path to search within it.`, }], }, }) const out = normalizeSessionLog(`${header({ cwd: ctx.cwd })}\n${ev}\n`, ctx) - expect(out).toContain('{{spillPath:bash.txt}}') + expect(out).toContain('{{spillLocator:bash.txt}}') expect(out).not.toContain('session-c22bc3f1d2af') expect(out).not.toContain('8a7b6c5d4e3f') }) @@ -114,13 +114,13 @@ describe('normalizeSessionLog', () => { data: { content: [{ type: 'text', - text: `Full formatted result saved to: /private${ctx.cwd}/.spill/session-c22bc3f1d2af/8a7b6c5d4e3f-bash.txt. Use read with offset/limit to inspect it.`, + text: `Full formatted result stored at: /private${ctx.cwd}/.spill/session-c22bc3f1d2af/8a7b6c5d4e3f-bash.txt. Use read with offset/limit, or grep this path to search within it.`, }], }, }) const out = normalizeSessionLog(`${header({ cwd: ctx.cwd })}\n${ev}\n`, ctx) - expect(out).toContain('{{spillPath:bash.txt}}') - expect(out).not.toContain('/private{{spillPath') + expect(out).toContain('{{spillLocator:bash.txt}}') + expect(out).not.toContain('/private{{spillLocator') }) it('scrubs fixed snapshot spill paths', () => { @@ -129,12 +129,12 @@ describe('normalizeSessionLog', () => { data: { content: [{ type: 'text', - text: 'Full formatted result saved to: /tmp/dsh-acp-snapshot-spill/session-c22bc3f1d2af/8a7b6c5d4e3f-bash.txt. Use read with offset/limit to inspect it.', + text: 'Full formatted result stored at: /tmp/dsh-acp-snapshot-spill/session-c22bc3f1d2af/8a7b6c5d4e3f-bash.txt. Use read with offset/limit, or grep this path to search within it.', }], }, }) const out = normalizeSessionLog(`${header({ cwd: ctx.cwd })}\n${ev}\n`, ctx) - expect(out).toContain('{{spillPath:bash.txt}}') + expect(out).toContain('{{spillLocator:bash.txt}}') expect(out).not.toContain('/tmp/dsh-acp-snapshot-spill') }) diff --git a/packages/web/tool-web/tests/spill.spec.ts b/packages/web/tool-web/tests/spill.spec.ts index e44cbb7b45..58599d2c54 100644 --- a/packages/web/tool-web/tests/spill.spec.ts +++ b/packages/web/tool-web/tests/spill.spec.ts @@ -1,10 +1,10 @@ /** * Showcase integration: the real `web_fetch` tool + the real spill stack * (`dsh-spill-local` backend + `dsh-spill-policy`), exercised through - * `ctx.tools.execute()`. Proves the RFC's default path — a large formatted fetch - * result is automatically retained and spilled with NO tool-specific spill code, - * and the model-facing text changes ONLY by the deliberate spill notice (the - * full formatted result lands in the spill file). + * `ctx.tools.execute()`. Proves the RFC's default local-backend path — a large + * formatted fetch result is automatically retained and spilled with NO + * tool-specific spill code, and the model-facing text changes ONLY by the + * deliberate spill notice (the full formatted result lands in the spill file). */ import { afterEach, beforeEach, describe, expect, it } from 'vitest' @@ -21,7 +21,7 @@ import ToolRegistry from '@deepseek-ai/dsh-tools' import type { ToolExecution } from '@deepseek-ai/dsh-tools' import WebService from '@deepseek-ai/dsh-web' import * as WebFetchLocal from '@deepseek-ai/dsh-web-fetch-local' -import LocalSpillFiles from '@deepseek-ai/dsh-spill-local' +import LocalSpillStore from '@deepseek-ai/dsh-spill-local' import * as SpillPolicy from '@deepseek-ai/dsh-spill-policy' import * as ToolWeb from '@deepseek-ai/dsh-tool-web' @@ -50,7 +50,7 @@ beforeEach(async () => { // Provider cap generous so the tool returns a large formatted result; the // policy cap is what triggers the spill (the RFC's separation of concerns). await ctx.plugin(WebFetchLocal, { maxBodyChars: 500_000 }) - await ctx.plugin(LocalSpillFiles, { root: spillRoot }) + await ctx.plugin(LocalSpillStore, { root: spillRoot }) await ctx.plugin(SpillPolicy, { maxInlineBytes: MAX_INLINE_BYTES }) await ctx.plugin(ToolWeb) }) @@ -68,7 +68,7 @@ function fetchCall(): Promise<{ isError: boolean; content: { type: string; text? } describe('web_fetch spill showcase', () => { - it('spills a large formatted result and returns a preview + spill path', async () => { + it('spills a large formatted result and returns a preview + spill locator', async () => { const out = await fetchCall() expect(out.isError).toBe(false) const text = out.content.map(b => b.text).join('') @@ -77,11 +77,11 @@ describe('web_fetch spill showcase', () => { expect(text.length).toBeLessThan(BODY.length) expect(Buffer.byteLength(text, 'utf8')).toBeLessThanOrEqual(MAX_INLINE_BYTES) expect(text).toContain(`Fetched ${base}`) // the head of the formatted result survives - expect(text).toContain('Full formatted result saved to:') - expect(text).toContain('Use read with offset/limit') + expect(text).toContain('Full formatted result stored at:') + expect(text).toContain('Use read with offset/limit, or grep this path') // The spill file holds the FULL formatted result the tool returned. - const match = /saved to: (\S+?)\. Use read/.exec(text) + const match = /stored at: (\S+?)\. Use read/.exec(text) expect(match).not.toBeNull() const spillPath = match![1]! const saved = readFileSync(spillPath, 'utf8') diff --git a/scripts/gen-doc-graphs.ts b/scripts/gen-doc-graphs.ts index a94f470299..74a8987911 100644 --- a/scripts/gen-doc-graphs.ts +++ b/scripts/gen-doc-graphs.ts @@ -236,13 +236,13 @@ const SERVICE_ROLES: ServiceRole[] = [ note: 'Search and fetch providers register into one ctx.web seam; tool-web owns the stable model-facing names.', }, { - key: 'spillFiles', + key: 'spillStore', pkg: 'spill', title: 'Spill storage seam', mode: 'seam', implementations: ['spill-local'], consumers: ['spill-policy'], - note: 'The backend saves oversized tool text to a session-scoped path; spill-policy is the tools/post-execute consumer that decides when to spill.', + note: 'The backend saves oversized tool text and returns a model-facing locator plus retrieval hint; spill-policy is the tools/post-execute consumer that decides when to spill.', }, { key: 'workflows', diff --git a/scripts/gen-tool-catalog.ts b/scripts/gen-tool-catalog.ts index 922a1994e8..34da17aa56 100644 --- a/scripts/gen-tool-catalog.ts +++ b/scripts/gen-tool-catalog.ts @@ -193,13 +193,13 @@ const TOOL_PACKAGES: ToolPackage[] = [ async mount(ctx) { // The tools inject `bash` (search executes fixed `rg` commands through // the executor seam, not ctx.fs); boot the local executor to satisfy it. - // `ctx.spillFiles` is optional (read via ctx.get) and does not affect the + // `ctx.spillStore` is optional (read via ctx.get) and does not affect the // schemas, so no spill backend is mounted. await ctx.plugin(LocalBashExecutor) await ctx.plugin(ToolFsSearch) }, note: - 'glob and grep are bash-backed discovery tools: they run fixed ripgrep commands through ctx.bash as ordinary foreground calls (never background tasks). Capped results save the complete formatted list through the optional ctx.spillFiles backend; returned paths are follow-up-readable in co-located bash/filesystem deployments.', + 'glob and grep are bash-backed discovery tools: they run fixed ripgrep commands through ctx.bash as ordinary foreground calls (never background tasks). Capped results save the complete formatted list through the optional ctx.spillStore backend; returned locators are follow-up-readable/searchable when the backend exposes local paths in co-located deployments.', }, { pkg: '@deepseek-ai/dsh-tool-skill', diff --git a/scripts/type-equiv.manifest.json b/scripts/type-equiv.manifest.json index 4968a0f19f..19e144c2c5 100644 --- a/scripts/type-equiv.manifest.json +++ b/scripts/type-equiv.manifest.json @@ -126,7 +126,7 @@ { "doc": "docs/core-data-structures/spill.md", "symbol": "SpillOwner", "source": "packages/spill/spill/src/types.ts" }, { "doc": "docs/core-data-structures/spill.md", "symbol": "SpillSource", "source": "packages/spill/spill/src/types.ts" }, { "doc": "docs/core-data-structures/spill.md", "symbol": "SpillRef", "source": "packages/spill/spill/src/types.ts" }, - { "doc": "docs/core-data-structures/spill.md", "symbol": "SpillPath", "source": "packages/spill/spill/src/types.ts" }, + { "doc": "docs/core-data-structures/spill.md", "symbol": "SpillLocator", "source": "packages/spill/spill/src/types.ts" }, { "doc": "docs/core-data-structures/workflow.md", "symbol": "WorkflowStartRequest", "source": "packages/workflow/workflow/src/types.ts" }, { "doc": "docs/core-data-structures/workflow.md", "symbol": "WorkflowMeta", "source": "packages/workflow/workflow/src/types.ts" }, From 6c57e6036d3d330415aada4b2bf40212ebd3ae64 Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Tue, 14 Jul 2026 10:40:53 +0800 Subject: [PATCH 23/24] docs(spill): clarify forked spill namespace --- docs/core-data-structures/spill.md | 4 ++-- .../2026-07-08-tool-output-spill-files.md | 2 +- packages/spill/spill/README.md | 4 ++-- packages/spill/spill/src/types.ts | 13 +++++++------ 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/docs/core-data-structures/spill.md b/docs/core-data-structures/spill.md index afc825f09d..4e8ced8258 100644 --- a/docs/core-data-structures/spill.md +++ b/docs/core-data-structures/spill.md @@ -6,7 +6,7 @@ Source: [`packages/spill/spill/src/types.ts`](../../packages/spill/spill/src/typ ## The save request -`saveText` is the whole seam: persist `content` verbatim, return an opaque locator, a backend-supplied retrieval hint, and the exact byte count. The request carries WHO the artifact belongs to (`owner`), WHERE it came from (`source`, descriptive provenance for naming and future cleanup — not access control), and a `suggestedName` the backend may use as a naming hint (it is not a path). +`saveText` is the whole seam: persist `content` verbatim, return an opaque locator, a backend-supplied retrieval hint, and the exact byte count. The request carries the save-time storage namespace (`owner`), WHERE it came from (`source`, descriptive provenance for naming and inspection — not access control), and a `suggestedName` the backend may use as a naming hint (it is not a path). ```ts type-equiv interface SaveTextSpill { @@ -23,7 +23,7 @@ interface SpillOwner { } ``` -`SpillOwner` scopes storage to a `SessionId` — spill is inherently session-scoped (its directory layout and future cleanup unit are per session), so the seam imports `dsh-session`'s `SessionId` directly rather than minting a decoupled token like the bash executor's cross-session `OwnerToken` ([bash.md](bash.md)). +`SpillOwner.sessionId` is the save-time storage namespace. Forked sessions inherit existing spill locators from the seeded log; those artifacts are not copied or re-owned, and spills produced after the fork use the child session id. A retention-period cleanup may expire old locators with other old session artifacts; the spill seam does not define a per-session cleanup policy. ```ts type-equiv interface SpillSource { diff --git a/docs/rfc/implemented/architecture/2026-07-08-tool-output-spill-files.md b/docs/rfc/implemented/architecture/2026-07-08-tool-output-spill-files.md index ac56c151fb..8e592a179c 100644 --- a/docs/rfc/implemented/architecture/2026-07-08-tool-output-spill-files.md +++ b/docs/rfc/implemented/architecture/2026-07-08-tool-output-spill-files.md @@ -53,7 +53,7 @@ interface SpillRef { } ``` -`SpillLocator` is a [branded](../../../../packages/util/brand) model-facing handle returned by the backend. The local backend renders it as a filesystem path; a remote or database backend can render a URI, key, or command token. Consumers treat it as opaque and render it with `retrievalHint` instead of assuming `read` is always the right retrieval mechanism. `SpillOwner` scopes storage to a `SessionId` — spill is inherently session-scoped, so the seam imports `dsh-session`'s `SessionId` directly rather than minting a decoupled token like the bash executor's `OwnerToken`. +`SpillLocator` is a [branded](../../../../packages/util/brand) model-facing handle returned by the backend. The local backend renders it as a filesystem path; a remote or database backend can render a URI, key, or command token. Consumers treat it as opaque and render it with `retrievalHint` instead of assuming `read` is always the right retrieval mechanism. `SpillOwner.sessionId` is the save-time storage namespace: forked sessions inherit existing spill locators from the seeded log without copying or re-owning them, and new spills after the fork use the child session id. A retention-period cleanup may expire old locators with other old session artifacts; the spill seam does not define a per-session cleanup policy. `dsh-spill-local` owns only storage details: session-scoped directory selection, safe names, path-traversal protection, the write, and returning `{ locator, bytes, retrievalHint }`. It does not own retention policy, tool-result replacement, search, or file inspection. Files land at `/session-/-`, where `root` is a configured path or a lazily-created private (0700) per-process temp dir, the session subdir is a short `sha256(sessionId)` prefix, and the leaf is a random hex prefix plus the caller's `suggestedName` sanitized to one path segment (mirrors the JSONL backend's `encodeSegment`). The write is `open(path, 'wx', 0o600)` — exclusive and owner-only, so a planted symlink cannot redirect it. The locator is the path, and the retrieval hint tells the model it can use `read` or `grep` on that path. diff --git a/packages/spill/spill/README.md b/packages/spill/spill/README.md index f550e31d84..21c7105f94 100644 --- a/packages/spill/spill/README.md +++ b/packages/spill/spill/README.md @@ -18,10 +18,10 @@ The split mirrors the bash/fs seams. A future remote or virtual backend (e.g. a |---|---| | `saveText(input)` | Persist `input.content` verbatim; resolves with a `SpillRef` (opaque locator, exact bytes written, and retrieval hint). **Rejects on a real storage failure** (permissions, ENOSPC, backend unavailable) — the caller decides how to degrade. | -Storage is scoped by the request's `owner` session; the backend chooses its own private representation and may derive names from — never trust as a path — the caller's `suggestedName`. The seam owns storage only: NO retention policy (that is [`@deepseek-ai/dsh-retention`](../../util/retention)), NO tool-result replacement (that is `@deepseek-ai/dsh-spill-policy`), NO retrieval/search API (the backend's `retrievalHint` tells the model what to do with the locator). +Storage is grouped by the request's `owner` session as a save-time namespace; the backend chooses its own private representation and may derive names from — never trust as a path — the caller's `suggestedName`. The seam owns storage only: NO retention policy (that is [`@deepseek-ai/dsh-retention`](../../util/retention)), NO tool-result replacement (that is `@deepseek-ai/dsh-spill-policy`), NO retrieval/search API (the backend's `retrievalHint` tells the model what to do with the locator). ## Vocabulary -`SaveTextSpill` (owner, source, suggestedName, content) is the request; `SpillRef` (locator, bytes, retrievalHint) is the result. `SpillLocator` is [branded](../../util/brand) and rendered to the model as an opaque string — a local path for `dsh-spill-local`, but a future backend may return a URI, key, or command token without changing policy/tool consumers. `SpillOwner` scopes storage to a `SessionId`; unlike the bash executor's decoupled `OwnerToken`, spill is inherently session-scoped, so the seam imports `dsh-session`'s `SessionId` directly. `SpillSource` (toolName, callId, label) is descriptive provenance for backend naming and future cleanup, not access control. See `src/types.ts` for the full contracts. +`SaveTextSpill` (owner, source, suggestedName, content) is the request; `SpillRef` (locator, bytes, retrievalHint) is the result. `SpillLocator` is [branded](../../util/brand) and rendered to the model as an opaque string — a local path for `dsh-spill-local`, but a future backend may return a URI, key, or command token without changing policy/tool consumers. `SpillOwner.sessionId` is the save-time storage namespace: forked sessions inherit existing locators from the seeded log without copying or re-owning them, and new spills after the fork use the child session id. `SpillSource` (toolName, callId, label) is descriptive provenance for backend naming and inspection, not access control. See `src/types.ts` for the full contracts. See the [tool output spill RFC](../../../docs/rfc/implemented/architecture/2026-07-08-tool-output-spill-files.md) for the design rationale, including why creation belongs to the runtime spill seam rather than the model-facing `write` tool. diff --git a/packages/spill/spill/src/types.ts b/packages/spill/spill/src/types.ts index 5290a9738e..96376bb268 100644 --- a/packages/spill/spill/src/types.ts +++ b/packages/spill/spill/src/types.ts @@ -28,10 +28,11 @@ export function SpillLocator(locator: string): SpillLocator { } /** - * Who a spilled file belongs to: the session whose tool call produced it. The - * backend scopes storage per session (its directory layout, its cleanup unit), - * so the owner is the session id, not a decoupled token — spill is inherently - * session-scoped, unlike the bash executor's cross-session `OwnerToken`. + * Save-time storage namespace for a spilled artifact. The session id lets a + * backend group storage under the producing session, but the returned + * {@link SpillLocator} is the model-facing handle. Forked sessions inherit + * locators already present in the seeded log; those artifacts are not copied or + * re-owned, and spills produced after the fork use the child session id. */ export interface SpillOwner { sessionId: SessionId @@ -39,8 +40,8 @@ export interface SpillOwner { /** * Provenance of one spilled artifact — recorded by the backend for a readable - * filename and future cleanup/inspection. Not interpreted for access control - * (the {@link SpillOwner} scopes storage); purely descriptive. + * filename and inspection. Not interpreted for access control; purely + * descriptive. */ export interface SpillSource { /** The tool whose result was spilled (e.g. `web_fetch`). */ From b69129601e5033a62532b0e72061bfab1cafc516 Mon Sep 17 00:00:00 2001 From: Dudu-0223 Date: Tue, 14 Jul 2026 11:02:49 +0800 Subject: [PATCH 24/24] test: satisfy post-merge push gates --- .../acp-agent/tests/snapshots/bash-spill/stdout.golden.jsonl | 2 +- packages/spill/spill-local/src/store.ts | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/examples/acp-agent/tests/snapshots/bash-spill/stdout.golden.jsonl b/examples/acp-agent/tests/snapshots/bash-spill/stdout.golden.jsonl index b3590d29bb..c4ada6ea9e 100644 --- a/examples/acp-agent/tests/snapshots/bash-spill/stdout.golden.jsonl +++ b/examples/acp-agent/tests/snapshots/bash-spill/stdout.golden.jsonl @@ -1,5 +1,5 @@ {"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentInfo":{"name":"deepseek-harness-acp","version":"0.0.1"},"agentCapabilities":{"loadSession":true,"promptCapabilities":{"image":false,"audio":false,"embeddedContext":false}},"authMethods":[]}} -{"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}"}} +{"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}","configOptions":[{"id":"permission","name":"Permissions","description":"The session permission preset: each choice bundles a sandbox mode and an approval policy.","category":"mode","type":"select","currentValue":"danger-full-access","options":[{"value":"workspace-write","name":"workspace-write","description":"Write inside the workspace; anything wider asks for your approval."},{"value":"danger-full-access","name":"danger-full-access","description":"Full file access, no approval prompts."}]}]}} {"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call","toolCallId":"call_spill","title":"node -e \"process.stdout.write('SPILL_START-' + 'x'.repeat(2000) + '-SPILL_END')\"","kind":"execute","status":"in_progress","rawInput":"node -e \"process.stdout.write('SPILL_START-' + 'x'.repeat(2000) + '-SPILL_END')\"","content":[{"type":"content","content":{"type":"text","text":"Print large deterministic output"}}]}}} {"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call_update","toolCallId":"call_spill","status":"completed","content":[{"type":"content","content":{"type":"text","text":"```console\nSPILL_START-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-SPILL_END\n\n(Omitted 1417 bytes. Full formatted result stored at: {{spillLocator:bash.txt}}. Use read with offset/limit, or grep this path to search within it.)\n```"}}]}}} {"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"DONE"}}}} diff --git a/packages/spill/spill-local/src/store.ts b/packages/spill/spill-local/src/store.ts index 44e4ee7129..e44418767a 100644 --- a/packages/spill/spill-local/src/store.ts +++ b/packages/spill/spill-local/src/store.ts @@ -29,6 +29,9 @@ export function privateRoot(): string { return defaultRoot } +// Deliberately mirrors the JSONL path encoder, but keeps spill's empty-name +// policy (`""` -> `"~"`) local so storage backends stay decoupled. +/* jscpd:ignore-start */ /** * Encode an arbitrary string as one safe path segment, injectively over ALL JS * (UTF-16) strings. A session id / suggested name is untrusted input, so this @@ -58,6 +61,7 @@ export function encodeSegment(raw: string): string { } return out } +/* jscpd:ignore-end */ /** * The session-scoped directory: `/session-`, a short stable hash.