From 45a5175e441ad073d82a8a0db688aa3572b90057 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Mon, 27 Jul 2026 00:22:13 +0800 Subject: [PATCH 01/14] feat(tool-web): replace the regex HTML-to-markdown converter with turndown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the turndown Agent Note from the NIH dependency audit (full variant, not the minimal entities-only fallback): dsh-tool-web's fetch rendering now converts HTML through turndown + @joplin/turndown-plugin-gfm (atx headings, fenced code, dash bullets, GFM tables/strikethrough) over the real domino DOM, with script/style/noscript removed wholesale. The hand-rolled ~86-line regex converter html.ts and its entity tables are deleted; renderBody wraps the conversion in try/catch falling back to the raw HTML body, because turndown's recursive DOM walk overflows with a RangeError on pathological nesting (measured: 4k levels on the main thread, 8k in a worker) where the regex version could never throw. Closure weight, measured: tool-web IS in the single-exe runtime closure, and the exe asset globs would pack ~7.9 MB of the three new packages — but ~6 MB of that is domino's test corpus, with runtime lib/ at ~550 KB against a ~174 MB artifact (<0.5% either way), so the swap wins. Per testing policy the previously-missing keyless web_fetch snapshot ships in the same change: the acp-agent `web-fetch` scenario boots a new web.cordis.yml overlay (web seam + real dsh-web-fetch-local provider + tool-web fetch-only + a loopback HTTP fixture server on a fixed port serving deterministic HTML with entities, a GFM table, and nesting), so recording and keyless replay both drive the real HTTP fetch and real conversion end to end; the scenario pins the new `web` header class. The Agent Note moves proposed -> implemented and is rewritten per the lifecycle contract (Decision/Consequences/Testing, closure verdict and alternatives recorded); tool-web and acp-agent READMEs updated in both languages and pairs re-recorded. --- ...ndown-for-tool-web-html-markdown.i18n.yaml | 4 +- ...-26-turndown-for-tool-web-html-markdown.md | 37 ++ ...-turndown-for-tool-web-html-markdown.zh.md | 37 ++ ...-26-turndown-for-tool-web-html-markdown.md | 32 -- ...-turndown-for-tool-web-html-markdown.zh.md | 32 -- docs/config-catalog.md | 2 +- examples/acp-agent/README.i18n.yaml | 4 +- examples/acp-agent/README.md | 2 +- examples/acp-agent/README.zh.md | 2 +- examples/acp-agent/tests/acp.snapshot.ts | 7 + .../tests/snapshots/web-fetch/input.json | 7 + .../tests/snapshots/web-fetch/session.jsonl | 127 +++++ .../snapshots/web-fetch/stdout.expected.jsonl | 4 + .../web-fetch/system-prompt.expected.md | 27 + .../web-fetch/tool-schemas.expected.json | 489 ++++++++++++++++++ .../acp-agent/web-fetch-fixture-server.mjs | 52 ++ examples/acp-agent/web.cordis.snapshot.yml | 31 ++ examples/acp-agent/web.cordis.yml | 21 + examples/package.json | 1 + packages/web/tool-web/README.i18n.yaml | 4 +- packages/web/tool-web/README.md | 4 +- packages/web/tool-web/README.zh.md | 4 +- packages/web/tool-web/package.json | 5 +- packages/web/tool-web/src/fetch.ts | 34 +- packages/web/tool-web/src/html.ts | 86 --- packages/web/tool-web/src/index.ts | 1 - .../web/tool-web/src/turndown-plugin-gfm.d.ts | 12 + packages/web/tool-web/tests/tool-web.spec.ts | 69 +-- pnpm-lock.yaml | 35 ++ 29 files changed, 962 insertions(+), 210 deletions(-) rename .agents/notes/{proposed => implemented}/simplification/2026-07-26-turndown-for-tool-web-html-markdown.i18n.yaml (60%) create mode 100644 .agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md create mode 100644 .agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.zh.md delete mode 100644 .agents/notes/proposed/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md delete mode 100644 .agents/notes/proposed/simplification/2026-07-26-turndown-for-tool-web-html-markdown.zh.md create mode 100644 examples/acp-agent/tests/snapshots/web-fetch/input.json create mode 100644 examples/acp-agent/tests/snapshots/web-fetch/session.jsonl create mode 100644 examples/acp-agent/tests/snapshots/web-fetch/stdout.expected.jsonl create mode 100644 examples/acp-agent/tests/snapshots/web-fetch/system-prompt.expected.md create mode 100644 examples/acp-agent/tests/snapshots/web-fetch/tool-schemas.expected.json create mode 100644 examples/acp-agent/web-fetch-fixture-server.mjs create mode 100644 examples/acp-agent/web.cordis.snapshot.yml create mode 100644 examples/acp-agent/web.cordis.yml delete mode 100644 packages/web/tool-web/src/html.ts create mode 100644 packages/web/tool-web/src/turndown-plugin-gfm.d.ts diff --git a/.agents/notes/proposed/simplification/2026-07-26-turndown-for-tool-web-html-markdown.i18n.yaml b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.i18n.yaml similarity index 60% rename from .agents/notes/proposed/simplification/2026-07-26-turndown-for-tool-web-html-markdown.i18n.yaml rename to .agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.i18n.yaml index ced514a423..60a5d9aca7 100644 --- a/.agents/notes/proposed/simplification/2026-07-26-turndown-for-tool-web-html-markdown.i18n.yaml +++ b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -2026-07-26-turndown-for-tool-web-html-markdown.md: 7f25e51bf6e6fc9313a880abee737bca80a472af -2026-07-26-turndown-for-tool-web-html-markdown.zh.md: 3a59b08e13fd392e4f34ac543f32f5b4648f3c1c +2026-07-26-turndown-for-tool-web-html-markdown.md: c72decc336055f3b78dafdf98f2be3771b833cdb +2026-07-26-turndown-for-tool-web-html-markdown.zh.md: 30667b62538ec50608cae461b5cdf651b48e2731 diff --git a/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md new file mode 100644 index 0000000000..c72decc336 --- /dev/null +++ b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md @@ -0,0 +1,37 @@ +# Agent Note: Replace tool-web's regex HTML-to-markdown converter with turndown + +Status: implemented + +English | [中文](2026-07-26-turndown-for-tool-web-html-markdown.zh.md) + +## Problem + +`dsh-tool-web`'s `src/html.ts` (~86 lines, ~40 lines of dedicated tests; deleted by this change) converted fetched HTML to markdown with regexes: strip script/style/noscript/comments, convert ``/``/`
  • `, decode numeric entities plus a 12-entry named-entity table, collapse whitespace. The module's own JSDoc said "A richer converter can replace it without changing the seam or tool schema", and the README's Known Limitations documented it as "a minimal regex converter, not an HTML parser — tables, images, and nested formatting are lost." The [web capability seam note](../architecture/2026-06-24-web-capability-seam.md) assigns HTML→markdown to this package as presentation, so the swap point was exactly here. The converter's output is model-visible on every fetched HTML page; no keyless snapshot exercised `web_fetch`, so no expected outputs pinned it. + +## Decision + +`packages/web/tool-web/src/fetch.ts` owns a module-level [`turndown`](https://github.com/mixmark-io/turndown) instance (`headingStyle: 'atx'`, `codeBlockStyle: 'fenced'`, `bulletListMarker: '-'` — fixed model-facing presentation, not deployment tunables) with `@joplin/turndown-plugin-gfm`'s composite `gfm` plugin for tables/strikethrough and `remove(['script', 'style', 'noscript'])` replacing the old wholesale drops. `renderBody`'s `html` arm calls it in a try/catch falling back to the raw HTML body: the regex version could never throw, while turndown/domino's recursive DOM walk overflows with a `RangeError` at a few thousand nesting levels (measured: 4k throws on the main thread, 8k in a worker thread), and a degraded page beats an error for a body the provider already decoded. `html.ts` and its conversion tests are deleted; the fallback and the status-header/truncation-footer formatting are tested in `tests/tool-web.spec.ts`, and the README's Known Limitations trades the regex-converter caveat for the pathological-nesting fallback. The gfm plugin ships no types; `src/turndown-plugin-gfm.d.ts` declares the one imported export over `@types/turndown` (a devDependency). + +The dependency-weight question the proposal flagged resolves in favor of the swap: `@deepseek-ai/dsh-tool-web` is in the single-file-executable closure ([single-exe note](../architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.md)), and the exe's asset globs would pack ~7.9 MB of the three packages as published — but ~6 MB of that is `@mixmark-io/domino`'s test corpus (`test/**`), with runtime `lib/` at ~550 KB against a ~174 MB artifact, under 0.5% either way. + +## Snapshot coverage + +The previously-missing keyless `web_fetch` snapshot ships with the change as the acp-agent scenario `web-fetch`: `examples/acp-agent/web.cordis.yml` composes the web seam, the real `dsh-web-fetch-local` provider, `tool-web` with `search: false`, and `web-fetch-fixture-server.mjs` — a loopback HTTP fixture on a fixed port (the fetched URL is part of the recorded transcript) serving deterministic HTML with named entities, a GFM table, and nested formatting. Recording and keyless replay both drive the real HTTP fetch and conversion; the pinned tool result is the turndown output, and the scenario pins the `web` header class (the `web_fetch` schema and guidance). + +## Alternatives considered + +- **`@mozilla/readability` + a DOM.** Solves a different problem (content extraction, not conversion) and drags a heavier DOM dependency; the seam only asks for markdown rendering of whatever the fetch returned. +- **Keep the regex converter.** It was an explicit v1 placeholder per its own JSDoc; keeping it meant model-visible quality (tables, images, nested formatting) stayed lost for the cost of maintaining bespoke entity tables. +- **The minimal `entities`-only variant.** The proposal's fallback position: replace only the entity-decoding third of `html.ts` with the zero-dependency `entities` package, deleting less but avoiding the dependency-weight question. Not taken because the closure math above made the weight immaterial while the full swap deletes the whole hand-rolled converter and its documented quality gaps. +- **`turndown-plugin-gfm` (the original) instead of `@joplin/turndown-plugin-gfm`.** The original is unmaintained (last publish 2018); the Joplin fork is current against turndown 7 and actively released. + +## Consequences + +- **Bought**: full-fidelity model-visible markdown — tables, images, strikethrough, nested emphasis, fenced code blocks, and the complete named-entity set — plus the deletion of the bespoke converter and its entity tables, with the README's regex-converter caveat narrowed to one degenerate case. +- **Paid**: two runtime dependencies (`turndown` → `@mixmark-io/domino`) enter tool-web and therefore the exe closure (~550 KB of runtime code as measured above), and a new failure mode — pathological nesting — is handled by falling back to raw HTML rather than converting. +- Model-visible output changed on every fetched HTML page; nothing pinned the old output, and the new snapshot pins the new one. + +## Testing + +- `packages/web/tool-web/tests/tool-web.spec.ts` covers the turndown conversion surface (entities, links, tables, nesting, script/style/noscript removal) through `renderBody`, and the raw-HTML fallback with a measured reliably-overflowing 20k-level nesting input; per-file coverage on the package src is 100%. +- The `web-fetch` acp-agent snapshot pins the assembled behavior keylessly end to end (real Loader composition, real HTTP fetch, real conversion). diff --git a/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.zh.md b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.zh.md new file mode 100644 index 0000000000..30667b6253 --- /dev/null +++ b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.zh.md @@ -0,0 +1,37 @@ +# Agent Note: 用 turndown 替换 tool-web 的正则 HTML 转 markdown 转换器 + +Status: implemented + +[English](2026-07-26-turndown-for-tool-web-html-markdown.md) | 中文 + +## 问题 + +`dsh-tool-web` 的 `src/html.ts`（约 86 行，另有约 40 行专属测试；已由本变更删除）曾用正则表达式把抓取到的 HTML 转成 markdown：剥离 script、style、noscript 标签与注释，转换 ``/``/`
  • `，解码数字实体外加一张 12 项的命名实体表，并折叠空白。该模块自身的 JSDoc 写明「A richer converter can replace it without changing the seam or tool schema」，README 的 Known Limitations 章节也把它记载为「a minimal regex converter, not an HTML parser — tables, images, and nested formatting are lost」。[web 能力 seam 决策记录](../architecture/2026-06-24-web-capability-seam.md)把 HTML 转 markdown 作为呈现职责划归本包（package），因此替换点恰好就在这里。每个抓取到的 HTML 页面上，该转换器的输出都对模型可见；此前没有任何无密钥快照执行到 `web_fetch`，因此没有预期输出固定它的行为。 + +## 决策 + +`packages/web/tool-web/src/fetch.ts` 持有一个模块级 [`turndown`](https://github.com/mixmark-io/turndown) 实例（`headingStyle: 'atx'`、`codeBlockStyle: 'fenced'`、`bulletListMarker: '-'`——固定的面向模型呈现方式，不是部署可调项），配合 `@joplin/turndown-plugin-gfm` 的组合 `gfm` 插件提供表格／删除线支持，并用 `remove(['script', 'style', 'noscript'])` 替代旧实现的整体剥离。`renderBody` 的 `html` 分支把调用包在 try/catch 中，失败时回退为原始 HTML 主体：正则版本从不可能抛异常，而 turndown/domino 的递归 DOM 遍历在数千层嵌套（实测：主线程 4k 层抛出，worker 线程 8k 层抛出）会以 `RangeError` 栈溢出，对提供方已经解码的主体来说，降级页面好过报错。`html.ts` 及其转换测试已删除；回退路径与状态头、截断页脚的格式化在 `tests/tool-web.spec.ts` 中有测试覆盖，README 的 Known Limitations 用病态嵌套回退条目替换了正则转换器的警示说明。gfm 插件不带类型声明；`src/turndown-plugin-gfm.d.ts` 基于 `@types/turndown`（devDependency）声明了唯一被导入的导出。 + +提案标记的依赖体积问题的裁决结果支持替换：`@deepseek-ai/dsh-tool-web` 在单文件可执行文件闭包内（[single-exe 决策记录](../architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.md)），可执行文件的资产 glob 会把这三个包按发布原样打入约 7.9 MB——但其中约 6 MB 是 `@mixmark-io/domino` 的测试语料（`test/**`），运行时 `lib/` 仅约 550 KB，相对约 174 MB 的产物，两种口径都不到 0.5%。 + +## 快照覆盖 + +此前缺失的无密钥 `web_fetch` 快照随本变更以 acp-agent 场景 `web-fetch` 落地：`examples/acp-agent/web.cordis.yml` 组合了 web seam、真实的 `dsh-web-fetch-local` 提供方、`search: false` 的 `tool-web`，以及 `web-fetch-fixture-server.mjs`——一个固定端口（抓取的 URL 是录制 transcript（文本记录）的一部分）上的回环 HTTP fixture，提供包含命名实体、GFM 表格与嵌套格式的确定性 HTML。录制与无密钥回放都驱动真实的 HTTP 抓取与转换；固定住的工具结果就是 turndown 的输出，该场景同时固定 `web` header 类（`web_fetch` 的 schema 与指引）。 + +## 曾考虑的替代方案 + +- **`@mozilla/readability` 加一个 DOM。** 它解决的是另一个问题（内容提取，而非格式转换），还会拖入更重的 DOM 依赖；这个 seam 只要求把抓取返回的内容渲染成 markdown。 +- **保留正则转换器。** 按其自身 JSDoc 的说法，它本来就是明确的 v1 占位实现；保留它意味着模型可见的质量（表格、图片、嵌套格式）继续缺失，代价还是维护一套自制实体表。 +- **仅引入 `entities` 的最小变体。** 提案中的退守方案：只用零依赖的 `entities` 包替换 `html.ts` 中的实体解码部分，删得更少但完全避开依赖体积问题。未采纳：上述闭包测算表明体积无关紧要，而完整替换能删掉整个手写转换器及其记录在案的质量缺口。 +- **用原版 `turndown-plugin-gfm` 而非 `@joplin/turndown-plugin-gfm`。** 原版已无人维护（最后发布于 2018 年）；Joplin 分叉与 turndown 7 保持同步并持续发布。 + +## 后果 + +- **收益**：模型可见的完整保真 markdown——表格、图片、删除线、嵌套强调、围栏代码块以及完整的命名实体集——并删除了自制转换器及其实体表，README 中的正则转换器警示收窄为一个退化用例。 +- **代价**：两个运行时依赖（`turndown` → `@mixmark-io/domino`）进入 tool-web 进而进入可执行文件闭包（如上实测约 550 KB 运行时代码），并新增一种失败模式——病态嵌套改为回退原始 HTML 而非转换。 +- 每个抓取到的 HTML 页面上模型可见的输出都已变化；旧输出本无任何固定，新快照固定了新输出。 + +## 测试 + +- `packages/web/tool-web/tests/tool-web.spec.ts` 通过 `renderBody` 覆盖 turndown 转换面（实体、链接、表格、嵌套、script/style/noscript 移除），并用实测可稳定溢出的 2 万层嵌套输入覆盖原始 HTML 回退；该包 src 的逐文件覆盖率为 100%。 +- acp-agent 的 `web-fetch` 快照无密钥地端到端固定组装后的行为（真实 Loader 组合、真实 HTTP 抓取、真实转换）。 diff --git a/.agents/notes/proposed/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md b/.agents/notes/proposed/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md deleted file mode 100644 index 7f25e51bf6..0000000000 --- a/.agents/notes/proposed/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md +++ /dev/null @@ -1,32 +0,0 @@ -# Agent Note: Replace tool-web's regex HTML-to-markdown converter with turndown - -Status: proposed - -English | [中文](2026-07-26-turndown-for-tool-web-html-markdown.zh.md) - -## Problem - -`packages/web/tool-web/src/html.ts` (~86 lines, ~40 lines of dedicated tests) converts fetched HTML to markdown with regexes: strip script/style/noscript/comments, convert ``/``/`
  • `, decode numeric entities plus a 12-entry named-entity table, collapse whitespace. The module's own JSDoc says "A richer converter can replace it without changing the seam or tool schema", and the README's Known Limitations documents it as "a minimal regex converter, not an HTML parser — tables, images, and nested formatting are lost." The [web capability seam note](../../implemented/architecture/2026-06-24-web-capability-seam.md) assigns HTML→markdown to this package as presentation, so the swap point is exactly here. The converter's output is model-visible on every fetched HTML page; no keyless snapshot currently exercises `web_fetch`, so no expected outputs pin it. - -## Proposal - -Replace `htmlToMarkdown` with `turndown` (`new TurndownService().turndown(html)`), optionally with `turndown-plugin-gfm` for tables. The consumer switch in `fetch.ts` and the status-header/truncation-footer formatting stay. Wrap the call in try/catch falling back to the raw text path: the regex version could never throw; turndown on pathological HTML could. Delete `html.ts` and its conversion tests; keep tests for the fallback and the surrounding formatting. Update the README's Known Limitations to drop the regex-converter caveat. - -If the "deliberately minimal fallback" stance is preferred instead, a minimal variant still deletes the worst part: replace the entity-decoding third of the file (~30 lines: `decodeEntities`, `NAMED_ENTITIES`, `safeFromCodePoint`) with the zero-dependency `entities` package (already in the lockfile transitively), erasing the documented "about a dozen entities" limitation at near-zero risk. - -## Alternatives considered - -- **`@mozilla/readability` + a DOM.** Solves a different problem (content extraction, not conversion) and drags a heavier DOM dependency; the seam only asks for markdown rendering of whatever the fetch returned. -- **Keep the regex converter.** It was an explicit v1 placeholder per its own JSDoc; keeping it means model-visible quality (tables, images, nested formatting) stays lost for the cost of maintaining bespoke entity tables. -- **The minimal `entities`-only variant.** Kept in the proposal as the fallback position; it deletes less but avoids the dependency-weight question entirely. - -## Acceptance criteria - -- `web_fetch` renders tables/nested formatting via turndown (or, minimal variant: decodes all named entities), with the README limitation updated. -- Unit tests cover the fallback path; `pnpm run test` passes for the package. -- A keyless snapshot exercising `web_fetch` markdown rendering is added per testing policy (the missing snapshot coverage is part of the change, and it pins the new output). - -## Risks - -- Model-visible output changes on every fetched HTML page — transcript drift is acceptable pre-release, and nothing currently pins the old output. -- Dependency weight: turndown's one dependency (`@mixmark-io/domino`) is a ~200 KB DOM that would enter the single-file-executable closure if tool-web ships in it ([single-exe note](../../implemented/architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.md)); the minimal `entities` variant avoids this if closure size is the deciding factor. diff --git a/.agents/notes/proposed/simplification/2026-07-26-turndown-for-tool-web-html-markdown.zh.md b/.agents/notes/proposed/simplification/2026-07-26-turndown-for-tool-web-html-markdown.zh.md deleted file mode 100644 index 3a59b08e13..0000000000 --- a/.agents/notes/proposed/simplification/2026-07-26-turndown-for-tool-web-html-markdown.zh.md +++ /dev/null @@ -1,32 +0,0 @@ -# Agent Note: 用 turndown 替换 tool-web 的正则 HTML 转 markdown 转换器 - -Status: proposed - -[English](2026-07-26-turndown-for-tool-web-html-markdown.md) | 中文 - -## 问题 - -`packages/web/tool-web/src/html.ts`（约 86 行，另有约 40 行专属测试）用正则表达式把抓取到的 HTML 转成 markdown：剥离 script、style、noscript 标签与注释，转换 ``/``/`
  • `，解码数字实体外加一张 12 项的命名实体表，并折叠空白。该模块自身的 JSDoc 写明「A richer converter can replace it without changing the seam or tool schema」，README 的 Known Limitations 章节也把它记载为「a minimal regex converter, not an HTML parser — tables, images, and nested formatting are lost」。[web 能力 seam 决策记录](../../implemented/architecture/2026-06-24-web-capability-seam.md)把 HTML 转 markdown 作为呈现职责划归本包（package），因此替换点恰好就在这里。每个抓取到的 HTML 页面上，该转换器的输出都对模型可见；当前没有任何无密钥快照执行到 `web_fetch`，因此没有预期输出固定它的行为。 - -## 提案 - -用 `turndown` 替换 `htmlToMarkdown`（`new TurndownService().turndown(html)`），可选择配合 `turndown-plugin-gfm` 支持表格。`fetch.ts` 中的消费方分支与状态头、截断页脚的格式化保持不变。把调用包在 try/catch 中，失败时回退到原始文本路径：正则版本从不可能抛异常，而 turndown 处理病态 HTML 时可能抛出。删除 `html.ts` 及其转换测试；保留回退路径与外围格式化的测试。更新 README 的 Known Limitations 章节，移除正则转换器的警示说明。 - -如果更倾向于「刻意保持最小回退实现」的立场，最小变体仍能删掉最糟的部分：用零依赖的 `entities` 包（已通过传递依赖存在于 lockfile 中）替换文件中占三分之一的实体解码部分（约 30 行：`decodeEntities`、`NAMED_ENTITIES`、`safeFromCodePoint`），以近乎为零的风险抹掉文档记载的「about a dozen entities」限制。 - -## 曾考虑的替代方案 - -- **`@mozilla/readability` 加一个 DOM。** 它解决的是另一个问题（内容提取，而非格式转换），还会拖入更重的 DOM 依赖；这个 seam 只要求把抓取返回的内容渲染成 markdown。 -- **保留正则转换器。** 按其自身 JSDoc 的说法，它本来就是明确的 v1 占位实现；保留它意味着模型可见的质量（表格、图片、嵌套格式）继续缺失，代价还是维护一套自制实体表。 -- **仅引入 `entities` 的最小变体。** 已作为退守方案保留在提案中；它删得更少，但完全避开了依赖体积问题。 - -## 验收标准 - -- `web_fetch` 经由 turndown 渲染表格与嵌套格式（或在最小变体下：解码全部命名实体），README 中的限制说明同步更新。 -- 单元测试覆盖回退路径；该包的 `pnpm run test` 通过。 -- 按测试政策补充一个执行 `web_fetch` markdown 渲染的无密钥快照（缺失的快照覆盖是本变更的一部分，它同时固定新输出）。 - -## 风险 - -- 模型可见的输出在每个抓取到的 HTML 页面上都会变化：预发布阶段的 transcript（文本记录）漂移可以接受，且当前没有任何东西固定旧输出。 -- 依赖体积：turndown 的唯一依赖（`@mixmark-io/domino`）是一个约 200 KB 的 DOM 实现，若 tool-web 进入单文件可执行文件，它会一并进入闭包（[single-exe 决策记录](../../implemented/architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.md)）；若闭包体积是决定因素，最小的 `entities` 变体可以避开这一点。 diff --git a/docs/config-catalog.md b/docs/config-catalog.md index 9880761894..30499c7df6 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -1672,7 +1672,7 @@ export interface Config { } ``` -Source: [`packages/web/tool-web/src/index.ts:29`](../packages/web/tool-web/src/index.ts) +Source: [`packages/web/tool-web/src/index.ts:28`](../packages/web/tool-web/src/index.ts) ## `@deepseek-ai/dsh-tool-workflow` diff --git a/examples/acp-agent/README.i18n.yaml b/examples/acp-agent/README.i18n.yaml index 22842fcd04..391967d91c 100644 --- a/examples/acp-agent/README.i18n.yaml +++ b/examples/acp-agent/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -README.md: 4b3d86b00613cc7c37a8898ef3b39d40a167e66b -README.zh.md: 5bcd85f2b4ae34a11b980bf196d3401f764004d8 +README.md: 0d63ec1f2d9165b9faf0817bd94fbe15b97fa961 +README.zh.md: 0c5f8866ea640843513fd9a4c15a17ed4db59d3b diff --git a/examples/acp-agent/README.md b/examples/acp-agent/README.md index 4b3d86b006..0d63ec1f2d 100644 --- a/examples/acp-agent/README.md +++ b/examples/acp-agent/README.md @@ -9,7 +9,7 @@ pnpm run demo:acp # needs DEEPSEEK_API_KEY (repo-root .env or env) pnpm run demo:code-mode acp # same protocol with the Code Mode tool transport ``` -The leaf loads the ACP app, DeepSeek adapter, sandboxed bash and filesystem stacks, one-shot approval policy, compaction, subagents, workflows, hooks, a derived session-query index, and repeat guard. The app creates one fresh agent per `session/new`, persists sessions to JSONL, and keeps stdout protocol-pure. [`session-query.cordis.yml`](session-query.cordis.yml) explicitly opts into the workspace-authorized query tools and generic timeout/spill policies for their dedicated snapshot; [`fs.cordis.yml`](fs.cordis.yml) adds spill storage for filesystem scenarios, while [`code-mode.cordis.yml`](code-mode.cordis.yml) adds `run_code` and its generated TypeScript SDK. +The leaf loads the ACP app, DeepSeek adapter, sandboxed bash and filesystem stacks, one-shot approval policy, compaction, subagents, workflows, hooks, a derived session-query index, and repeat guard. The app creates one fresh agent per `session/new`, persists sessions to JSONL, and keeps stdout protocol-pure. [`session-query.cordis.yml`](session-query.cordis.yml) explicitly opts into the workspace-authorized query tools and generic timeout/spill policies for their dedicated snapshot; [`fs.cordis.yml`](fs.cordis.yml) adds spill storage for filesystem scenarios, [`code-mode.cordis.yml`](code-mode.cordis.yml) adds `run_code` and its generated TypeScript SDK, and [`web.cordis.yml`](web.cordis.yml) adds the web seam, the local fetch provider, `web_fetch`, and a loopback HTML fixture server for the web-fetch snapshot. ## Protocol channel diff --git a/examples/acp-agent/README.zh.md b/examples/acp-agent/README.zh.md index 5bcd85f2b4..0c5f8866ea 100644 --- a/examples/acp-agent/README.zh.md +++ b/examples/acp-agent/README.zh.md @@ -9,7 +9,7 @@ pnpm run demo:acp # needs DEEPSEEK_API_KEY (repo-root .env or env) pnpm run demo:code-mode acp # same protocol with the Code Mode tool transport ``` -该叶节点加载 ACP 应用、DeepSeek 适配器、受沙箱限制的 bash 与文件系统栈、一次性批准策略、压缩（compaction）、subagent、工作流、钩子、派生会话查询索引和重复守卫。应用为每次 `session/new` 创建一个新 agent，将会话持久化到 JSONL，并保持 stdout 只含协议内容。[`session-query.cordis.yml`](session-query.cordis.yml) 为其专用快照显式选用 workspace 授权的查询工具和通用超时／溢出策略；[`fs.cordis.yml`](fs.cordis.yml) 为文件系统场景添加溢出存储，[`code-mode.cordis.yml`](code-mode.cordis.yml) 则添加 `run_code` 及其生成的 TypeScript SDK。 +该叶节点加载 ACP 应用、DeepSeek 适配器、受沙箱限制的 bash 与文件系统栈、一次性批准策略、压缩（compaction）、subagent、工作流、钩子、派生会话查询索引和重复守卫。应用为每次 `session/new` 创建一个新 agent，将会话持久化到 JSONL，并保持 stdout 只含协议内容。[`session-query.cordis.yml`](session-query.cordis.yml) 为其专用快照显式选用 workspace 授权的查询工具和通用超时／溢出策略；[`fs.cordis.yml`](fs.cordis.yml) 为文件系统场景添加溢出存储，[`code-mode.cordis.yml`](code-mode.cordis.yml) 添加 `run_code` 及其生成的 TypeScript SDK，[`web.cordis.yml`](web.cordis.yml) 则为 web-fetch 快照添加 web seam、本地抓取提供方、`web_fetch` 与一个回环 HTML fixture 服务器。 ## 协议通道 diff --git a/examples/acp-agent/tests/acp.snapshot.ts b/examples/acp-agent/tests/acp.snapshot.ts index 8e74711a57..ed4a7d6a58 100644 --- a/examples/acp-agent/tests/acp.snapshot.ts +++ b/examples/acp-agent/tests/acp.snapshot.ts @@ -41,6 +41,7 @@ const PACKED_CHUNKS_CONFIG = fileURLToPath(new URL('../packed-chunks.cordis.yml' const SESSION_SANDBOX_ROOT_CONFIG = fileURLToPath(new URL('../session-sandbox-root.cordis.yml', import.meta.url)) const RETRY_CONFIG = fileURLToPath(new URL('../retry.cordis.yml', import.meta.url)) const LSP_CONFIG = fileURLToPath(new URL('./lsp.cordis.yml', import.meta.url)) +const WEB_CONFIG = fileURLToPath(new URL('../web.cordis.yml', import.meta.url)) const SNAPSHOTS_DIR = join(dirname(fileURLToPath(import.meta.url)), 'snapshots') const PACKED_CHUNKS_SOURCE = 'hook-cc-pretool-deny' @@ -110,6 +111,12 @@ const SCENARIOS: Scenario[] = [ { name: 'todo-write', hasModelTurn: true, recorded: true }, { name: 'skill-load', hasModelTurn: true, recorded: false, pinsHeader: true, headerClass: 'skill' }, { name: 'lsp-definition', hasModelTurn: true, recorded: false, pinsHeader: true, headerClass: 'lsp', configPath: LSP_CONFIG }, + // web_fetch markdown rendering end to end: the overlay's loopback fixture + // server supplies deterministic HTML (entities, a GFM table, nesting), the + // REAL local fetch provider retrieves it, and the tool result pins the + // turndown conversion. The fetched URL (fixed port) is part of the recorded + // transcript; replay re-executes the real fetch against the same fixture. + { name: 'web-fetch', hasModelTurn: true, recorded: true, pinsHeader: true, headerClass: 'web', configPath: WEB_CONFIG }, { name: 'workspace-edit', hasModelTurn: true, diff --git a/examples/acp-agent/tests/snapshots/web-fetch/input.json b/examples/acp-agent/tests/snapshots/web-fetch/input.json new file mode 100644 index 0000000000..dc1993235d --- /dev/null +++ b/examples/acp-agent/tests/snapshots/web-fetch/input.json @@ -0,0 +1,7 @@ +{ + "steps": [ + { "op": "initialize" }, + { "op": "newSession" }, + { "op": "prompt", "text": "Use the web_fetch tool exactly once to fetch http://127.0.0.1:43117/menu.html, then reply with exactly DONE. Do not describe the content." } + ] +} diff --git a/examples/acp-agent/tests/snapshots/web-fetch/session.jsonl b/examples/acp-agent/tests/snapshots/web-fetch/session.jsonl new file mode 100644 index 0000000000..c6c34bc8e3 --- /dev/null +++ b/examples/acp-agent/tests/snapshots/web-fetch/session.jsonl @@ -0,0 +1,127 @@ +{"type":"session","version":0,"id":"c12fa9af-1042-4a92-9ba4-4a968ff23495","createdAt":1785078727712,"cwd":"/tmp/acp-snap-cwd-hqkZWE","delegationDepth":0} +{"type":"turn/start","seq":0,"time":1785078727718,"data":{"turn":1,"trigger":{"kind":"message","source":{"kind":"user"}}}} +{"type":"user/message","seq":1,"time":1785078727719,"data":{"content":[{"type":"text","text":"Use the web_fetch tool exactly once to fetch http://127.0.0.1:43117/menu.html, then reply with exactly DONE. Do not describe the content."}],"source":{"kind":"user"}},"surfaceOp":"append"} +{"type":"session/title","seq":2,"time":1785078727721,"data":{"title":"Use the web_fetch tool exactly","messageSeqs":[1],"source":{"kind":"fallback"}}} +{"type":"step/start","seq":3,"time":1785078727730,"data":{"turn":1,"step":1}} +{"type":"request/header","seq":4,"time":1785078727731,"data":{"header":{"config":{"provider":"deepseek","model":"deepseek-v4-pro"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}} +{"type":"assistant/chunk","seq":5,"time":1785078728804,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"reasoning"}}} +{"type":"assistant/chunk","seq":6,"time":1785078728805,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"The"}}} +{"type":"assistant/chunk","seq":7,"time":1785078728943,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" user"}}} +{"type":"assistant/chunk","seq":8,"time":1785078728989,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" wants"}}} +{"type":"assistant/chunk","seq":9,"time":1785078728989,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" me"}}} +{"type":"assistant/chunk","seq":10,"time":1785078728989,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" to"}}} +{"type":"assistant/chunk","seq":11,"time":1785078728990,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" use"}}} +{"type":"assistant/chunk","seq":12,"time":1785078728990,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" the"}}} +{"type":"assistant/chunk","seq":13,"time":1785078728990,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" web"}}} +{"type":"assistant/chunk","seq":14,"time":1785078729038,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"_f"}}} +{"type":"assistant/chunk","seq":15,"time":1785078729038,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"etch"}}} +{"type":"assistant/chunk","seq":16,"time":1785078729039,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" tool"}}} +{"type":"assistant/chunk","seq":17,"time":1785078729039,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" exactly"}}} +{"type":"assistant/chunk","seq":18,"time":1785078729085,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" once"}}} +{"type":"assistant/chunk","seq":19,"time":1785078729086,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" to"}}} +{"type":"assistant/chunk","seq":20,"time":1785078729086,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" fetch"}}} +{"type":"assistant/chunk","seq":21,"time":1785078729086,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" http"}}} +{"type":"assistant/chunk","seq":22,"time":1785078729086,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"://"}}} +{"type":"assistant/chunk","seq":23,"time":1785078729086,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"127"}}} +{"type":"assistant/chunk","seq":24,"time":1785078729132,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"."}}} +{"type":"assistant/chunk","seq":25,"time":1785078729133,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"0"}}} +{"type":"assistant/chunk","seq":26,"time":1785078729133,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"."}}} +{"type":"assistant/chunk","seq":27,"time":1785078729133,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"0"}}} +{"type":"assistant/chunk","seq":28,"time":1785078729133,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"."}}} +{"type":"assistant/chunk","seq":29,"time":1785078729133,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"1"}}} +{"type":"assistant/chunk","seq":30,"time":1785078729182,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":":"}}} +{"type":"assistant/chunk","seq":31,"time":1785078729182,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"431"}}} +{"type":"assistant/chunk","seq":32,"time":1785078729183,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"17"}}} +{"type":"assistant/chunk","seq":33,"time":1785078729183,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"/m"}}} +{"type":"assistant/chunk","seq":34,"time":1785078729183,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"enu"}}} +{"type":"assistant/chunk","seq":35,"time":1785078729183,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":".html"}}} +{"type":"assistant/chunk","seq":36,"time":1785078729230,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":","}}} +{"type":"assistant/chunk","seq":37,"time":1785078729230,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" then"}}} +{"type":"assistant/chunk","seq":38,"time":1785078729230,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" reply"}}} +{"type":"assistant/chunk","seq":39,"time":1785078729230,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" with"}}} +{"type":"assistant/chunk","seq":40,"time":1785078729230,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" exactly"}}} +{"type":"assistant/chunk","seq":41,"time":1785078729231,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" \""}}} +{"type":"assistant/chunk","seq":42,"time":1785078729276,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"D"}}} +{"type":"assistant/chunk","seq":43,"time":1785078729277,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"ONE"}}} +{"type":"assistant/chunk","seq":44,"time":1785078729277,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"\"."}}} +{"type":"assistant/chunk","seq":45,"time":1785078729277,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" Let"}}} +{"type":"assistant/chunk","seq":46,"time":1785078729322,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" me"}}} +{"type":"assistant/chunk","seq":47,"time":1785078729323,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" do"}}} +{"type":"assistant/chunk","seq":48,"time":1785078729323,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" that"}}} +{"type":"assistant/chunk","seq":49,"time":1785078729323,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"."}}} +{"type":"assistant/chunk","seq":50,"time":1785078729463,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":1,"blockType":"tool-call"}}} +{"type":"assistant/chunk","seq":51,"time":1785078729464,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":""}}} +{"type":"assistant/chunk","seq":52,"time":1785078729511,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"{"}}} +{"type":"assistant/chunk","seq":53,"time":1785078729511,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"\""}}} +{"type":"assistant/chunk","seq":54,"time":1785078729511,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"url"}}} +{"type":"assistant/chunk","seq":55,"time":1785078729511,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"\""}}} +{"type":"assistant/chunk","seq":56,"time":1785078729511,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":": "}}} +{"type":"assistant/chunk","seq":57,"time":1785078729557,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"\""}}} +{"type":"assistant/chunk","seq":58,"time":1785078729557,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"http"}}} +{"type":"assistant/chunk","seq":59,"time":1785078729557,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"://"}}} +{"type":"assistant/chunk","seq":60,"time":1785078729558,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"127"}}} +{"type":"assistant/chunk","seq":61,"time":1785078729604,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"."}}} +{"type":"assistant/chunk","seq":62,"time":1785078729604,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"0"}}} +{"type":"assistant/chunk","seq":63,"time":1785078729604,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"."}}} +{"type":"assistant/chunk","seq":64,"time":1785078729604,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"0"}}} +{"type":"assistant/chunk","seq":65,"time":1785078729604,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"."}}} +{"type":"assistant/chunk","seq":66,"time":1785078729605,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"1"}}} +{"type":"assistant/chunk","seq":67,"time":1785078729651,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":":"}}} +{"type":"assistant/chunk","seq":68,"time":1785078729652,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"431"}}} +{"type":"assistant/chunk","seq":69,"time":1785078729652,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"17"}}} +{"type":"assistant/chunk","seq":70,"time":1785078729652,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"/m"}}} +{"type":"assistant/chunk","seq":71,"time":1785078729652,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"enu"}}} +{"type":"assistant/chunk","seq":72,"time":1785078729652,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":".html"}}} +{"type":"assistant/chunk","seq":73,"time":1785078729697,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"\""}}} +{"type":"assistant/chunk","seq":74,"time":1785078729698,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"}"}}} +{"type":"assistant/chunk","seq":75,"time":1785078729803,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":0,"block":{"type":"reasoning","text":"The user wants me to use the web_fetch tool exactly once to fetch http://127.0.0.1:43117/menu.html, then reply with exactly \"DONE\". Let me do that."}}}} +{"type":"assistant/chunk","seq":76,"time":1785078729803,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":1,"block":{"type":"tool-call","id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","arguments":"{\"url\": \"http://127.0.0.1:43117/menu.html\"}"}}}} +{"type":"assistant/chunk","seq":77,"time":1785078729803,"data":{"turn":1,"step":1,"chunk":{"type":"usage","usage":{"inputTokens":5405,"outputTokens":103,"cacheReadTokens":0,"reasoningTokens":44}}}} +{"type":"assistant/chunk","seq":78,"time":1785078729804,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}} +{"type":"assistant/message","seq":79,"time":1785078729807,"data":{"turn":1,"step":1,"content":[{"type":"reasoning","text":"The user wants me to use the web_fetch tool exactly once to fetch http://127.0.0.1:43117/menu.html, then reply with exactly \"DONE\". Let me do that."},{"type":"tool-call","id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","arguments":"{\"url\": \"http://127.0.0.1:43117/menu.html\"}"}],"provenance":{"provider":"deepseek","model":"deepseek-v4-pro"},"usage":{"inputTokens":5405,"outputTokens":103,"cacheReadTokens":0,"reasoningTokens":44}},"sourceEventSeqs":[5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78],"surfaceOp":"append"} +{"type":"tool/call","seq":80,"time":1785078729809,"data":{"turn":1,"step":1,"callId":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","arguments":"{\"url\": \"http://127.0.0.1:43117/menu.html\"}"}} +{"type":"tool/result","seq":81,"time":1785078729843,"data":{"turn":1,"step":1,"callId":"call_00_sxjOyfDYN07koiE7jiIa5326","content":[{"type":"text","text":"Fetched http://127.0.0.1:43117/menu.html (HTTP 200)\n\nMenu\n\n# Café menu\n\nPrices include **service & _tax_** — updated daily.\n\n- Espresso\n- Flat white\n\n| Drink | Price |\n| --- | --- |\n| Espresso | €2 |\n| Flat white | €3 |\n\nSee [today’s specials](https://fixture.invalid/specials)."}],"isError":false},"sourceEventSeqs":[80],"surfaceOp":"append"} +{"type":"step/end","seq":82,"time":1785078729847,"data":{"turn":1,"step":1}} +{"type":"step/start","seq":83,"time":1785078729848,"data":{"turn":1,"step":2}} +{"type":"assistant/chunk","seq":84,"time":1785078730611,"data":{"turn":1,"step":2,"chunk":{"type":"block-start","index":0,"blockType":"reasoning"}}} +{"type":"assistant/chunk","seq":85,"time":1785078730612,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"The"}}} +{"type":"assistant/chunk","seq":86,"time":1785078730770,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" user"}}} +{"type":"assistant/chunk","seq":87,"time":1785078730824,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" asked"}}} +{"type":"assistant/chunk","seq":88,"time":1785078730825,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" me"}}} +{"type":"assistant/chunk","seq":89,"time":1785078730825,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" to"}}} +{"type":"assistant/chunk","seq":90,"time":1785078730825,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" fetch"}}} +{"type":"assistant/chunk","seq":91,"time":1785078730861,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" the"}}} +{"type":"assistant/chunk","seq":92,"time":1785078730862,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" URL"}}} +{"type":"assistant/chunk","seq":93,"time":1785078730909,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":","}}} +{"type":"assistant/chunk","seq":94,"time":1785078730956,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" then"}}} +{"type":"assistant/chunk","seq":95,"time":1785078731002,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" reply"}}} +{"type":"assistant/chunk","seq":96,"time":1785078731003,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" with"}}} +{"type":"assistant/chunk","seq":97,"time":1785078731003,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" exactly"}}} +{"type":"assistant/chunk","seq":98,"time":1785078731003,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" \""}}} +{"type":"assistant/chunk","seq":99,"time":1785078731050,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"D"}}} +{"type":"assistant/chunk","seq":100,"time":1785078731050,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"ONE"}}} +{"type":"assistant/chunk","seq":101,"time":1785078731050,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"\"."}}} +{"type":"assistant/chunk","seq":102,"time":1785078731050,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" I"}}} +{"type":"assistant/chunk","seq":103,"time":1785078731050,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"'ve"}}} +{"type":"assistant/chunk","seq":104,"time":1785078731051,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" fetched"}}} +{"type":"assistant/chunk","seq":105,"time":1785078731097,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" it"}}} +{"type":"assistant/chunk","seq":106,"time":1785078731140,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"."}}} +{"type":"assistant/chunk","seq":107,"time":1785078731141,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" Now"}}} +{"type":"assistant/chunk","seq":108,"time":1785078731141,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" I"}}} +{"type":"assistant/chunk","seq":109,"time":1785078731141,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" just"}}} +{"type":"assistant/chunk","seq":110,"time":1785078731189,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" reply"}}} +{"type":"assistant/chunk","seq":111,"time":1785078731189,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" with"}}} +{"type":"assistant/chunk","seq":112,"time":1785078731235,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" \""}}} +{"type":"assistant/chunk","seq":113,"time":1785078731235,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"D"}}} +{"type":"assistant/chunk","seq":114,"time":1785078731235,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"ONE"}}} +{"type":"assistant/chunk","seq":115,"time":1785078731235,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"\"."}}} +{"type":"assistant/chunk","seq":116,"time":1785078731235,"data":{"turn":1,"step":2,"chunk":{"type":"block-start","index":1,"blockType":"text"}}} +{"type":"assistant/chunk","seq":117,"time":1785078731236,"data":{"turn":1,"step":2,"chunk":{"type":"text-delta","index":1,"text":"D"}}} +{"type":"assistant/chunk","seq":118,"time":1785078731282,"data":{"turn":1,"step":2,"chunk":{"type":"text-delta","index":1,"text":"ONE"}}} +{"type":"assistant/chunk","seq":119,"time":1785078731282,"data":{"turn":1,"step":2,"chunk":{"type":"block-end","index":0,"block":{"type":"reasoning","text":"The user asked me to fetch the URL, then reply with exactly \"DONE\". I've fetched it. Now I just reply with \"DONE\"."}}}} +{"type":"assistant/chunk","seq":120,"time":1785078731282,"data":{"turn":1,"step":2,"chunk":{"type":"block-end","index":1,"block":{"type":"text","text":"DONE"}}}} +{"type":"assistant/chunk","seq":121,"time":1785078731282,"data":{"turn":1,"step":2,"chunk":{"type":"usage","usage":{"inputTokens":239,"outputTokens":34,"cacheReadTokens":5376,"reasoningTokens":31}}}} +{"type":"assistant/chunk","seq":122,"time":1785078731282,"data":{"turn":1,"step":2,"chunk":{"type":"finish","reason":{"kind":"stop"}}}} +{"type":"assistant/message","seq":123,"time":1785078731283,"data":{"turn":1,"step":2,"content":[{"type":"reasoning","text":"The user asked me to fetch the URL, then reply with exactly \"DONE\". I've fetched it. Now I just reply with \"DONE\"."},{"type":"text","text":"DONE"}],"provenance":{"provider":"deepseek","model":"deepseek-v4-pro"},"usage":{"inputTokens":239,"outputTokens":34,"cacheReadTokens":5376,"reasoningTokens":31}},"sourceEventSeqs":[84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122],"surfaceOp":"append"} +{"type":"step/end","seq":124,"time":1785078731286,"data":{"turn":1,"step":2}} +{"type":"turn/end","seq":125,"time":1785078731286,"data":{"turn":1,"reason":{"kind":"completed"}}} diff --git a/examples/acp-agent/tests/snapshots/web-fetch/stdout.expected.jsonl b/examples/acp-agent/tests/snapshots/web-fetch/stdout.expected.jsonl new file mode 100644 index 0000000000..82ae8907ca --- /dev/null +++ b/examples/acp-agent/tests/snapshots/web-fetch/stdout.expected.jsonl @@ -0,0 +1,4 @@ +{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentInfo":{"name":"deepseek-harness-acp","version":"0.0.1"},"agentCapabilities":{"promptCapabilities":{"image":false,"audio":false,"embeddedContext":false}},"authMethods":[]}} +{"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}"}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"DONE"}}}} +{"jsonrpc":"2.0","id":3,"result":{"stopReason":"end_turn"}} diff --git a/examples/acp-agent/tests/snapshots/web-fetch/system-prompt.expected.md b/examples/acp-agent/tests/snapshots/web-fetch/system-prompt.expected.md new file mode 100644 index 0000000000..45705db0a5 --- /dev/null +++ b/examples/acp-agent/tests/snapshots/web-fetch/system-prompt.expected.md @@ -0,0 +1,27 @@ +You are an AI agent powered by the DeepSeek Harness SDK. + +You are a coding assistant powered by the deepseek-v4-pro model. Your working directory is {{cwd}}. Your bash tool runs under a file sandbox — a `[sandbox: file access denied …]` result is policy, not a command bug. + +Verify your work by running the code or tests. Keep answers brief and factual. + + +Use the read tool — not shell commands like cat — to inspect text files. Results include line numbers. Use offset and limit to continue reading large files. + +Use the write tool to create files or completely replace file contents. Existing files are overwritten, so read an existing file first (the default fs-policy requires it) and prefer edit for targeted changes. + +Use the edit tool for targeted changes to existing UTF-8 text files. It replaces literal old_string with new_string; by default old_string must appear exactly once. If old_string appears multiple times, provide a more specific old_string or set replace_all to true. Read the file first (the default fs-policy requires it), unless you just created or edited it in this session. + +Check the [exit code: N] marker on every bash result; investigate failures before moving on. + +Track every background task id you start. You are notified in-session when a task finishes — do not busy-poll or sleep on one; keep working on independent steps and do not duplicate a running task's work. Before giving a final answer, collect every still-relevant task with task_output (set wait: true only when you are genuinely blocked on it), and task_kill tasks that stopped mattering. + +Use the web_fetch tool to retrieve the content of a specific HTTP(S) URL (for example a result from web_search). It returns the page content decoded to text. Cite the URL as a markdown link when you use its content. + +Use goal tools for one long-running completion objective in the current session. create_goal may infer goal intent from a direct human request in any language; do not create a goal for routine single-turn work. Call get_goal before update_goal and copy its exact goal_id and revision. After session resume or fork, an active goal is disarmed: when a human asks to continue or resume in any wording or language, use update_goal action resume to rearm it. Mark complete only when the objective is actually achieved. Mark blocked only after the same blocking condition persists for at least 3 consecutive goal rounds, and report that concrete condition in blocked_reason; difficulty, uncertainty, or useful remaining work is not blocked. + +Approval prompts are disabled in this session: actions that require approval are rejected automatically — do not request sandbox escalation (do not set `sandbox_permissions`). + + +Use the workflow tool ONLY when the user explicitly asks for a workflow or for large multi-agent orchestration: you write a JavaScript script (the tool description documents the exact format) that fans work out across many subagents with phases and structured results. For one or two delegations, prefer plain subagent calls. + +Use the ralph tool ONLY when the direct human explicitly asks for a Ralph loop or fresh-agent iterative execution. Each Ralph round starts a fresh child with no conversation seed and uses the shared workspace as durable memory. Completion and blockers are worker reports, not independent evaluation. Use same-session goal tools for ordinary long-running objectives, and plain subagents or workflows for bounded delegation and fan-out. diff --git a/examples/acp-agent/tests/snapshots/web-fetch/tool-schemas.expected.json b/examples/acp-agent/tests/snapshots/web-fetch/tool-schemas.expected.json new file mode 100644 index 0000000000..1ee86b38ba --- /dev/null +++ b/examples/acp-agent/tests/snapshots/web-fetch/tool-schemas.expected.json @@ -0,0 +1,489 @@ +{ + "initial": [ + { + "name": "bash", + "description": "Execute a bash command (`bash -c`) and return its stdout/stderr. Each call runs in a fresh shell: no state (cwd, variables, functions) persists between calls — pass `workdir` instead of using `cd`. Non-zero exits are reported as `[exit code: N]`. Current harness environment facts are exposed through managed `$DSH_*` variables; inspect them when needed. Commands may run under a file sandbox; a blocked file operation is reported as `[sandbox: file access denied under mode]` — a policy denial, not a bug in the command; do not retry another way. Long output is truncated to its tail; the full output is saved to a file whose path is reported when available. Set `run_in_background: true` for long-running commands: the call returns a task id immediately; read its output with `task_output` and stop it with `task_kill`. Attempting a command the sandbox may deny is safe and expected: run it and read the marker rather than assuming the denial. When a command is denied and a wider mode would let it succeed, escalate immediately in the same turn — the one sanctioned exception to a denial: retry the exact same command once with `sandbox_permissions` (the narrowest wider mode that suffices) plus a one-sentence `justification`. Do not detour through chat to ask permission first — the approval prompt raised by that retry is how the user consents. If the session states approval prompts are disabled, there is no exception: a denial is final — do not set `sandbox_permissions`. Never escalate speculatively: ground the request in a real denial — normally the one this command just hit; escalating up front is fine only when this session already denied the same access. A rejected escalation is final for that command — stop and explain, never work around it — but it does not forbid attempting or escalating other commands later.", + "parameters": { + "type": "object", + "properties": { + "command": { + "type": "string", + "description": "The bash command to execute." + }, + "description": { + "type": "string", + "description": "Clear, concise description of what this command does in active voice, 5-10 words (shown in the UI). Examples: \"ls\" → \"List files in current directory\"; \"git status\" → \"Show working tree status\"; \"npm install\" → \"Install package dependencies\"." + }, + "timeoutMs": { + "type": "number", + "description": "Timeout in milliseconds. The executor applies its configured default and cap, and kills the command on expiry." + }, + "workdir": { + "type": "string", + "description": "Working directory for this command. Defaults to the session workspace; a relative path is resolved against it." + }, + "run_in_background": { + "type": "boolean", + "description": "Run in the background and return a task id immediately (collect with task_output, stop with task_kill). No timeout applies." + }, + "sandbox_permissions": { + "type": "string", + "description": "The wider sandbox mode this command needs. Only valid as a one-shot retry of a command the sandbox just denied; requires justification and user approval.", + "enum": [ + "workspace-write", + "danger-full-access" + ] + }, + "justification": { + "type": "string", + "description": "Required with sandbox_permissions: one sentence for the user explaining why this exact command needs the wider access." + } + }, + "required": [ + "command", + "description" + ] + } + }, + { + "name": "create_goal", + "description": "Create one persisted same-session completion goal when the current direct human request is a long-running objective that should continue across autonomous goal rounds. You may infer that intent without requiring the user to say \"create a goal\". Do not use this for trivial single-turn work. Execution rejects non-human and subagent authority.", + "parameters": { + "type": "object", + "properties": { + "objective": { + "type": "string", + "description": "The concrete completion objective inferred from the direct human request." + }, + "max_goal_rounds": { + "type": "number", + "description": "Optional positive safe-integer limit on automatic continuation rounds." + } + }, + "required": [ + "objective" + ] + } + }, + { + "name": "edit", + "description": "Edit an existing UTF-8 text file by replacing literal text.", + "parameters": { + "type": "object", + "properties": { + "file_path": { + "type": "string", + "description": "Path to edit, resolved by the filesystem backend." + }, + "old_string": { + "type": "string", + "description": "Literal text to replace. Must match exactly." + }, + "new_string": { + "type": "string", + "description": "Literal replacement text. Use an empty string to delete the match." + }, + "replace_all": { + "type": "boolean", + "description": "Replace all matches. Defaults to false; when false, old_string must appear exactly once." + }, + "sandbox_permissions": { + "type": "string", + "description": "The wider sandbox mode this file operation needs. Only valid as a one-shot retry of an operation the sandbox just denied; requires justification and user approval.", + "enum": [ + "workspace-write", + "danger-full-access" + ] + }, + "justification": { + "type": "string", + "description": "Required with sandbox_permissions: one sentence for the user explaining why this exact file operation needs the wider access." + } + }, + "required": [ + "file_path", + "old_string", + "new_string" + ] + } + }, + { + "name": "get_goal", + "description": "Read the current same-session goal, including its exact id/revision, objective, phase, completed continuation rounds, round limit, blocker reason when present, and whether another continuation is armed. Call this before updating a goal.", + "parameters": { + "type": "object", + "properties": {} + } + }, + { + "name": "ralph", + "description": "Run a foreground fresh-agent Ralph loop toward one immutable objective. Use only when the direct human explicitly asks for Ralph or fresh-agent iteration. Each round opens a new child with no parent conversation or prior child session; the shared workspace is long-term memory, and only a bounded structured report crosses rounds. The call returns when a worker reports completion or a concrete blocker, or at the round limit. Ordinary long-running same-session work belongs to goal tools.", + "parameters": { + "type": "object", + "properties": { + "objective": { + "type": "string", + "description": "The immutable completion objective for every fresh Ralph round." + }, + "maxRounds": { + "type": "number", + "description": "Optional positive safe-integer round cap, bounded by the deployment ceiling." + } + }, + "required": [ + "objective" + ] + } + }, + { + "name": "read", + "description": "Read a UTF-8 text file and return line-numbered content.", + "parameters": { + "type": "object", + "properties": { + "file_path": { + "type": "string", + "description": "Path to read, resolved by the filesystem backend." + }, + "offset": { + "type": "number", + "description": "1-based first line to return. Defaults to 1." + }, + "limit": { + "type": "number", + "description": "Maximum number of lines to return. Defaults to 2000." + } + }, + "required": [ + "file_path" + ] + } + }, + { + "name": "skill", + "description": "Load the full instructions for an available skill. Call this with the exact skill name from the session skill catalog before acting on a task that names or clearly matches that skill.", + "parameters": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "The exact skill name from the available skills list." + } + }, + "required": [ + "name" + ] + } + }, + { + "name": "subagent", + "description": "Delegate a self-contained task to a subagent (a separate agent that works in its own context) and return its final result. Use this to offload focused, independent work — research, a scoped implementation, an analysis — so it does not consume this conversation's context. The subagent runs to completion and you receive only its final answer, not its intermediate steps. Give it a complete, standalone prompt: it does not see this conversation. Set `run_in_background: true` to return a task id; collect with `task_output` and stop with `task_kill`.", + "parameters": { + "type": "object", + "properties": { + "description": { + "type": "string", + "description": "A short (3-5 word) description of the delegated task, for display." + }, + "prompt": { + "type": "string", + "description": "The complete, self-contained task for the subagent. It does not share this conversation's context, so include everything it needs." + }, + "run_in_background": { + "type": "boolean", + "description": "Run as a background task and return its id; collect with task_output or stop with task_kill." + } + }, + "required": [ + "description", + "prompt" + ] + } + }, + { + "name": "subagent_fork", + "description": "Delegate a task to a subagent that inherits this conversation: a child agent seeded with all completed turns so far (it does not see the current in-flight turn), returning only its final result. Use this when the subtask builds on this conversation's context — a follow-up analysis, a review, a continuation — without consuming this conversation's context for the work itself. You receive only its final answer, not its intermediate steps. Set `run_in_background: true` to return a task id; collect with `task_output` and stop with `task_kill`.", + "parameters": { + "type": "object", + "properties": { + "description": { + "type": "string", + "description": "A short (3-5 word) description of the delegated task, for display." + }, + "prompt": { + "type": "string", + "description": "The task for the subagent. It already sees this conversation's completed turns, so build on them freely and state only what is new." + }, + "run_in_background": { + "type": "boolean", + "description": "Run as a background task and return its id; collect with task_output or stop with task_kill." + } + }, + "required": [ + "description", + "prompt" + ] + } + }, + { + "name": "task_kill", + "description": "Request cancellation of a running background task by task id. Returns immediately; the task settles as killed once its work actually stops.", + "parameters": { + "type": "object", + "properties": { + "task_id": { + "type": "string", + "description": "Task id returned by the tool that started the background work." + }, + "reason": { + "type": "string", + "description": "Optional short reason, recorded in the log and forwarded to the task." + } + }, + "required": [ + "task_id" + ] + } + }, + { + "name": "task_list", + "description": "List your background tasks (running and finished) with their ids, kinds, and statuses.", + "parameters": { + "type": "object", + "properties": {} + } + }, + { + "name": "task_output", + "description": "Read a background task. Stream tasks return only output since the previous read; final-output tasks return their result after settlement. Every response ends with `[status: ...]`. Reads are non-blocking unless `wait: true`, which waits up to the configured cap.", + "parameters": { + "type": "object", + "properties": { + "task_id": { + "type": "string", + "description": "Task id returned by the tool that started the background work." + }, + "wait": { + "type": "boolean", + "description": "Block until the task reaches a terminal status or the timeout expires. A timed-out wait returns [status: running] and leaves the task alive." + }, + "timeout_ms": { + "type": "number", + "description": "Max wait in milliseconds (only meaningful with wait: true). Defaults to the configured wait timeout; capped by the configured maximum." + } + }, + "required": [ + "task_id" + ] + } + }, + { + "name": "todo_write", + "description": "Record and update a structured task list for the current work. Send the ENTIRE list every call — it REPLACES the previous list (there are no partial updates, no per-item edits). Use it to plan multi-step work and show progress: add one todo per concrete step before you start. Keep AT MOST ONE todo `in_progress` at a time; while work remains, exactly one active task should be `in_progress`. Mark a todo `completed` the moment it is done (do not batch completions), and allow no `in_progress` item only once all work is complete. Skip the list for trivial single-step tasks. Statuses: `pending` (not started), `in_progress` (being worked on now), `completed` (finished).", + "parameters": { + "type": "object", + "properties": { + "todos": { + "type": "array", + "description": "The COMPLETE task list, replacing any previous list.", + "items": { + "type": "object", + "additionalProperties": true, + "properties": { + "content": { + "type": "string", + "description": "What the task is — a short imperative line." + }, + "status": { + "type": "string", + "description": "pending (not started) | in_progress (now) | completed (done).", + "enum": [ + "pending", + "in_progress", + "completed" + ] + } + }, + "required": [ + "content", + "status" + ] + } + } + }, + "required": [ + "todos" + ] + } + }, + { + "name": "update_goal", + "description": "Update the exact current goal revision. edit, pause, and resume require a direct top-level human request. During an automatic continuation of the current goal, complete and blocked are also allowed. blocked is rejected before the configured minimum round count; the model remains responsible for judging that the same condition persisted across those rounds and must explain it in blocked_reason.", + "parameters": { + "type": "object", + "properties": { + "goal_id": { + "type": "string", + "description": "Exact id returned by get_goal." + }, + "revision": { + "type": "number", + "description": "Exact positive revision returned by get_goal." + }, + "action": { + "type": "string", + "description": "edit | pause | resume | complete | blocked", + "enum": [ + "edit", + "pause", + "resume", + "complete", + "blocked" + ] + }, + "objective": { + "type": "string", + "description": "Replacement objective; valid only with action edit." + }, + "max_goal_rounds": { + "type": "number", + "description": "Replacement cap; valid only with action edit." + }, + "blocked_reason": { + "type": "string", + "description": "Concrete blocking condition; required only with action blocked." + } + }, + "required": [ + "goal_id", + "revision", + "action" + ] + } + }, + { + "name": "web_fetch", + "description": "Fetch the content of a specific HTTP(S) URL and return it decoded to text.", + "parameters": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "The HTTP(S) URL to fetch." + } + }, + "required": [ + "url" + ] + } + }, + { + "name": "workflow", + "description": "Run a JavaScript workflow script that orchestrates subagents at scale. Use this for work that fans out across many independent pieces — an audit over many files, a migration, multi-angle research, adversarial verification of findings — where you write the orchestration as a script instead of delegating turn by turn.\n\nThe workflow's identity rides the `meta` parameter as JSON: required `name` (short kebab-case) and `description` strings, optional `whenToUse` string and `phases` array (`{title, detail?, provider?, model?}`). The `script` parameter is the plain JavaScript body ONLY (NOT TypeScript, and NO `export const meta` statement — meta is a parameter, not code), running with top-level await; end with `return ` — the value must be JSON-serializable and is this tool's result.\n\nScript-body hooks:\n- `agent(prompt, opts?): Promise` — run one subagent to completion. Without `opts.schema` it resolves to the child's final text; with `opts.schema` (an object-rooted JSON Schema using ONLY type/properties/required/additionalProperties/items/enum/const/oneOf — no pattern/format/numeric bounds) it resolves to the validated object. Resolves `null` when the child fails (filter with `.filter(Boolean)`). Other opts: `label` (display), `phase` (progress group), and independent `provider`/`model` LLM target overrides (either may be provided alone). Anything else (`effort`/`isolation`/`agentType`) is rejected loudly.\n- `pipeline(items, ...stages): Promise` — run each item through the stages independently with NO barrier between stages (prefer this for multi-stage work). Each stage receives `(prev, item, index)`. An ordinary stage throw drops that ITEM to `null` and skips its remaining stages.\n- `parallel(thunks): Promise` — run zero-argument functions concurrently and await ALL of them (a barrier; use only when a stage genuinely needs every prior result together). A throwing thunk resolves to `null`.\n- `phase(title)` — start a progress phase; `log(message)` — narrate progress; `args` — the tool call's `args` input, verbatim.\n\nMisused hooks (bad arguments, unknown options, unsupported schemas, tripped caps) throw errors that ALWAYS kill the script — they never dissolve into a per-item `null`.\n\nConstraints: concurrency and total-agent caps apply; no filesystem, network, timers, or Node.js APIs are provided — the agents do the work, the script only coordinates them. The run executes in the foreground: this call returns when the whole script finishes.", + "parameters": { + "type": "object", + "properties": { + "script": { + "type": "string", + "description": "The plain-JS workflow script body (top-level await allowed; NO `export const meta` statement; end with `return `)." + }, + "meta": { + "type": "object", + "description": "The workflow identity block (plain JSON — never code).", + "additionalProperties": true, + "properties": { + "name": { + "type": "string", + "description": "Short kebab-case workflow name." + }, + "description": { + "type": "string", + "description": "One-line description of what the workflow does." + }, + "whenToUse": { + "type": "string", + "description": "Optional guidance on when this workflow applies." + }, + "phases": { + "type": "array", + "description": "Optional phase declarations matched by phase() calls.", + "items": { + "type": "object", + "additionalProperties": true, + "properties": { + "title": { + "type": "string", + "description": "The phase title phase() calls match by exact string." + }, + "detail": { + "type": "string", + "description": "Optional one-line description of the phase." + }, + "provider": { + "type": "string", + "description": "Optional provider override this phase is expected to use." + }, + "model": { + "type": "string", + "description": "Optional model override this phase is expected to use." + } + }, + "required": [ + "title" + ] + } + } + }, + "required": [ + "name", + "description" + ] + }, + "args": { + "type": "object", + "description": "Optional JSON input exposed to the script as the `args` global (wrap a bare list as a field, e.g. {\"files\": [...]}).", + "additionalProperties": true + } + }, + "required": [ + "script", + "meta" + ] + } + }, + { + "name": "write", + "description": "Create or fully replace a UTF-8 text file.", + "parameters": { + "type": "object", + "properties": { + "file_path": { + "type": "string", + "description": "Path to write, resolved by the filesystem backend." + }, + "content": { + "type": "string", + "description": "Full UTF-8 text content to write." + }, + "sandbox_permissions": { + "type": "string", + "description": "The wider sandbox mode this file operation needs. Only valid as a one-shot retry of an operation the sandbox just denied; requires justification and user approval.", + "enum": [ + "workspace-write", + "danger-full-access" + ] + }, + "justification": { + "type": "string", + "description": "Required with sandbox_permissions: one sentence for the user explaining why this exact file operation needs the wider access." + } + }, + "required": [ + "file_path", + "content" + ] + } + } + ], + "changes": [] +} diff --git a/examples/acp-agent/web-fetch-fixture-server.mjs b/examples/acp-agent/web-fetch-fixture-server.mjs new file mode 100644 index 0000000000..34a45fdd15 --- /dev/null +++ b/examples/acp-agent/web-fetch-fixture-server.mjs @@ -0,0 +1,52 @@ +/** + * Deterministic loopback HTTP fixture for the web-fetch snapshot scenario: a + * small HTML page (headings, named entities, a GFM table, nested formatting) + * on a fixed port, so recording and keyless replay drive the REAL + * `dsh-web-fetch-local` transport and `dsh-tool-web` markdown rendering + * without external network. The port is fixed because the fetched URL is part + * of the recorded model transcript. + */ +import { createServer } from 'node:http' + +/** Fixed loopback port the scenario prompt points `web_fetch` at. */ +const PORT = 43117 + +const PAGE = ` +Menu + +

    Café menu

    +

    Prices include service & tax — updated daily.

    +
    • Espresso
    • Flat white
    +
    DrinkPrice
    Espresso€2
    Flat white€3
    +

    See today’s specials.

    + +` + +/** Cordis plugin name. */ +export const name = 'web-fetch-fixture-server' + +/** + * Start the fixture server on 127.0.0.1 and register its shutdown. + * @param ctx - Cordis context; the effect disposes the server with the fiber. + */ +export async function apply(ctx) { + const server = createServer((req, res) => { + if (req.url === '/menu.html') { + res.writeHead(200, { 'content-type': 'text/html; charset=utf-8' }) + res.end(PAGE) + return + } + res.writeHead(404, { 'content-type': 'text/plain; charset=utf-8' }) + res.end('not found') + }) + await new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(PORT, '127.0.0.1', () => resolve(undefined)) + }) + // The fixture must never hold the process open past protocol shutdown. + server.unref() + ctx.effect(() => () => { + server.close() + server.closeAllConnections() + }, 'web-fetch-fixture-server') +} diff --git a/examples/acp-agent/web.cordis.snapshot.yml b/examples/acp-agent/web.cordis.snapshot.yml new file mode 100644 index 0000000000..015e67e221 --- /dev/null +++ b/examples/acp-agent/web.cordis.snapshot.yml @@ -0,0 +1,31 @@ +# Keyless replay counterpart to web.cordis.yml: the web stack and loopback +# fixture server stay real (the tool call re-executes the actual HTTP fetch and +# markdown rendering); only the model adapter is replaced by replay. +- id: base + name: '@cordisjs/plugin-include' + config: + path: ./cordis.yml + patches: + - id: llm-deepseek + name: '@deepseek-ai/dsh-llm-deepseek' + disabled: true + - insert: + - id: web + name: '@deepseek-ai/dsh-web' + - id: web-fetch-local + name: '@deepseek-ai/dsh-web-fetch-local' + - id: web-fetch-fixture + name: './web-fetch-fixture-server.mjs' + - id: tool-web + name: '@deepseek-ai/dsh-tool-web' + config: + search: false + - id: llm-replay + name: '@deepseek-ai/dsh-llm-replay' + config: + providers: + - id: deepseek + name: DeepSeek + models: + - id: deepseek-v4-flash + - id: deepseek-v4-pro diff --git a/examples/acp-agent/web.cordis.yml b/examples/acp-agent/web.cordis.yml new file mode 100644 index 0000000000..1ed0b3efba --- /dev/null +++ b/examples/acp-agent/web.cordis.yml @@ -0,0 +1,21 @@ +# Web-fetch composition for the web-fetch snapshot scenario: the web seam, the +# real local HTTP fetch provider, the model-facing web tools (fetch only, so +# the pinned header carries exactly the surface under test), and the loopback +# fixture server the scenario prompt fetches — deterministic content, no +# external network, in recording and replay alike. +- id: base + name: '@cordisjs/plugin-include' + config: + path: ./cordis.yml + patches: + - insert: + - id: web + name: '@deepseek-ai/dsh-web' + - id: web-fetch-local + name: '@deepseek-ai/dsh-web-fetch-local' + - id: web-fetch-fixture + name: './web-fetch-fixture-server.mjs' + - id: tool-web + name: '@deepseek-ai/dsh-tool-web' + config: + search: false diff --git a/examples/package.json b/examples/package.json index 395c135a2d..3d0cc13718 100644 --- a/examples/package.json +++ b/examples/package.json @@ -63,6 +63,7 @@ "@deepseek-ai/dsh-tool-session-query": "workspace:*", "@deepseek-ai/dsh-tool-subagent": "workspace:*", "@deepseek-ai/dsh-tool-todo": "workspace:*", + "@deepseek-ai/dsh-tool-web": "workspace:*", "@deepseek-ai/dsh-tool-workflow": "workspace:*", "@deepseek-ai/dsh-tools": "workspace:*", "@deepseek-ai/dsh-user-approval": "workspace:*", diff --git a/packages/web/tool-web/README.i18n.yaml b/packages/web/tool-web/README.i18n.yaml index eb3fa4731d..1e746ed566 100644 --- a/packages/web/tool-web/README.i18n.yaml +++ b/packages/web/tool-web/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -README.md: 5e567115c386d14b7e412ed2502e7290826a5e5e -README.zh.md: b17fe4107908381806d4029481bbf03696c4f313 +README.md: 5fe48ced81a2cd02197cf8cc10a7d6567b17ffca +README.zh.md: 34ad08e290166ee6db2cd7b836746541d18aad52 diff --git a/packages/web/tool-web/README.md b/packages/web/tool-web/README.md index 5e567115c3..5fe48ced81 100644 --- a/packages/web/tool-web/README.md +++ b/packages/web/tool-web/README.md @@ -11,7 +11,7 @@ Each tool is registered independently; a product that wants only one disables th | Tool | Args | Behavior | |---|---|---| | `web_search` | `query` (string) | Discovery. Returns an optional answer plus source URLs. `max_results` is **not** model-facing — the tool sets the bound (the `searchMaxResults` config, default 8) and passes it to the seam. | -| `web_fetch` | `url` (string) | Retrieves a specific URL. HTML bodies are rendered to markdown-ish text; text bodies pass through. A non-2xx status is reported, not an error. The tool-call timeout is deployment policy (`dsh-timeout-policy`), not a model argument. | +| `web_fetch` | `url` (string) | Retrieves a specific URL. HTML bodies are rendered to markdown (turndown with GFM tables/strikethrough); text bodies pass through. A non-2xx status is reported, not an error. The tool-call timeout is deployment policy (`dsh-timeout-policy`), not a model argument. | Both tools opt into concurrent scheduling because provider reads return content without mutating parent-agent state. @@ -126,6 +126,6 @@ Append-only; newly visible content follows the reusable request prefix and does ## Known Limitations and Deferred Work -- **`htmlToMarkdown` is a minimal regex converter, not an HTML parser** — it strips script/style/noscript, keeps headings/bullets/links, and decodes about a dozen named entities; tables, images, and nested formatting are lost. +- **HTML→markdown conversion falls back to raw HTML on pathological input** — [turndown](https://github.com/mixmark-io/turndown) (with GFM tables/strikethrough) converts fetched HTML through a real DOM, but its recursive walk overflows on absurdly deep nesting (thousands of levels); such a body passes through unconverted rather than erroring ([Agent Note](../../../.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md)). - **The model-facing surface is minimal by design, with promotions deferred** — `max_results` stays a config bound (not a model argument), and `web_fetch` takes only `url` (no `format`/`prompt`/LLM-summarization mode); both are named later steps in [the seam Agent Note](../../../.agents/notes/implemented/architecture/2026-06-24-web-capability-seam.md). - **No web-specific permission policy** — both tools execute without requesting `ctx.approval`; a deployment that needs confirmation must add a `tools/pre-execute` policy, and the package does not define persistent URL/domain grants. diff --git a/packages/web/tool-web/README.zh.md b/packages/web/tool-web/README.zh.md index b17fe41079..34ad08e290 100644 --- a/packages/web/tool-web/README.zh.md +++ b/packages/web/tool-web/README.zh.md @@ -11,7 +11,7 @@ | 工具 | 参数 | 行为 | |---|---|---| | `web_search` | `query`（string） | 发现。返回可选答案与源 URL。`max_results` **不** 面向模型：工具设置上限（`searchMaxResults` 配置，默认 8）并传给 seam。 | -| `web_fetch` | `url`（string） | 获取特定 URL。HTML 主体渲染为近似 markdown 的文本；文本主体原样通过。非 2xx 状态会报告，而非报错。工具调用超时是部署策略（`dsh-timeout-policy`），不是模型参数。 | +| `web_fetch` | `url`（string） | 获取特定 URL。HTML 主体渲染为 markdown（turndown，带 GFM 表格／删除线）；文本主体原样通过。非 2xx 状态会报告，而非报错。工具调用超时是部署策略（`dsh-timeout-policy`），不是模型参数。 | 两个工具都选择并发调度，因为提供方读取会返回内容，不会修改父 agent 状态。 @@ -126,6 +126,6 @@ Use the web_fetch tool to retrieve the content of a specific HTTP(S) URL (for ex ## 已知限制与暂缓事项 -- **`htmlToMarkdown` 是最小正则转换器，不是 HTML parser**：它会移除 script/style/noscript，保留标题／项目符号／链接，并解码约十余个命名 entity；表格、图片与嵌套格式会丢失。 +- **HTML→markdown 转换在病态输入上回退为原始 HTML**：[turndown](https://github.com/mixmark-io/turndown)（带 GFM 表格／删除线）通过真实 DOM 转换抓取到的 HTML，但其递归遍历在极深嵌套（数千层）上会栈溢出；此类主体不经转换原样通过，而非报错（[决策记录](../../../.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md)）。 - **面向模型的表层有意保持最小，提升项暂缓**：`max_results` 保持为配置上限（不是模型参数），`web_fetch` 只接受 `url`（没有 `format`／`prompt`／LLM 摘要模式）；两项都列为 [seam Agent Note](../../../.agents/notes/implemented/architecture/2026-06-24-web-capability-seam.md) 中的后续步骤。 - **没有 web 专用权限策略**：两个工具都不会请求 `ctx.approval` 就直接执行；需要确认的部署必须添加 `tools/pre-execute` 策略，该包不定义持久 URL／domain 授权。 diff --git a/packages/web/tool-web/package.json b/packages/web/tool-web/package.json index ec1d33f4d4..9e1a54b6cd 100644 --- a/packages/web/tool-web/package.json +++ b/packages/web/tool-web/package.json @@ -35,10 +35,13 @@ "cordis": "^4.0.0-rc.7" }, "dependencies": { - "schemastery": "^3.18.0" + "@joplin/turndown-plugin-gfm": "^1.0.67", + "schemastery": "^3.18.0", + "turndown": "^7.2.4" }, "devDependencies": { "@deepseek-ai/dsh-agent": "workspace:^", + "@types/turndown": "^5.0.6", "@deepseek-ai/dsh-invariants": "workspace:^", "@deepseek-ai/dsh-llm": "workspace:^", "@deepseek-ai/dsh-session": "workspace:^", diff --git a/packages/web/tool-web/src/fetch.ts b/packages/web/tool-web/src/fetch.ts index cc2ae52970..60c0f33507 100644 --- a/packages/web/tool-web/src/fetch.ts +++ b/packages/web/tool-web/src/fetch.ts @@ -6,12 +6,29 @@ */ import type { Context } from 'cordis' +import TurndownService from 'turndown' +import { gfm } from '@joplin/turndown-plugin-gfm' import { defineTool } from '@deepseek-ai/dsh-tools' import type { GenericCallView } from '@deepseek-ai/dsh-tools' import type { WebFetchBody, WebFetchResult } from '@deepseek-ai/dsh-web' import { assertNever } from '@deepseek-ai/dsh-llm' import type {} from '@deepseek-ai/dsh-system-prompt' -import { htmlToMarkdown } from './html.ts' + +/** + * The shared HTML→markdown converter: turndown over its bundled domino DOM, + * with GitHub-flavored tables/strikethrough (`@joplin/turndown-plugin-gfm`). + * The style options are fixed model-facing presentation (matching the repo's + * markdown conventions), not deployment tunables. `remove` drops non-content + * elements wholesale — turndown's default keeps their text. The instance is + * stateless across `turndown()` calls and safe to share. + */ +const turndown = new TurndownService({ + headingStyle: 'atx', + codeBlockStyle: 'fenced', + bulletListMarker: '-', +}) +turndown.use(gfm) +turndown.remove(['script', 'style', 'noscript']) /** * Validate value constraints the schema DSL can't express: a non-blank `url`. @@ -30,14 +47,23 @@ export function parseFetchArgs(args: { url: string }): { url: string } { /** * Render a fetched body to model-facing markdown text. * - * @param body - the decoded body; `html` is converted via - * {@link htmlToMarkdown}, `text` passes through verbatim. + * @param body - the decoded body; `html` is converted via turndown, `text` + * passes through verbatim. When turndown throws (deeply pathological HTML + * overflows its recursive DOM walk), the raw HTML passes through instead — + * a degraded page beats an error for a body the provider already decoded. * @returns the text for the tool's output block. */ export function renderBody(body: WebFetchBody): string { switch (body.kind) { case 'html': - return htmlToMarkdown(body.content) + try { + return turndown.turndown(body.content) + } catch { + // turndown's DOM walk recurses per element; pathological nesting (a + // few thousand levels) throws RangeError. Provider errors stay + // structured WebErrors upstream; conversion failure downgrades to raw HTML. + return body.content + } case 'text': return body.content /* v8 ignore next 2 -- WebFetchBody is a closed union; this arm is unreachable and only makes adding a kind a compile error. */ diff --git a/packages/web/tool-web/src/html.ts b/packages/web/tool-web/src/html.ts deleted file mode 100644 index 1d6ffdb9a3..0000000000 --- a/packages/web/tool-web/src/html.ts +++ /dev/null @@ -1,86 +0,0 @@ -/** - * Minimal dependency-free HTML-to-readable-text conversion for `web_fetch`, not a full parser. It - * removes non-content elements and tags, decodes common entities, collapses whitespace, and keeps - * basic headings, lists, and links. A richer converter can replace it without changing the seam or - * tool schema. - * @module @deepseek-ai/dsh-tool-web/html - */ - -/** Decode the handful of HTML entities common in textual content. */ -function decodeEntities(text: string): string { - return text - .replace(/&(#[xX][0-9a-fA-F]+|#[0-9]+|[a-zA-Z]+);/g, (match, entity: string) => { - if (entity.startsWith('#x') || entity.startsWith('#X')) { - const code = Number.parseInt(entity.slice(2), 16) - return safeFromCodePoint(code, match) - } - if (entity.startsWith('#')) { - const code = Number.parseInt(entity.slice(1), 10) - return safeFromCodePoint(code, match) - } - return NAMED_ENTITIES[entity] ?? match - }) -} - -const NAMED_ENTITIES: Record = { - amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ', - copy: '©', reg: '®', trade: '™', hellip: '…', mdash: '—', ndash: '–', -} - -function safeFromCodePoint(code: number, fallback: string): string { - try { - return String.fromCodePoint(code) - } catch { - // An out-of-range code point (RangeError) is the only failure here; keep the - // original entity text rather than throwing out of pure presentation. - return fallback - } -} - -/** - * Convert an HTML document to a readable markdown-ish text approximation. - * Best-effort and lossy by design — fidelity is the job of a future heavier - * converter, not this fallback. - * - * @param html - the raw HTML source. - * @returns plain text with markdown headings, list bullets, and links; - * whitespace collapsed to at most one blank line and trimmed. - */ -export function htmlToMarkdown(html: string): string { - let text = html - // Drop non-content elements entirely (including their contents). - .replace(/]*>[\s\S]*?<\/script>/gi, '') - .replace(/]*>[\s\S]*?<\/style>/gi, '') - .replace(/]*>[\s\S]*?<\/noscript>/gi, '') - .replace(//g, '') - - // Convert links to markdown before stripping tags. - text = text.replace(/]*\bhref\s*=\s*["']([^"']*)["'][^>]*>([\s\S]*?)<\/a>/gi, (_match, href: string, label: string) => { - const cleanLabel = label.replace(/<[^>]+>/g, '').trim() - return cleanLabel.length > 0 ? `[${cleanLabel}](${href})` : href - }) - - // Headings → markdown hashes. - text = text.replace(/]*>([\s\S]*?)<\/h\1>/gi, (_match, level: string, body: string) => { - const hashes = '#'.repeat(Number(level)) - return `\n\n${hashes} ${body.replace(/<[^>]+>/g, '').trim()}\n\n` - }) - - // List items → bullets. - text = text.replace(/]*>([\s\S]*?)<\/li>/gi, (_match, body: string) => `\n- ${body.replace(/<[^>]+>/g, '').trim()}`) - - // Block-level breaks become paragraph breaks. - text = text - .replace(/<\/(p|div|section|article|header|footer|tr|table|ul|ol|blockquote)>/gi, '\n\n') - .replace(//gi, '\n') - - // Drop all remaining tags, decode entities, collapse whitespace. - text = text.replace(/<[^>]+>/g, '') - text = decodeEntities(text) - text = text - .replace(/[ \t\f\v]+/g, ' ') - .replace(/ *\n */g, '\n') - .replace(/\n{3,}/g, '\n\n') - .trim() - return text -} diff --git a/packages/web/tool-web/src/index.ts b/packages/web/tool-web/src/index.ts index 7096371ed1..e7ac4b2453 100644 --- a/packages/web/tool-web/src/index.ts +++ b/packages/web/tool-web/src/index.ts @@ -14,7 +14,6 @@ import { applyWebFetchTool } from './fetch.ts' export { WEB_SEARCH_MAX_RESULTS, applyWebSearchTool, formatSearchOutput, parseSearchArgs, presentSearchCall } from './search.ts' export { applyWebFetchTool, formatFetchOutput, parseFetchArgs, presentFetchCall, renderBody } from './fetch.ts' -export { htmlToMarkdown } from './html.ts' /** Cordis plugin name used by loader diagnostics. */ export const name = 'tool-web' diff --git a/packages/web/tool-web/src/turndown-plugin-gfm.d.ts b/packages/web/tool-web/src/turndown-plugin-gfm.d.ts new file mode 100644 index 0000000000..66c9d929e4 --- /dev/null +++ b/packages/web/tool-web/src/turndown-plugin-gfm.d.ts @@ -0,0 +1,12 @@ +/** + * Ambient module declaration for `@joplin/turndown-plugin-gfm`, which ships no + * types and has no DefinitelyTyped package. Only the composite `gfm` plugin is + * declared; the package's individual plugins (`tables`, `strikethrough`, …) + * stay undeclared until something imports them. + */ +declare module '@joplin/turndown-plugin-gfm' { + import type TurndownService from 'turndown' + + /** The composite GitHub-flavored-markdown plugin (tables, strikethrough, task lists, highlighted code blocks). */ + export const gfm: TurndownService.Plugin +} diff --git a/packages/web/tool-web/tests/tool-web.spec.ts b/packages/web/tool-web/tests/tool-web.spec.ts index 093184c4a7..f9ffb1b5c5 100644 --- a/packages/web/tool-web/tests/tool-web.spec.ts +++ b/packages/web/tool-web/tests/tool-web.spec.ts @@ -14,7 +14,6 @@ import { presentSearchCall, presentFetchCall, renderBody, - htmlToMarkdown, WEB_SEARCH_MAX_RESULTS, } from '@deepseek-ai/dsh-tool-web' @@ -82,6 +81,11 @@ describe('search formatting', () => { expect(parseSearchArgs({ query: 'hi' })).toEqual({ query: 'hi' }) }) + it('falls back to the raw URL as a source label when the URL is unparseable', () => { + const out = formatSearchOutput({ truncated: false, sources: [{ url: 'not a url' }] }) + expect(out).toContain('[not a url](not a url)') + }) + it('presents a search call as a search-kind card titled by the query', () => { expect(presentSearchCall({ query: 'find me' })).toEqual({ card: 'generic', title: 'find me', kind: 'search', rawInput: 'find me' }) }) @@ -112,6 +116,29 @@ describe('fetch formatting', () => { expect(renderBody({ kind: 'html', content: '

    y

    ' })).toBe('y') }) + it('converts html via turndown: entities, links, tables, nesting; drops script/style/noscript', () => { + expect(renderBody({ + kind: 'html', + content: '

    Tom & Jerry © Résumé

    link', + })).toBe('Tom & Jerry © Résumé\n\n[link](https://a.test)') + expect(renderBody({ kind: 'html', content: '

    Heading

    • one
    • two
    ' })) + .toBe('## Heading\n\n- one\n- two') + expect(renderBody({ kind: 'html', content: '
    AB
    12
    ' })) + .toBe('| A | B |\n| --- | --- |\n| 1 | 2 |') + expect(renderBody({ kind: 'html', content: '

    bold italic

    quoted

    ' })) + .toBe('**bold _italic_**\n\n> quoted') + }) + + it('falls back to the raw html body when turndown throws on pathological nesting', { timeout: 60_000 }, () => { + // Nesting past V8's default stack overflows turndown/domino's recursive + // walk with a RangeError (measured: 4k levels throw on the main thread, + // 8k in a worker); 20k adds margin over either stack size. The raw body + // must pass through instead of throwing. + const depth = 20_000 + const pathological = '
    '.repeat(depth) + 'x' + '
    '.repeat(depth) + expect(renderBody({ kind: 'html', content: pathological })).toBe(pathological) + }) + it('validates url (non-empty), no timeout parameter', () => { expect(() => parseFetchArgs({ url: ' ' })).toThrow('non-empty') expect(parseFetchArgs({ url: 'https://a.test' })).toEqual({ url: 'https://a.test' }) @@ -122,46 +149,6 @@ describe('fetch formatting', () => { }) }) -describe('htmlToMarkdown', () => { - it('drops scripts/styles, keeps text, decodes entities, converts links', () => { - const md = htmlToMarkdown('

    Tom & Jerry

    link') - expect(md).not.toContain('bad()') - expect(md).not.toContain('.x{}') - expect(md).toContain('Tom & Jerry') - expect(md).toContain('[link](https://a.test)') - }) - - it('decodes numeric entities and collapses whitespace', () => { - expect(htmlToMarkdown('

    a'b

    ')).toBe("a'b") - expect(htmlToMarkdown('
    x
    \n\n\n
    y
    ')).toBe('x\n\ny') - }) - - it('decodes hex entities and named entities, and leaves unknown/out-of-range ones intact', () => { - expect(htmlToMarkdown('

    AB

    ')).toBe('AB') - expect(htmlToMarkdown('

    © —

    ')).toBe('© —') - expect(htmlToMarkdown('

    ¬areal;

    ')).toBe('¬areal;') - // An out-of-range code point keeps the original entity text (fromCodePoint fallback). - expect(htmlToMarkdown('

    �

    ')).toBe('�') - expect(htmlToMarkdown('

    �

    ')).toBe('�') - }) - - it('renders a link with an empty label as its bare href', () => { - expect(htmlToMarkdown('')).toBe('https://a.test') - }) - - it('converts headings and list items to markdown', () => { - expect(htmlToMarkdown('

    Heading

    after

    ')).toContain('## Heading') - const list = htmlToMarkdown('
    • one
    • two
    ') - expect(list).toContain('- one') - expect(list).toContain('- two') - }) - - it('falls back to the raw URL as a source label when the URL is unparseable', () => { - const out = formatSearchOutput({ truncated: false, sources: [{ url: 'not a url' }] }) - expect(out).toContain('[not a url](not a url)') - }) -}) - describe('tool-web registration', () => { it('registers both tools by default', async () => { const { fiber, ctx } = await mountTools() diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1946a841bb..430cf6ea0a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -523,6 +523,9 @@ importers: '@deepseek-ai/dsh-tool-todo': specifier: workspace:* version: link:../packages/todo/tool-todo + '@deepseek-ai/dsh-tool-web': + specifier: workspace:* + version: link:../packages/web/tool-web '@deepseek-ai/dsh-tool-workflow': specifier: workspace:* version: link:../packages/workflow/tool-workflow @@ -4259,9 +4262,15 @@ importers: packages/web/tool-web: dependencies: + '@joplin/turndown-plugin-gfm': + specifier: ^1.0.67 + version: 1.0.67 schemastery: specifier: ^3.18.0 version: 3.18.0 + turndown: + specifier: ^7.2.4 + version: 7.2.4 devDependencies: '@deepseek-ai/dsh-agent': specifier: workspace:^ @@ -4299,6 +4308,9 @@ importers: '@deepseek-ai/dsh-web-search-exa': specifier: workspace:^ version: link:../web-search-exa + '@types/turndown': + specifier: ^5.0.6 + version: 5.0.6 cordis: specifier: ^4.0.0-rc.7 version: 4.0.0-rc.7(@cordisjs/plugin-include@1.0.4)(@cordisjs/plugin-loader@1.0.0-rc.5) @@ -5971,6 +5983,9 @@ packages: resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} engines: {node: '>=12'} + '@joplin/turndown-plugin-gfm@1.0.67': + resolution: {integrity: sha512-FZfW5EZfidhzd1IaY1uxHnIZPTVOxAdleMZ4/1U6Nt5b7+Qj5JThDnaIomuJtetnUBzuRNbe9FWMuqD4B3dlWA==} + '@jridgewell/gen-mapping@0.3.13': resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} @@ -6076,6 +6091,9 @@ packages: '@opentelemetry/api': optional: true + '@mixmark-io/domino@2.2.0': + resolution: {integrity: sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==} + '@modelcontextprotocol/sdk@1.29.0': resolution: {integrity: sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ==} engines: {node: '>=18'} @@ -7005,6 +7023,9 @@ packages: '@types/trusted-types@2.0.7': resolution: {integrity: sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==} + '@types/turndown@5.0.6': + resolution: {integrity: sha512-ru00MoyeeouE5BX4gRL+6m/BsDfbRayOskWqUvh7CLGW+UXxHQItqALa38kKnOiZPqJrtzJUgAC2+F0rL1S4Pg==} + '@types/unist@2.0.11': resolution: {integrity: sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==} @@ -9463,6 +9484,10 @@ packages: engines: {node: '>=18.0.0'} hasBin: true + turndown@7.2.4: + resolution: {integrity: sha512-I8yFsfRzmzK0WV1pNNOA4A7y4RDfFxPRxb3t+e3ui14qSGOxGtiSP6GjeX+Y6CHb7HYaFj7ECUD7VE5kQMZWGQ==} + engines: {node: '>=18', npm: '>=9'} + type-check@0.4.0: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} engines: {node: '>= 0.8.0'} @@ -10860,6 +10885,8 @@ snapshots: wrap-ansi: 8.1.0 wrap-ansi-cjs: wrap-ansi@7.0.0 + '@joplin/turndown-plugin-gfm@1.0.67': {} + '@jridgewell/gen-mapping@0.3.13': dependencies: '@jridgewell/sourcemap-codec': 1.5.5 @@ -10951,6 +10978,8 @@ snapshots: - bufferutil - utf-8-validate + '@mixmark-io/domino@2.2.0': {} + '@modelcontextprotocol/sdk@1.29.0(zod@4.4.3)': dependencies: '@hono/node-server': 1.19.14(hono@4.12.29) @@ -11705,6 +11734,8 @@ snapshots: '@types/trusted-types@2.0.7': optional: true + '@types/turndown@5.0.6': {} + '@types/unist@2.0.11': {} '@types/unist@3.0.3': {} @@ -14645,6 +14676,10 @@ snapshots: optionalDependencies: fsevents: 2.3.3 + turndown@7.2.4: + dependencies: + '@mixmark-io/domino': 2.2.0 + type-check@0.4.0: dependencies: prelude-ls: 1.2.1 From c1153577378f271c1145f12f07185be591193fa2 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Mon, 27 Jul 2026 04:37:00 +0800 Subject: [PATCH 02/14] =?UTF-8?q?ci:=20experiment=20=E2=80=94=20Wine-run?= =?UTF-8?q?=20Windows=20blocking=20gates=20on=20a=20Linux=20runner?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...27-wine-windows-gates-experiment.i18n.yaml | 6 + ...026-07-27-wine-windows-gates-experiment.md | 44 +++++++ ...-07-27-wine-windows-gates-experiment.zh.md | 44 +++++++ .github/workflows/exp-wine-windows.yml | 123 ++++++++++++++++++ 4 files changed, 217 insertions(+) create mode 100644 .agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.i18n.yaml create mode 100644 .agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.md create mode 100644 .agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.zh.md create mode 100644 .github/workflows/exp-wine-windows.yml diff --git a/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.i18n.yaml b/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.i18n.yaml new file mode 100644 index 0000000000..eb3909cc4e --- /dev/null +++ b/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.i18n.yaml @@ -0,0 +1,6 @@ +# Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each +# side as of the last confirmed-consistent state. Both languages carry equal authority; +# after editing either side, bring the other along and re-record with: +# pnpm run verify-translation-pairing --write +2026-07-27-wine-windows-gates-experiment.md: 9f7856dfef229f8f02c85f5968082a0c857bbc94 +2026-07-27-wine-windows-gates-experiment.zh.md: cb185293d7f22723a96448a774bd27dd31e1bc28 diff --git a/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.md b/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.md new file mode 100644 index 0000000000..9f7856dfef --- /dev/null +++ b/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.md @@ -0,0 +1,44 @@ +# Agent Note: Wine-run Windows blocking gates on Linux runners + +Status: proposed + +English | [中文](2026-07-27-wine-windows-gates-experiment.zh.md) + +## Problem + +The pull-request Windows lane exists to prove the two blocking win32 surfaces — the workspace build and the production site — plus an observational portability inventory, and it runs on a dedicated paid Windows larger-runner pool; the master serial reference adds a second hosted Windows job. That pool is the only reason a Windows VM exists anywhere in this pipeline, and its provisioning, pricing, and slow setup dominate the lane's cost. + +The open question: can a plain Linux runner produce an equivalent win32 signal for the blocking surfaces, so the dedicated Windows pool can shrink to a master-only reference or disappear from the pull-request path entirely? + +## Proposal + +[exp-wine-windows.yml](../../../../.github/workflows/exp-wine-windows.yml) (self-path-filtered, plus manual dispatch) runs the blocking gate commands on `ubuntu-latest` under Wine with real Windows binaries: a downloaded win-x64 Node.js executes `tsc -b`, `tsdown`, and the VitePress production build, so the win32 branches of the toolchain — backslash path handling, `CreateProcess` spawn semantics, PE loading of `@esbuild/win32-x64`, and the rolldown/rollup MSVC `.node` addons — actually execute. + +Dependencies install natively on Linux with `supportedArchitectures` extended to win32-x64, which materializes the Windows platform packages in the same store; the cmd-shim layer is bypassed by invoking each tool's JavaScript entrypoint directly, the same processes `run-gates` ultimately spawns. + +This is deliberately a fidelity probe, not a drop-in replacement: Wine reimplements the Win32 API over a case-sensitive ext4 (NTFS case-insensitivity is not emulated by default), provides no ConPTY, and substitutes its own security-descriptor and `MoveFileExW` semantics — exactly the surfaces the repo's `win32.ts` modules and PTY backend care about. The experiment measures which blocking gates pass, which fail for Wine reasons rather than product reasons, and the wall-clock cost relative to the recorded Windows benchmark lanes. + +Promotion, if the verdict is positive: fold the Wine lane in as the pull-request Windows signal for blocking gates and demote the real-Windows pool to the master serial reference; otherwise record the failure class here and keep the pool. + +## Alternatives considered + +**Keep the dedicated Windows pool (status quo).** It is the baseline being priced; nothing is wrong with its signal, only with paying for a Windows VM pool whose blocking surface is two build commands. + +**A full Windows guest under QEMU/KVM inside the Linux runner.** Real NT kernel, so full fidelity including case-insensitive NTFS and ConPTY — but tens of minutes of image download and unattended install before the first gate runs. Explored as the sibling experiment branch `exp/kvm-windows-ci`; the two experiments price fidelity against latency. + +**Filesystem-semantics lanes on Linux (casefolded ext4, filename lint).** Catches the highest-frequency Windows breakage class for near-zero cost but proves nothing about win32 binaries. Explored as the sibling experiment branch `exp/casefold-windows-ci`. + +**Windows containers.** Not possible: Windows containers require a Windows host kernel; a hosted Linux runner cannot run them. + +**Dropping the Windows lane.** Rejected — win32 is a first-class product target: the koffi-backed DACL and durable-namespace modules, ConPTY-based PTY sessions, and Windows path policy all ship in `packages/`. + +## Acceptance criteria + +- The workflow completes on `ubuntu-latest` with an independent pass/fail verdict per blocking gate (tsc, tsdown, production site) and a recorded wall-clock comparison against the Windows benchmark lanes. +- A decision is recorded here: promote the lane, keep it as a non-blocking canary, or reject it with the observed failure class. + +## Risks + +- False greens: Wine's case-sensitive filesystem and permissive path handling can pass code that breaks on real NTFS, so this lane can complement but never fully replace a real-kernel check for release qualification. +- False reds: missing or stubbed Win32 APIs under Wine fail gates for non-product reasons, and each such failure costs triage time to classify. +- Throughput: Wine's syscall translation on the 2-core standard runner may push the blocking gates past the paid Windows lane's wall clock, erasing the cost argument; the run records the numbers either way. diff --git a/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.zh.md b/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.zh.md new file mode 100644 index 0000000000..cb185293d7 --- /dev/null +++ b/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.zh.md @@ -0,0 +1,44 @@ +# Agent Note: 在 Linux runner 上用 Wine 运行 Windows 阻断门禁 + +Status: proposed + +[English](2026-07-27-wine-windows-gates-experiment.md) | 中文 + +## 问题 + +Pull request 的 Windows 通道存在的意义是证明两个阻断性 win32 表面——workspace 构建与生产站点——外加一份观察性可移植性清单，它运行在一个专用的付费 Windows larger-runner 池上；master 串行参照又增加一个托管 Windows 作业。该池是这条流水线中唯一需要 Windows VM 的理由，而其供给、计价与缓慢的准备阶段主导了该通道的成本。 + +悬而未决的问题是：一台普通 Linux runner 能否为阻断表面产出等效的 win32 信号，让专用 Windows 池收缩为仅 master 的参照、甚至完全退出 pull request 路径？ + +## 提案 + +[exp-wine-windows.yml](../../../../.github/workflows/exp-wine-windows.yml)（自身路径过滤，外加手动触发）在 `ubuntu-latest` 上通过 Wine 用真实 Windows 二进制运行阻断门禁命令：下载的 win-x64 Node.js 执行 `tsc -b`、`tsdown` 与 VitePress 生产构建，因此工具链的 win32 分支——反斜杠路径处理、`CreateProcess` 派生语义、`@esbuild/win32-x64` 的 PE 加载、以及 rolldown/rollup 的 MSVC `.node` 插件——都真正执行。 + +依赖在 Linux 上原生安装，`supportedArchitectures` 扩展到 win32-x64，使 Windows 平台包物化进同一个 store；通过直接调用各工具的 JavaScript 入口绕开 cmd-shim 层，这正是 `run-gates` 最终派生的那些进程。 + +这刻意是一次保真度探针，而非直接替换：Wine 在大小写敏感的 ext4 之上重实现 Win32 API（默认不模拟 NTFS 的大小写不敏感）、不提供 ConPTY、并用自己的安全描述符与 `MoveFileExW` 语义替代——恰是本仓库 `win32.ts` 模块与 PTY 后端关心的表面。实验度量哪些阻断门禁通过、哪些因 Wine 原因而非产品原因失败，以及相对已记录 Windows 基准通道的墙钟成本。 + +若结论为正则晋升：把 Wine 通道并入为 pull request 的阻断门禁 Windows 信号，将真实 Windows 池降级为 master 串行参照；否则在此记录失败类别并保留该池。 + +## 考虑过的替代方案 + +**保留专用 Windows 池（现状）。** 它正是被计价的基线；其信号没有问题，问题只在于为一个阻断表面仅是两条构建命令的 Windows VM 池付费。 + +**在 Linux runner 内用 QEMU/KVM 跑完整 Windows 客户机。** 真实 NT 内核，保真度完整，包括大小写不敏感的 NTFS 与 ConPTY——但首个门禁运行前要花数十分钟下载镜像并做无人值守安装。作为兄弟实验分支 `exp/kvm-windows-ci` 探索；两个实验共同为保真度与延迟定价。 + +**Linux 上的文件系统语义通道（casefold ext4、文件名 lint）。** 以近零成本捕获最高频的 Windows 破坏类别，但对 win32 二进制什么也证明不了。作为兄弟实验分支 `exp/casefold-windows-ci` 探索。 + +**Windows 容器。** 不可行：Windows 容器要求 Windows 宿主内核；托管 Linux runner 无法运行。 + +**砍掉 Windows 通道。** 已否决——win32 是一等产品目标：基于 koffi 的 DACL 与持久命名空间模块、基于 ConPTY 的 PTY 会话、以及 Windows 路径策略都随 `packages/` 交付。 + +## 验收标准 + +- 该 workflow 在 `ubuntu-latest` 上完成，对每个阻断门禁（tsc、tsdown、生产站点）给出独立的通过/失败裁决，并记录与 Windows 基准通道的墙钟对比。 +- 在此记录一项决定：晋升该通道、保留为非阻断金丝雀、或以观察到的失败类别否决。 + +## 风险 + +- 假绿：Wine 的大小写敏感文件系统与宽松路径处理可能放过在真实 NTFS 上会坏的代码，因此该通道可以补充、但永远无法完全替代发布资格所需的真实内核检查。 +- 假红：Wine 下缺失或桩化的 Win32 API 会因非产品原因让门禁失败，每次此类失败都要花分诊时间归类。 +- 吞吐：Wine 的系统调用翻译在 2 核标准 runner 上可能让阻断门禁的墙钟超过付费 Windows 通道，抹掉成本论点；无论结果如何，运行都会记录数字。 diff --git a/.github/workflows/exp-wine-windows.yml b/.github/workflows/exp-wine-windows.yml new file mode 100644 index 0000000000..499429f807 --- /dev/null +++ b/.github/workflows/exp-wine-windows.yml @@ -0,0 +1,123 @@ +# EXPERIMENT: run the blocking Windows CI gates on a Linux runner through +# Wine, and execute the gate commands with a real Windows Node.js binary. +# Dependency provisioning happens natively on Linux with +# `supportedArchitectures` extended to win32-x64 so the Windows +# esbuild/rolldown/rollup binaries are present in the store. The pnpm-run/cmd +# shim layer is deliberately bypassed (a Linux install writes POSIX shims +# only), so each gate invokes its tool's JavaScript entrypoint directly — the +# same commands run-gates ultimately spawns. Owning rationale and promotion +# criteria: +# .agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.md +name: Experiment Wine Windows gates + +on: + workflow_dispatch: + pull_request: + paths: + - .github/workflows/exp-wine-windows.yml + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + PRIMARY_NODE_VERSION: '24' + +jobs: + wine-blocking-gates: + name: wine / blocking windows gates + # Deliberately the cheapest hosted substrate: if Wine holds up here, the + # lane needs no special pool at all. + runs-on: ubuntu-latest + timeout-minutes: 120 + env: + WINEDEBUG: '-all' + WINEARCH: win64 + # Skip Wine Mono / Gecko installers: Node needs neither. + WINEDLLOVERRIDES: 'mscoree,mshtml=' + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + + - uses: actions/setup-node@v6 + with: + node-version: ${{ env.PRIMARY_NODE_VERSION }} + + - name: Enable corepack and install with win32-x64 artifacts + run: | + corepack enable + # Experiment-only install-time override: also materialize the + # win32-x64 platform packages (@esbuild/win32-x64, rolldown and + # rollup MSVC bindings) that the Windows toolchain resolves at + # runtime. supportedArchitectures is not recorded in the lockfile, + # so --frozen-lockfile stays valid. + cat >> pnpm-workspace.yaml <<'EOF' + + supportedArchitectures: + os: [current, win32] + cpu: [current, x64] + EOF + pnpm install --frozen-lockfile + + - name: Install Wine (64-bit) + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends wine64 + WINE_BIN=$(command -v wine || command -v wine64) + echo "WINE_BIN=$WINE_BIN" >> "$GITHUB_ENV" + "$WINE_BIN" --version + + - name: Fetch Windows Node.js + run: | + version=$(curl -fsSL https://nodejs.org/dist/index.json \ + | jq -r --arg p "v${PRIMARY_NODE_VERSION}." '[.[] | select(.version | startswith($p))][0].version') + echo "Windows Node: $version" + curl -fsSL -o "$RUNNER_TEMP/node-win.zip" \ + "https://nodejs.org/dist/${version}/node-${version}-win-x64.zip" + unzip -q "$RUNNER_TEMP/node-win.zip" -d "$RUNNER_TEMP/node-win" + echo "NODE_WIN=$RUNNER_TEMP/node-win/node-${version}-win-x64/node.exe" >> "$GITHUB_ENV" + + - name: Boot Wine prefix and smoke Windows Node + run: | + "$WINE_BIN" wineboot --init || true + wineserver -w || true + "$WINE_BIN" "$NODE_WIN" -p "'smoke: ' + process.platform + ' ' + process.arch + ' ' + process.version" + + # The continue-on-error gates below mirror ci-windows-blocking + # (scripts/run-gates.ts): `build` = tsc -b + tsdown, `production site` = + # vitepress build. Each reports independently so one failure does not + # hide the others' results; the summary step at the end owns the job + # conclusion. + - name: 'Gate: tsc -b (Windows node under Wine)' + id: tsc + continue-on-error: true + timeout-minutes: 45 + run: '"$WINE_BIN" "$NODE_WIN" node_modules/typescript/bin/tsc -b --pretty false' + + - name: 'Gate: tsdown (Windows node under Wine)' + id: tsdown + continue-on-error: true + timeout-minutes: 30 + run: '"$WINE_BIN" "$NODE_WIN" node_modules/tsdown/dist/run.mjs' + + - name: 'Gate: production site (Windows node under Wine)' + id: site + continue-on-error: true + timeout-minutes: 30 + working-directory: website + run: '"$WINE_BIN" "$NODE_WIN" node_modules/vitepress/bin/vitepress.js build .' + + - name: Report gate outcomes + env: + TSC: ${{ steps.tsc.outcome }} + TSDOWN: ${{ steps.tsdown.outcome }} + SITE: ${{ steps.site.outcome }} + run: | + echo "tsc: $TSC" + echo "tsdown: $TSDOWN" + echo "production site: $SITE" + [ "$TSC" = success ] && [ "$TSDOWN" = success ] && [ "$SITE" = success ] From edcc0540f02a265bbfc75e23e72f61b30edf8f4d Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Mon, 27 Jul 2026 04:59:38 +0800 Subject: [PATCH 03/14] ci(exp-wine): install the wine dispatcher package, fall back to the wine64 loader path --- .github/workflows/exp-wine-windows.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/exp-wine-windows.yml b/.github/workflows/exp-wine-windows.yml index 499429f807..94ff45544f 100644 --- a/.github/workflows/exp-wine-windows.yml +++ b/.github/workflows/exp-wine-windows.yml @@ -66,8 +66,19 @@ jobs: - name: Install Wine (64-bit) run: | sudo apt-get update - sudo apt-get install -y --no-install-recommends wine64 - WINE_BIN=$(command -v wine || command -v wine64) + # `wine` is the /usr/bin/wine dispatcher; its dependency pulls the + # wine64 loader. Ubuntu's wine64 package alone leaves nothing on + # PATH (the loader sits at /usr/lib/wine/wine64). + sudo apt-get install -y --no-install-recommends wine + WINE_BIN='' + for candidate in "$(command -v wine || true)" "$(command -v wine64 || true)" /usr/lib/wine/wine64; do + if [ -n "$candidate" ] && [ -x "$candidate" ]; then WINE_BIN="$candidate"; break; fi + done + if [ -z "$WINE_BIN" ]; then + echo '::error::no wine binary found after install' + dpkg -L wine wine64 2>/dev/null | grep -E '/bin/|wine64$' || true + exit 1 + fi echo "WINE_BIN=$WINE_BIN" >> "$GITHUB_ENV" "$WINE_BIN" --version From 8345d6eae843793664547133aefa32c928a2a7aa Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Mon, 27 Jul 2026 05:04:26 +0800 Subject: [PATCH 04/14] =?UTF-8?q?ci(exp-wine):=20route=20wine-node=20stdio?= =?UTF-8?q?=20through=20files=20=E2=80=94=20runner=20pipes=20hit=20EBADF?= =?UTF-8?q?=20at=20Node=20bootstrap?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/exp-wine-windows.yml | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/exp-wine-windows.yml b/.github/workflows/exp-wine-windows.yml index 94ff45544f..fdb45712c2 100644 --- a/.github/workflows/exp-wine-windows.yml +++ b/.github/workflows/exp-wine-windows.yml @@ -96,7 +96,20 @@ jobs: run: | "$WINE_BIN" wineboot --init || true wineserver -w || true - "$WINE_BIN" "$NODE_WIN" -p "'smoke: ' + process.platform + ' ' + process.arch + ' ' + process.version" + # Node under Wine cannot attach stdio to the Actions runner's pipes + # (Socket open EBADF at bootstrap), so every invocation runs through + # this wrapper: stdio to a regular file, replayed after exit. + cat > "$RUNNER_TEMP/wine-node.sh" <<'SH' + #!/usr/bin/env bash + set -u + log="$1"; shift + "$WINE_BIN" "$NODE_WIN" "$@" < /dev/null > "$log" 2>&1 + status=$? + tail -n 300 "$log" + exit "$status" + SH + chmod +x "$RUNNER_TEMP/wine-node.sh" + "$RUNNER_TEMP/wine-node.sh" "$RUNNER_TEMP/smoke.log" -p "'smoke: ' + process.platform + ' ' + process.arch + ' ' + process.version" # The continue-on-error gates below mirror ci-windows-blocking # (scripts/run-gates.ts): `build` = tsc -b + tsdown, `production site` = @@ -107,20 +120,20 @@ jobs: id: tsc continue-on-error: true timeout-minutes: 45 - run: '"$WINE_BIN" "$NODE_WIN" node_modules/typescript/bin/tsc -b --pretty false' + run: '"$RUNNER_TEMP/wine-node.sh" "$RUNNER_TEMP/tsc.log" node_modules/typescript/bin/tsc -b --pretty false' - name: 'Gate: tsdown (Windows node under Wine)' id: tsdown continue-on-error: true timeout-minutes: 30 - run: '"$WINE_BIN" "$NODE_WIN" node_modules/tsdown/dist/run.mjs' + run: '"$RUNNER_TEMP/wine-node.sh" "$RUNNER_TEMP/tsdown.log" node_modules/tsdown/dist/run.mjs' - name: 'Gate: production site (Windows node under Wine)' id: site continue-on-error: true timeout-minutes: 30 working-directory: website - run: '"$WINE_BIN" "$NODE_WIN" node_modules/vitepress/bin/vitepress.js build .' + run: '"$RUNNER_TEMP/wine-node.sh" "$RUNNER_TEMP/site.log" node_modules/vitepress/bin/vitepress.js build .' - name: Report gate outcomes env: From f34396b00db4614124efa66ca3c25b659b059630 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Mon, 27 Jul 2026 05:14:00 +0800 Subject: [PATCH 05/14] =?UTF-8?q?ci(exp-wine):=20hoisted=20node=5Fmodules?= =?UTF-8?q?=20layout=20=E2=80=94=20Wine=20node=20does=20not=20realpath=20p?= =?UTF-8?q?npm=20symlinks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/exp-wine-windows.yml | 34 ++++++++++++++++++++------ 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/.github/workflows/exp-wine-windows.yml b/.github/workflows/exp-wine-windows.yml index fdb45712c2..567e41a447 100644 --- a/.github/workflows/exp-wine-windows.yml +++ b/.github/workflows/exp-wine-windows.yml @@ -50,19 +50,37 @@ jobs: - name: Enable corepack and install with win32-x64 artifacts run: | corepack enable - # Experiment-only install-time override: also materialize the - # win32-x64 platform packages (@esbuild/win32-x64, rolldown and - # rollup MSVC bindings) that the Windows toolchain resolves at - # runtime. supportedArchitectures is not recorded in the lockfile, - # so --frozen-lockfile stays valid. + # Experiment-only install-time overrides. supportedArchitectures + # additionally materializes the win32-x64 platform packages + # (@esbuild/win32-x64, rolldown and rollup MSVC bindings) that the + # Windows toolchain resolves at runtime. nodeLinker: hoisted lays + # node_modules out flat with real files: Windows Node under Wine + # does not realpath pnpm's Unix symlinks, so the default isolated + # layout breaks transitive ESM resolution (tsdown -> ansis, + # vite -> rollup). Neither override is recorded in the lockfile, so + # --frozen-lockfile stays valid. cat >> pnpm-workspace.yaml <<'EOF' + nodeLinker: hoisted supportedArchitectures: os: [current, win32] cpu: [current, x64] EOF pnpm install --frozen-lockfile + - name: Resolve tool entrypoints in the hoisted layout + run: | + resolve() { + local name="$1"; shift + for p in "$@"; do + if [ -f "$p" ]; then echo "$name=$PWD/$p" >> "$GITHUB_ENV"; return 0; fi + done + echo "::error::$name not found at any of: $*"; return 1 + } + resolve TSC_JS node_modules/typescript/bin/tsc + resolve TSDOWN_JS node_modules/tsdown/dist/run.mjs + resolve VITEPRESS_JS website/node_modules/vitepress/bin/vitepress.js node_modules/vitepress/bin/vitepress.js + - name: Install Wine (64-bit) run: | sudo apt-get update @@ -120,20 +138,20 @@ jobs: id: tsc continue-on-error: true timeout-minutes: 45 - run: '"$RUNNER_TEMP/wine-node.sh" "$RUNNER_TEMP/tsc.log" node_modules/typescript/bin/tsc -b --pretty false' + run: '"$RUNNER_TEMP/wine-node.sh" "$RUNNER_TEMP/tsc.log" "$TSC_JS" -b --pretty false' - name: 'Gate: tsdown (Windows node under Wine)' id: tsdown continue-on-error: true timeout-minutes: 30 - run: '"$RUNNER_TEMP/wine-node.sh" "$RUNNER_TEMP/tsdown.log" node_modules/tsdown/dist/run.mjs' + run: '"$RUNNER_TEMP/wine-node.sh" "$RUNNER_TEMP/tsdown.log" "$TSDOWN_JS"' - name: 'Gate: production site (Windows node under Wine)' id: site continue-on-error: true timeout-minutes: 30 working-directory: website - run: '"$RUNNER_TEMP/wine-node.sh" "$RUNNER_TEMP/site.log" node_modules/vitepress/bin/vitepress.js build .' + run: '"$RUNNER_TEMP/wine-node.sh" "$RUNNER_TEMP/site.log" "$VITEPRESS_JS" build .' - name: Report gate outcomes env: From 241a7e6c72854d2bf57b6280d849338961ff6f85 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Mon, 27 Jul 2026 05:23:25 +0800 Subject: [PATCH 06/14] =?UTF-8?q?ci(exp-wine):=20pre-create=20the=20vue=20?= =?UTF-8?q?link=20VitePress=20needs=20=E2=80=94=20Wine=20cannot=20create?= =?UTF-8?q?=20Windows=20symlinks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/exp-wine-windows.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/exp-wine-windows.yml b/.github/workflows/exp-wine-windows.yml index 567e41a447..9ebc3ccc79 100644 --- a/.github/workflows/exp-wine-windows.yml +++ b/.github/workflows/exp-wine-windows.yml @@ -80,6 +80,13 @@ jobs: resolve TSC_JS node_modules/typescript/bin/tsc resolve TSDOWN_JS node_modules/tsdown/dist/run.mjs resolve VITEPRESS_JS website/node_modules/vitepress/bin/vitepress.js node_modules/vitepress/bin/vitepress.js + # VitePress links vue into the site's node_modules at build time; + # Wine cannot CREATE Windows symlinks (ENOTSUP) but follows + # pre-existing Unix ones, so lay the link down host-side. + if [ -d node_modules/vue ] && [ ! -e website/node_modules/vue ]; then + mkdir -p website/node_modules + ln -s ../../node_modules/vue website/node_modules/vue + fi - name: Install Wine (64-bit) run: | From 3649df14073816443422a3413ff51a5801030011 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Mon, 27 Jul 2026 12:11:27 +0800 Subject: [PATCH 07/14] =?UTF-8?q?ci(exp-wine):=20speed=20rework=20?= =?UTF-8?q?=E2=80=94=20pnpm=20store=20+=20wine=20apt=20caches,=20concurren?= =?UTF-8?q?t=20provisioning=20and=20gates,=20checksum-pinned=20Node,=208-c?= =?UTF-8?q?ore=20dispatch=20leg;=20fold=20PR=20#689=20lessons=20into=20the?= =?UTF-8?q?=20note?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...27-wine-windows-gates-experiment.i18n.yaml | 6 +- ...026-07-27-wine-windows-gates-experiment.md | 11 +- ...-07-27-wine-windows-gates-experiment.zh.md | 11 +- .github/workflows/exp-wine-windows.yml | 253 +++++++++++------- 4 files changed, 172 insertions(+), 109 deletions(-) diff --git a/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.i18n.yaml b/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.i18n.yaml index eb3909cc4e..fb51fef157 100644 --- a/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.i18n.yaml +++ b/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.i18n.yaml @@ -1,6 +1,6 @@ # Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: -# pnpm run verify-translation-pairing --write -2026-07-27-wine-windows-gates-experiment.md: 9f7856dfef229f8f02c85f5968082a0c857bbc94 -2026-07-27-wine-windows-gates-experiment.zh.md: cb185293d7f22723a96448a774bd27dd31e1bc28 +# pnpm run verify-translation-pairing --write .agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.md +2026-07-27-wine-windows-gates-experiment.md: 9e2db947eceee7e3e2fee63f8fe2ac90de1cd13d +2026-07-27-wine-windows-gates-experiment.zh.md: a4b938faa6ae27bd068db9a952ebb1432ec7ca3f diff --git a/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.md b/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.md index 9f7856dfef..9e2db947ec 100644 --- a/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.md +++ b/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.md @@ -12,9 +12,11 @@ The open question: can a plain Linux runner produce an equivalent win32 signal f ## Proposal -[exp-wine-windows.yml](../../../../.github/workflows/exp-wine-windows.yml) (self-path-filtered, plus manual dispatch) runs the blocking gate commands on `ubuntu-latest` under Wine with real Windows binaries: a downloaded win-x64 Node.js executes `tsc -b`, `tsdown`, and the VitePress production build, so the win32 branches of the toolchain — backslash path handling, `CreateProcess` spawn semantics, PE loading of `@esbuild/win32-x64`, and the rolldown/rollup MSVC `.node` addons — actually execute. +[exp-wine-windows.yml](../../../../.github/workflows/exp-wine-windows.yml) (self-path-filtered, plus manual dispatch) runs the blocking gate commands on `ubuntu-latest` under Wine with real Windows binaries: a checksum-verified win-x64 Node.js executes `tsc -b`, `tsdown`, and the VitePress production build, so the win32 branches of the toolchain — backslash path handling, `CreateProcess` spawn semantics, PE loading of `@esbuild/win32-x64`, and the rolldown/rollup MSVC `.node` addons — actually execute. -Dependencies install natively on Linux with `supportedArchitectures` extended to win32-x64, which materializes the Windows platform packages in the same store; the cmd-shim layer is bypassed by invoking each tool's JavaScript entrypoint directly, the same processes `run-gates` ultimately spawns. +Dependencies install natively on Linux with `supportedArchitectures` extended to win32-x64, which materializes the Windows platform packages in the same store; the cmd-shim layer is bypassed by invoking each tool's JavaScript entrypoint directly, the same processes `run-gates` ultimately spawns. `nodeLinker: hoisted` is load-bearing, not stylistic: the independent prototype in [PR #689](https://github.com/deepseek-harness/deepseek-harness/pull/689) kept pnpm's default isolated layout — including a faithful offline Windows-pnpm re-install over a Linux-prefetched store — and Windows Node under Wine still could not resolve `@esbuild/win32-x64` or load the koffi prebuild through the isolated symlink chain, failing before any repository gate ran. A flat layout with real files is what makes the gates reachable at all; #689's checksum pinning is adopted, while its Windows-pnpm-installs-the-tree goal is explicitly given up (the install contract stays Linux-tested here). + +The lane targets the wall clock of the Linux CI jobs (about two minutes), from four levers: the master-refreshed pnpm store cache (restore-only, same key as ci.yml), Wine provisioning (apt install, Windows Node download, `wineboot`) running concurrently with `pnpm install`, the two blocking surfaces running concurrently — the same shape `run-gates` gives them on native Windows — and an apt-archive cache keyed on the runner image so Wine's package downloads are paid once per image version. This is deliberately a fidelity probe, not a drop-in replacement: Wine reimplements the Win32 API over a case-sensitive ext4 (NTFS case-insensitivity is not emulated by default), provides no ConPTY, and substitutes its own security-descriptor and `MoveFileExW` semantics — exactly the surfaces the repo's `win32.ts` modules and PTY backend care about. The experiment measures which blocking gates pass, which fail for Wine reasons rather than product reasons, and the wall-clock cost relative to the recorded Windows benchmark lanes. @@ -26,6 +28,8 @@ Promotion, if the verdict is positive: fold the Wine lane in as the pull-request **A full Windows guest under QEMU/KVM inside the Linux runner.** Real NT kernel, so full fidelity including case-insensitive NTFS and ConPTY — but tens of minutes of image download and unattended install before the first gate runs. Explored as the sibling experiment branch `exp/kvm-windows-ci`; the two experiments price fidelity against latency. +**Windows pnpm performing the install under Wine ([PR #689](https://github.com/deepseek-harness/deepseek-harness/pull/689)).** The higher-fidelity variant of this same idea: MinGit and pnpm staged into the prefix, a Linux prefetch filling the store, then `pnpm install --offline` run by Windows Node so the install contract itself executes as win32. It reached the install but not the gates — Wine's networking could not reach the registry directly, and the isolated `node_modules` layout defeated resolution of the Windows platform packages even after a clean offline install. This lane trades that fidelity away (hoisted layout, Linux-side install) to reach the gates; the two records are complementary halves of the same verdict. + **Filesystem-semantics lanes on Linux (casefolded ext4, filename lint).** Catches the highest-frequency Windows breakage class for near-zero cost but proves nothing about win32 binaries. Explored as the sibling experiment branch `exp/casefold-windows-ci`. **Windows containers.** Not possible: Windows containers require a Windows host kernel; a hosted Linux runner cannot run them. @@ -34,7 +38,8 @@ Promotion, if the verdict is positive: fold the Wine lane in as the pull-request ## Acceptance criteria -- The workflow completes on `ubuntu-latest` with an independent pass/fail verdict per blocking gate (tsc, tsdown, production site) and a recorded wall-clock comparison against the Windows benchmark lanes. +- The workflow completes on `ubuntu-latest` with an independent pass/fail verdict per blocking surface (build, production site) and a recorded wall-clock comparison against both the paid Windows lane and the Linux CI jobs. +- End-to-end wall clock lands in the same band as the Linux CI jobs (minutes, not tens of minutes), demonstrating the pool-replacement case on cost as well as signal. - A decision is recorded here: promote the lane, keep it as a non-blocking canary, or reject it with the observed failure class. ## Risks diff --git a/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.zh.md b/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.zh.md index cb185293d7..a4b938faa6 100644 --- a/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.zh.md +++ b/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.zh.md @@ -12,9 +12,11 @@ Pull request 的 Windows 通道存在的意义是证明两个阻断性 win32 表 ## 提案 -[exp-wine-windows.yml](../../../../.github/workflows/exp-wine-windows.yml)（自身路径过滤，外加手动触发）在 `ubuntu-latest` 上通过 Wine 用真实 Windows 二进制运行阻断门禁命令：下载的 win-x64 Node.js 执行 `tsc -b`、`tsdown` 与 VitePress 生产构建，因此工具链的 win32 分支——反斜杠路径处理、`CreateProcess` 派生语义、`@esbuild/win32-x64` 的 PE 加载、以及 rolldown/rollup 的 MSVC `.node` 插件——都真正执行。 +[exp-wine-windows.yml](../../../../.github/workflows/exp-wine-windows.yml)（自身路径过滤，外加手动触发）在 `ubuntu-latest` 上通过 Wine 用真实 Windows 二进制运行阻断门禁命令：校验和验证过的 win-x64 Node.js 执行 `tsc -b`、`tsdown` 与 VitePress 生产构建，因此工具链的 win32 分支——反斜杠路径处理、`CreateProcess` 派生语义、`@esbuild/win32-x64` 的 PE 加载、以及 rolldown/rollup 的 MSVC `.node` 插件——都真正执行。 -依赖在 Linux 上原生安装，`supportedArchitectures` 扩展到 win32-x64，使 Windows 平台包物化进同一个 store；通过直接调用各工具的 JavaScript 入口绕开 cmd-shim 层，这正是 `run-gates` 最终派生的那些进程。 +依赖在 Linux 上原生安装，`supportedArchitectures` 扩展到 win32-x64，使 Windows 平台包物化进同一个 store；通过直接调用各工具的 JavaScript 入口绕开 cmd-shim 层，这正是 `run-gates` 最终派生的那些进程。`nodeLinker: hoisted` 是承重的，不是风格问题：[PR #689](https://github.com/deepseek-harness/deepseek-harness/pull/689) 的独立原型保留了 pnpm 默认的 isolated 布局——包括在 Linux 预取的 store 上忠实地用 Windows pnpm 离线重装——而 Wine 下的 Windows Node 依然无法穿过 isolated 符号链接链解析 `@esbuild/win32-x64` 或加载 koffi 预编译产物，在任何仓库门禁运行前就失败了。扁平的真实文件布局才让门禁变得可达；本通道采纳了 #689 的校验和固定，同时明确放弃其"Windows pnpm 安装依赖树"的目标（安装契约在此仍由 Linux 侧验证）。 + +该通道以 Linux CI 作业的墙钟（约两分钟）为目标，靠四个杠杆：master 刷新的 pnpm store 缓存（只恢复，与 ci.yml 同键）、Wine 供给（apt 安装、Windows Node 下载、`wineboot`）与 `pnpm install` 并发运行、两个阻断表面并发运行——与 `run-gates` 在原生 Windows 上给它们的形状相同——以及按 runner 镜像为键的 apt 归档缓存，使 Wine 的包下载每个镜像版本只付一次。 这刻意是一次保真度探针，而非直接替换：Wine 在大小写敏感的 ext4 之上重实现 Win32 API（默认不模拟 NTFS 的大小写不敏感）、不提供 ConPTY、并用自己的安全描述符与 `MoveFileExW` 语义替代——恰是本仓库 `win32.ts` 模块与 PTY 后端关心的表面。实验度量哪些阻断门禁通过、哪些因 Wine 原因而非产品原因失败，以及相对已记录 Windows 基准通道的墙钟成本。 @@ -26,6 +28,8 @@ Pull request 的 Windows 通道存在的意义是证明两个阻断性 win32 表 **在 Linux runner 内用 QEMU/KVM 跑完整 Windows 客户机。** 真实 NT 内核，保真度完整，包括大小写不敏感的 NTFS 与 ConPTY——但首个门禁运行前要花数十分钟下载镜像并做无人值守安装。作为兄弟实验分支 `exp/kvm-windows-ci` 探索；两个实验共同为保真度与延迟定价。 +**在 Wine 下由 Windows pnpm 执行安装（[PR #689](https://github.com/deepseek-harness/deepseek-harness/pull/689)）。** 同一想法的更高保真度变体：把 MinGit 与 pnpm 放进 prefix，用 Linux 预取填充 store，再由 Windows Node 运行 `pnpm install --offline`，让安装契约本身以 win32 身份执行。它到达了安装但没到达门禁——Wine 的网络无法直接访问 registry，且 isolated 的 `node_modules` 布局即便在干净的离线安装后也挫败了 Windows 平台包的解析。本通道用掉这份保真度（hoisted 布局、Linux 侧安装）来换取门禁可达；两份记录是同一裁决互补的两半。 + **Linux 上的文件系统语义通道（casefold ext4、文件名 lint）。** 以近零成本捕获最高频的 Windows 破坏类别，但对 win32 二进制什么也证明不了。作为兄弟实验分支 `exp/casefold-windows-ci` 探索。 **Windows 容器。** 不可行：Windows 容器要求 Windows 宿主内核；托管 Linux runner 无法运行。 @@ -34,7 +38,8 @@ Pull request 的 Windows 通道存在的意义是证明两个阻断性 win32 表 ## 验收标准 -- 该 workflow 在 `ubuntu-latest` 上完成，对每个阻断门禁（tsc、tsdown、生产站点）给出独立的通过/失败裁决，并记录与 Windows 基准通道的墙钟对比。 +- 该 workflow 在 `ubuntu-latest` 上完成，对每个阻断表面（构建、生产站点）给出独立的通过/失败裁决，并记录与付费 Windows 通道及 Linux CI 作业两者的墙钟对比。 +- 端到端墙钟落在 Linux CI 作业的同一档位（分钟级，而非数十分钟），从成本与信号两方面共同论证替换池的理由。 - 在此记录一项决定：晋升该通道、保留为非阻断金丝雀、或以观察到的失败类别否决。 ## 风险 diff --git a/.github/workflows/exp-wine-windows.yml b/.github/workflows/exp-wine-windows.yml index 9ebc3ccc79..e67c0e7d79 100644 --- a/.github/workflows/exp-wine-windows.yml +++ b/.github/workflows/exp-wine-windows.yml @@ -1,10 +1,16 @@ # EXPERIMENT: run the blocking Windows CI gates on a Linux runner through -# Wine, and execute the gate commands with a real Windows Node.js binary. -# Dependency provisioning happens natively on Linux with +# Wine with a real Windows Node.js binary, at roughly the wall clock of the +# Linux CI jobs (~2 min). Speed comes from four levers: the master-refreshed +# pnpm store cache, provisioning Wine concurrently with the dependency +# install, running the two blocking surfaces concurrently (the same shape +# run-gates gives them on native Windows), and an apt package cache for Wine +# itself. Dependency provisioning happens natively on Linux with # `supportedArchitectures` extended to win32-x64 so the Windows -# esbuild/rolldown/rollup binaries are present in the store. The pnpm-run/cmd -# shim layer is deliberately bypassed (a Linux install writes POSIX shims -# only), so each gate invokes its tool's JavaScript entrypoint directly — the +# esbuild/rolldown/rollup binaries are present, and `nodeLinker: hoisted` +# because Windows Node under Wine does not realpath pnpm's isolated-layout +# Unix symlinks — the sibling prototype in PR #689 kept the isolated layout +# and failed on exactly that. The pnpm-run/cmd shim layer is deliberately +# bypassed; each gate invokes its tool's JavaScript entrypoint directly — the # same commands run-gates ultimately spawns. Owning rationale and promotion # criteria: # .agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.md @@ -28,11 +34,17 @@ env: jobs: wine-blocking-gates: - name: wine / blocking windows gates - # Deliberately the cheapest hosted substrate: if Wine holds up here, the - # lane needs no special pool at all. - runs-on: ubuntu-latest - timeout-minutes: 120 + name: wine / blocking windows gates (${{ matrix.runner }}) + # Pull requests run the free standard runner only; a manual dispatch adds + # the 8-core benchmark pool for a like-for-like core-count comparison. + # The larger leg stays dispatch-only because those restricted pools can + # queue indefinitely (observed on the sibling KVM experiment). + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + runner: ${{ fromJSON(github.event_name == 'workflow_dispatch' && '["ubuntu-latest", "dsh-ubuntu-24-04-8core"]' || '["ubuntu-latest"]') }} + timeout-minutes: 30 env: WINEDEBUG: '-all' WINEARCH: win64 @@ -47,18 +59,39 @@ jobs: with: node-version: ${{ env.PRIMARY_NODE_VERSION }} - - name: Enable corepack and install with win32-x64 artifacts + # The default-branch pnpm store cache ci.yml maintains; restore-only, + # same key, so this lane rides the cache master already refreshes. + - uses: actions/cache/restore@v4 + with: + path: /home/runner/.local/share/pnpm/store/v11 + key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm- + + - name: Compose Wine apt cache key + id: wine-cache-key + run: echo "key=wine-debs-${ImageOS:-linux}-${ImageVersion:-v0}" >> "$GITHUB_OUTPUT" + + - uses: actions/cache@v4 + with: + path: ~/wine-debs + key: ${{ steps.wine-cache-key.outputs.key }} + + - name: Install dependencies and provision Wine concurrently run: | corepack enable + # Experiment-only install-time overrides. supportedArchitectures # additionally materializes the win32-x64 platform packages - # (@esbuild/win32-x64, rolldown and rollup MSVC bindings) that the - # Windows toolchain resolves at runtime. nodeLinker: hoisted lays - # node_modules out flat with real files: Windows Node under Wine - # does not realpath pnpm's Unix symlinks, so the default isolated - # layout breaks transitive ESM resolution (tsdown -> ansis, - # vite -> rollup). Neither override is recorded in the lockfile, so - # --frozen-lockfile stays valid. + # (@esbuild/win32-x64, rolldown and rollup MSVC bindings) the + # Windows toolchain resolves at runtime; nodeLinker: hoisted lays + # node_modules out flat with real files because Windows Node under + # Wine does not realpath pnpm's isolated-layout symlinks (PR #689's + # failure mode). Neither override is recorded in the lockfile, so + # --frozen-lockfile stays valid. --ignore-scripts skips the Linux + # esbuild/node-pty/lefthook lifecycle scripts: no gate in this lane + # loads them, and the win32 binaries ship prebuilt in their + # packages. cat >> pnpm-workspace.yaml <<'EOF' nodeLinker: hoisted @@ -66,61 +99,60 @@ jobs: os: [current, win32] cpu: [current, x64] EOF - pnpm install --frozen-lockfile - - name: Resolve tool entrypoints in the hoisted layout - run: | - resolve() { - local name="$1"; shift - for p in "$@"; do - if [ -f "$p" ]; then echo "$name=$PWD/$p" >> "$GITHUB_ENV"; return 0; fi + pnpm install --frozen-lockfile --ignore-scripts & + install_pid=$! + + provision_wine() { + set -euo pipefail + # Wine from the apt cache when present; else download the full + # dependency closure once and keep it for the next run. The + # `wine` dispatcher package (not bare `wine64`) is what puts a + # binary on PATH. + if compgen -G "$HOME/wine-debs/*.deb" > /dev/null; then + sudo apt-get install -y --no-install-recommends "$HOME"/wine-debs/*.deb + else + sudo apt-get update + sudo apt-get install -y --no-install-recommends --download-only wine + mkdir -p "$HOME/wine-debs" + cp /var/cache/apt/archives/*.deb "$HOME/wine-debs/" 2>/dev/null || true + sudo apt-get install -y --no-install-recommends wine + fi + WINE_BIN='' + for candidate in "$(command -v wine || true)" "$(command -v wine64 || true)" /usr/lib/wine/wine64; do + if [ -n "$candidate" ] && [ -x "$candidate" ]; then WINE_BIN="$candidate"; break; fi done - echo "::error::$name not found at any of: $*"; return 1 + [ -n "$WINE_BIN" ] || { echo '::error::no wine binary found after install'; exit 1; } + echo "WINE_BIN=$WINE_BIN" >> "$GITHUB_ENV" + + # Windows Node for the repo's primary line, checksum-verified + # against the same dist directory (adopted from PR #689). + version=$(curl -fsSL https://nodejs.org/dist/index.json \ + | jq -r --arg p "v${PRIMARY_NODE_VERSION}." '[.[] | select(.version | startswith($p))][0].version') + echo "Windows Node: $version" + curl -fsSL -o "$RUNNER_TEMP/node-win.zip" \ + "https://nodejs.org/dist/${version}/node-${version}-win-x64.zip" + curl -fsSL "https://nodejs.org/dist/${version}/SHASUMS256.txt" \ + | awk -v a="node-${version}-win-x64.zip" '$2 == a { print $1 " '"$RUNNER_TEMP"'/node-win.zip" }' \ + | sha256sum --check - + unzip -q "$RUNNER_TEMP/node-win.zip" -d "$RUNNER_TEMP/node-win" + echo "NODE_WIN=$RUNNER_TEMP/node-win/node-${version}-win-x64/node.exe" >> "$GITHUB_ENV" + + "$WINE_BIN" wineboot --init || true + wineserver -w || true } - resolve TSC_JS node_modules/typescript/bin/tsc - resolve TSDOWN_JS node_modules/tsdown/dist/run.mjs - resolve VITEPRESS_JS website/node_modules/vitepress/bin/vitepress.js node_modules/vitepress/bin/vitepress.js - # VitePress links vue into the site's node_modules at build time; - # Wine cannot CREATE Windows symlinks (ENOTSUP) but follows - # pre-existing Unix ones, so lay the link down host-side. - if [ -d node_modules/vue ] && [ ! -e website/node_modules/vue ]; then - mkdir -p website/node_modules - ln -s ../../node_modules/vue website/node_modules/vue - fi + provision_wine & + wine_pid=$! - - name: Install Wine (64-bit) - run: | - sudo apt-get update - # `wine` is the /usr/bin/wine dispatcher; its dependency pulls the - # wine64 loader. Ubuntu's wine64 package alone leaves nothing on - # PATH (the loader sits at /usr/lib/wine/wine64). - sudo apt-get install -y --no-install-recommends wine - WINE_BIN='' - for candidate in "$(command -v wine || true)" "$(command -v wine64 || true)" /usr/lib/wine/wine64; do - if [ -n "$candidate" ] && [ -x "$candidate" ]; then WINE_BIN="$candidate"; break; fi - done - if [ -z "$WINE_BIN" ]; then - echo '::error::no wine binary found after install' - dpkg -L wine wine64 2>/dev/null | grep -E '/bin/|wine64$' || true - exit 1 - fi - echo "WINE_BIN=$WINE_BIN" >> "$GITHUB_ENV" - "$WINE_BIN" --version + install_status=0 + wait "$install_pid" || install_status=$? + wine_status=0 + wait "$wine_pid" || wine_status=$? + if (( install_status != 0 )); then exit "$install_status"; fi + exit "$wine_status" - - name: Fetch Windows Node.js + - name: Resolve entrypoints, link vue, smoke Windows Node run: | - version=$(curl -fsSL https://nodejs.org/dist/index.json \ - | jq -r --arg p "v${PRIMARY_NODE_VERSION}." '[.[] | select(.version | startswith($p))][0].version') - echo "Windows Node: $version" - curl -fsSL -o "$RUNNER_TEMP/node-win.zip" \ - "https://nodejs.org/dist/${version}/node-${version}-win-x64.zip" - unzip -q "$RUNNER_TEMP/node-win.zip" -d "$RUNNER_TEMP/node-win" - echo "NODE_WIN=$RUNNER_TEMP/node-win/node-${version}-win-x64/node.exe" >> "$GITHUB_ENV" - - - name: Boot Wine prefix and smoke Windows Node - run: | - "$WINE_BIN" wineboot --init || true - wineserver -w || true # Node under Wine cannot attach stdio to the Actions runner's pipes # (Socket open EBADF at bootstrap), so every invocation runs through # this wrapper: stdio to a regular file, replayed after exit. @@ -134,39 +166,60 @@ jobs: exit "$status" SH chmod +x "$RUNNER_TEMP/wine-node.sh" + + resolve() { + local name="$1"; shift + for p in "$@"; do + if [ -f "$p" ]; then echo "$name=$PWD/$p" >> "$GITHUB_ENV"; return 0; fi + done + echo "::error::$name not found at any of: $*"; return 1 + } + resolve TSC_JS node_modules/typescript/bin/tsc + resolve TSDOWN_JS node_modules/tsdown/dist/run.mjs + resolve VITEPRESS_JS website/node_modules/vitepress/bin/vitepress.js node_modules/vitepress/bin/vitepress.js + + # VitePress links vue into the site's node_modules at build time; + # Wine cannot CREATE Windows symlinks (ENOTSUP) but follows + # pre-existing Unix ones, so lay the link down host-side. + if [ -d node_modules/vue ] && [ ! -e website/node_modules/vue ]; then + mkdir -p website/node_modules + ln -s ../../node_modules/vue website/node_modules/vue + fi + "$RUNNER_TEMP/wine-node.sh" "$RUNNER_TEMP/smoke.log" -p "'smoke: ' + process.platform + ' ' + process.arch + ' ' + process.version" - # The continue-on-error gates below mirror ci-windows-blocking - # (scripts/run-gates.ts): `build` = tsc -b + tsdown, `production site` = - # vitepress build. Each reports independently so one failure does not - # hide the others' results; the summary step at the end owns the job - # conclusion. - - name: 'Gate: tsc -b (Windows node under Wine)' - id: tsc - continue-on-error: true - timeout-minutes: 45 - run: '"$RUNNER_TEMP/wine-node.sh" "$RUNNER_TEMP/tsc.log" "$TSC_JS" -b --pretty false' - - - name: 'Gate: tsdown (Windows node under Wine)' - id: tsdown - continue-on-error: true - timeout-minutes: 30 - run: '"$RUNNER_TEMP/wine-node.sh" "$RUNNER_TEMP/tsdown.log" "$TSDOWN_JS"' - - - name: 'Gate: production site (Windows node under Wine)' - id: site - continue-on-error: true - timeout-minutes: 30 - working-directory: website - run: '"$RUNNER_TEMP/wine-node.sh" "$RUNNER_TEMP/site.log" "$VITEPRESS_JS" build .' - - - name: Report gate outcomes - env: - TSC: ${{ steps.tsc.outcome }} - TSDOWN: ${{ steps.tsdown.outcome }} - SITE: ${{ steps.site.outcome }} + # The two blocking surfaces run concurrently, the same shape run-gates + # gives ci-windows-blocking on native Windows (DSH_GATE_CONCURRENCY): + # `build` = tsc -b then tsdown, `production site` = the VitePress + # build. Both statuses are captured so one failure cannot hide the + # other's result. + - name: Run blocking Windows gates concurrently under Wine + timeout-minutes: 20 run: | - echo "tsc: $TSC" - echo "tsdown: $TSDOWN" - echo "production site: $SITE" - [ "$TSC" = success ] && [ "$TSDOWN" = success ] && [ "$SITE" = success ] + build_gate() { + "$RUNNER_TEMP/wine-node.sh" "$RUNNER_TEMP/tsc.log" "$TSC_JS" -b --pretty false || return $? + "$RUNNER_TEMP/wine-node.sh" "$RUNNER_TEMP/tsdown.log" "$TSDOWN_JS" + } + site_gate() { + cd website + "$RUNNER_TEMP/wine-node.sh" "$RUNNER_TEMP/site.log" "$VITEPRESS_JS" build . + } + start=$SECONDS + build_gate > "$RUNNER_TEMP/build-gate.out" 2>&1 & + build_pid=$! + site_gate > "$RUNNER_TEMP/site-gate.out" 2>&1 & + site_pid=$! + build_status=0 + wait "$build_pid" || build_status=$? + site_status=0 + wait "$site_pid" || site_status=$? + echo "== build gate (exit $build_status, $((SECONDS - start))s elapsed) ==" + tail -n 120 "$RUNNER_TEMP/build-gate.out" + echo "== production site gate (exit $site_status, $((SECONDS - start))s elapsed) ==" + tail -n 120 "$RUNNER_TEMP/site-gate.out" + if (( build_status != 0 )); then exit "$build_status"; fi + exit "$site_status" + + - name: Shut down wineserver + if: always() + run: wineserver -k 2>/dev/null || true From 38eb521e004b46d293ebc391ca0a498c7d814151 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Mon, 27 Jul 2026 12:30:10 +0800 Subject: [PATCH 08/14] ci(exp-wine): document apt-cache scoping across triggers --- .github/workflows/exp-wine-windows.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/exp-wine-windows.yml b/.github/workflows/exp-wine-windows.yml index e67c0e7d79..e9a79a18a7 100644 --- a/.github/workflows/exp-wine-windows.yml +++ b/.github/workflows/exp-wine-windows.yml @@ -68,6 +68,11 @@ jobs: restore-keys: | ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm- + # Keyed on the runner image so a new image version re-downloads once. + # Cache scoping: each trigger seeds its own scope (pull_request → the + # PR merge ref, dispatch → the branch); only same-scope reruns hit. + # Promotion to ci.yml would let master seed the shared default-branch + # scope every trigger reads, as the pnpm store cache already does. - name: Compose Wine apt cache key id: wine-cache-key run: echo "key=wine-debs-${ImageOS:-linux}-${ImageVersion:-v0}" >> "$GITHUB_OUTPUT" From b052cd11613d4343fae8fb19f6df6f68275731f1 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Mon, 27 Jul 2026 12:39:29 +0800 Subject: [PATCH 09/14] docs(exp-wine): record measured warm-cache result and the queued 8-core leg --- .../2026-07-27-wine-windows-gates-experiment.i18n.yaml | 4 ++-- .../process/2026-07-27-wine-windows-gates-experiment.md | 2 ++ .../process/2026-07-27-wine-windows-gates-experiment.zh.md | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.i18n.yaml b/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.i18n.yaml index fb51fef157..c39841966d 100644 --- a/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.i18n.yaml +++ b/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.md -2026-07-27-wine-windows-gates-experiment.md: 9e2db947eceee7e3e2fee63f8fe2ac90de1cd13d -2026-07-27-wine-windows-gates-experiment.zh.md: a4b938faa6ae27bd068db9a952ebb1432ec7ca3f +2026-07-27-wine-windows-gates-experiment.md: 47a37ddb48f4321f916c7f7a0cb96ae80b133103 +2026-07-27-wine-windows-gates-experiment.zh.md: 3a912861110a06b39bfb2c37395fc6a061bdfbe6 diff --git a/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.md b/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.md index 9e2db947ec..47a37ddb48 100644 --- a/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.md +++ b/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.md @@ -18,6 +18,8 @@ Dependencies install natively on Linux with `supportedArchitectures` extended to The lane targets the wall clock of the Linux CI jobs (about two minutes), from four levers: the master-refreshed pnpm store cache (restore-only, same key as ci.yml), Wine provisioning (apt install, Windows Node download, `wineboot`) running concurrently with `pnpm install`, the two blocking surfaces running concurrently — the same shape `run-gates` gives them on native Windows — and an apt-archive cache keyed on the runner image so Wine's package downloads are paid once per image version. +Measured on 2026-07-27: 2m46s end-to-end on a warm-cache pull-request run (setup and cache restores ~17s, concurrent install+provision 33s, concurrent gates 110s), against 1.5–2.5 minutes for the Linux CI jobs and 7–9 minutes for the paid Windows lane; a cold-cache run pays roughly one extra minute. The 8-core benchmark leg never left the queue — the restricted `dsh-ubuntu-*` pools were also observed queueing indefinitely from the sibling KVM experiment — so the standard-runner number stands as the result, and no larger box is needed to hit the target. + This is deliberately a fidelity probe, not a drop-in replacement: Wine reimplements the Win32 API over a case-sensitive ext4 (NTFS case-insensitivity is not emulated by default), provides no ConPTY, and substitutes its own security-descriptor and `MoveFileExW` semantics — exactly the surfaces the repo's `win32.ts` modules and PTY backend care about. The experiment measures which blocking gates pass, which fail for Wine reasons rather than product reasons, and the wall-clock cost relative to the recorded Windows benchmark lanes. Promotion, if the verdict is positive: fold the Wine lane in as the pull-request Windows signal for blocking gates and demote the real-Windows pool to the master serial reference; otherwise record the failure class here and keep the pool. diff --git a/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.zh.md b/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.zh.md index a4b938faa6..3a91286111 100644 --- a/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.zh.md +++ b/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.zh.md @@ -18,6 +18,8 @@ Pull request 的 Windows 通道存在的意义是证明两个阻断性 win32 表 该通道以 Linux CI 作业的墙钟（约两分钟）为目标，靠四个杠杆：master 刷新的 pnpm store 缓存（只恢复，与 ci.yml 同键）、Wine 供给（apt 安装、Windows Node 下载、`wineboot`）与 `pnpm install` 并发运行、两个阻断表面并发运行——与 `run-gates` 在原生 Windows 上给它们的形状相同——以及按 runner 镜像为键的 apt 归档缓存，使 Wine 的包下载每个镜像版本只付一次。 +2026-07-27 实测：热缓存 pull request 运行端到端 2 分 46 秒（准备与缓存恢复约 17 秒，并发安装+供给 33 秒，并发门禁 110 秒），对照 Linux CI 作业的 1.5–2.5 分钟与付费 Windows 通道的 7–9 分钟；冷缓存约多付一分钟。8 核基准腿从未离开队列——受限的 `dsh-ubuntu-*` 池在兄弟 KVM 实验中也被观察到无限排队——因此标准 runner 的数字即为结果，达标不需要更大的机器。 + 这刻意是一次保真度探针，而非直接替换：Wine 在大小写敏感的 ext4 之上重实现 Win32 API（默认不模拟 NTFS 的大小写不敏感）、不提供 ConPTY、并用自己的安全描述符与 `MoveFileExW` 语义替代——恰是本仓库 `win32.ts` 模块与 PTY 后端关心的表面。实验度量哪些阻断门禁通过、哪些因 Wine 原因而非产品原因失败，以及相对已记录 Windows 基准通道的墙钟成本。 若结论为正则晋升：把 Wine 通道并入为 pull request 的阻断门禁 Windows 信号，将真实 Windows 池降级为 master 串行参照；否则在此记录失败类别并保留该池。 From 109b469a7e52a1a62e9355833001a0257bb7740d Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Mon, 27 Jul 2026 12:41:36 +0800 Subject: [PATCH 10/14] fix(tool-web): bound conversion depth and complete fetch output Two review findings on the turndown swap, both verified empirically: - Unclosed-tag nesting makes the synchronous turndown/domino walk superlinear (measured: depth 512 ~0.15s, 2k ~2s, 20k ~5s), during which the cooperative fetchTimeoutMs timer cannot fire. renderBody now preflights nesting depth with a linear tag scan and passes bodies past 512 levels through raw; the try/catch stays for markup the scan cannot see (comment-hidden tags), simulated in tests via a converter throw. - Markdown escaping can expand converted HTML ~2x (100k underscores render as 200k chars), so provider body caps no longer bounded the model-visible result. formatFetchOutput now caps the complete output (header + body + footer) under new fetchMaxOutputChars config (default 200000 = 2x the local provider's default body cap), reusing the truncation notice. README EN+ZH, config catalog, Agent Note EN+ZH updated; the new web-fetch fixture is migrated to the packed layout master now requires; tool-web coverage stays 100% per-file. --- ...ndown-for-tool-web-html-markdown.i18n.yaml | 4 +- ...-26-turndown-for-tool-web-html-markdown.md | 4 +- ...-turndown-for-tool-web-html-markdown.zh.md | 4 +- docs/config-catalog.md | 6 +- .../tests/snapshots/web-fetch/session.jsonl | 102 +----------------- packages/web/tool-web/README.i18n.yaml | 4 +- packages/web/tool-web/README.md | 5 +- packages/web/tool-web/README.zh.md | 5 +- packages/web/tool-web/src/fetch.ts | 86 ++++++++++++--- packages/web/tool-web/src/index.ts | 19 +++- packages/web/tool-web/tests/tool-web.spec.ts | 70 ++++++++++-- 11 files changed, 172 insertions(+), 137 deletions(-) diff --git a/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.i18n.yaml b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.i18n.yaml index 60a5d9aca7..a114e32885 100644 --- a/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.i18n.yaml +++ b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -2026-07-26-turndown-for-tool-web-html-markdown.md: c72decc336055f3b78dafdf98f2be3771b833cdb -2026-07-26-turndown-for-tool-web-html-markdown.zh.md: 30667b62538ec50608cae461b5cdf651b48e2731 +2026-07-26-turndown-for-tool-web-html-markdown.md: c7ef4bf538cc949eec8463c8a2ac750685d1a715 +2026-07-26-turndown-for-tool-web-html-markdown.zh.md: 3104dac3cd6db516396b6773f5d2185f3da22ca3 diff --git a/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md index c72decc336..c7ef4bf538 100644 --- a/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md +++ b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md @@ -10,7 +10,7 @@ English | [中文](2026-07-26-turndown-for-tool-web-html-markdown.zh.md) ## Decision -`packages/web/tool-web/src/fetch.ts` owns a module-level [`turndown`](https://github.com/mixmark-io/turndown) instance (`headingStyle: 'atx'`, `codeBlockStyle: 'fenced'`, `bulletListMarker: '-'` — fixed model-facing presentation, not deployment tunables) with `@joplin/turndown-plugin-gfm`'s composite `gfm` plugin for tables/strikethrough and `remove(['script', 'style', 'noscript'])` replacing the old wholesale drops. `renderBody`'s `html` arm calls it in a try/catch falling back to the raw HTML body: the regex version could never throw, while turndown/domino's recursive DOM walk overflows with a `RangeError` at a few thousand nesting levels (measured: 4k throws on the main thread, 8k in a worker thread), and a degraded page beats an error for a body the provider already decoded. `html.ts` and its conversion tests are deleted; the fallback and the status-header/truncation-footer formatting are tested in `tests/tool-web.spec.ts`, and the README's Known Limitations trades the regex-converter caveat for the pathological-nesting fallback. The gfm plugin ships no types; `src/turndown-plugin-gfm.d.ts` declares the one imported export over `@types/turndown` (a devDependency). +`packages/web/tool-web/src/fetch.ts` owns a module-level [`turndown`](https://github.com/mixmark-io/turndown) instance (`headingStyle: 'atx'`, `codeBlockStyle: 'fenced'`, `bulletListMarker: '-'` — fixed model-facing presentation, not deployment tunables) with `@joplin/turndown-plugin-gfm`'s composite `gfm` plugin for tables/strikethrough and `remove(['script', 'style', 'noscript'])` replacing the old wholesale drops. `renderBody`'s `html` arm guards the conversion twice: a linear tag-scan preflight passes bodies nested past 512 levels through raw (the synchronous walk is superlinear on unclosed nesting — measured seconds at 20k levels — during which the cooperative timeout cannot fire), and a try/catch falls back to the raw HTML when turndown still throws on markup the scan cannot see; a degraded page beats an error for a body the provider already decoded. `formatFetchOutput` bounds the complete output (`fetchMaxOutputChars` config, default 200,000) because markdown escaping can expand converted HTML to ~2× a provider's body cap. `html.ts` and its conversion tests are deleted; the fallback and the status-header/truncation-footer formatting are tested in `tests/tool-web.spec.ts`, and the README's Known Limitations trades the regex-converter caveat for the pathological-nesting fallback. The gfm plugin ships no types; `src/turndown-plugin-gfm.d.ts` declares the one imported export over `@types/turndown` (a devDependency). The dependency-weight question the proposal flagged resolves in favor of the swap: `@deepseek-ai/dsh-tool-web` is in the single-file-executable closure ([single-exe note](../architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.md)), and the exe's asset globs would pack ~7.9 MB of the three packages as published — but ~6 MB of that is `@mixmark-io/domino`'s test corpus (`test/**`), with runtime `lib/` at ~550 KB against a ~174 MB artifact, under 0.5% either way. @@ -33,5 +33,5 @@ The previously-missing keyless `web_fetch` snapshot ships with the change as the ## Testing -- `packages/web/tool-web/tests/tool-web.spec.ts` covers the turndown conversion surface (entities, links, tables, nesting, script/style/noscript removal) through `renderBody`, and the raw-HTML fallback with a measured reliably-overflowing 20k-level nesting input; per-file coverage on the package src is 100%. +- `packages/web/tool-web/tests/tool-web.spec.ts` covers the turndown conversion surface (entities, links, tables, nesting, script/style/noscript removal) through `renderBody`, the fast raw-HTML passthrough for 20k-level nesting, the depth scan's void/self-closing/unbalanced cases, the residual converter-throw fallback, and the whole-output cap at expanding, exact, and tiny budgets; per-file coverage on the package src is 100%. - The `web-fetch` acp-agent snapshot pins the assembled behavior keylessly end to end (real Loader composition, real HTTP fetch, real conversion). diff --git a/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.zh.md b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.zh.md index 30667b6253..3104dac3cd 100644 --- a/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.zh.md +++ b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.zh.md @@ -10,7 +10,7 @@ Status: implemented ## 决策 -`packages/web/tool-web/src/fetch.ts` 持有一个模块级 [`turndown`](https://github.com/mixmark-io/turndown) 实例（`headingStyle: 'atx'`、`codeBlockStyle: 'fenced'`、`bulletListMarker: '-'`——固定的面向模型呈现方式，不是部署可调项），配合 `@joplin/turndown-plugin-gfm` 的组合 `gfm` 插件提供表格／删除线支持，并用 `remove(['script', 'style', 'noscript'])` 替代旧实现的整体剥离。`renderBody` 的 `html` 分支把调用包在 try/catch 中，失败时回退为原始 HTML 主体：正则版本从不可能抛异常，而 turndown/domino 的递归 DOM 遍历在数千层嵌套（实测：主线程 4k 层抛出，worker 线程 8k 层抛出）会以 `RangeError` 栈溢出，对提供方已经解码的主体来说，降级页面好过报错。`html.ts` 及其转换测试已删除；回退路径与状态头、截断页脚的格式化在 `tests/tool-web.spec.ts` 中有测试覆盖，README 的 Known Limitations 用病态嵌套回退条目替换了正则转换器的警示说明。gfm 插件不带类型声明；`src/turndown-plugin-gfm.d.ts` 基于 `@types/turndown`（devDependency）声明了唯一被导入的导出。 +`packages/web/tool-web/src/fetch.ts` 持有一个模块级 [`turndown`](https://github.com/mixmark-io/turndown) 实例（`headingStyle: 'atx'`、`codeBlockStyle: 'fenced'`、`bulletListMarker: '-'`——固定的面向模型呈现方式，不是部署可调项），配合 `@joplin/turndown-plugin-gfm` 的组合 `gfm` 插件提供表格／删除线支持，并用 `remove(['script', 'style', 'noscript'])` 替代旧实现的整体剥离。`renderBody` 的 `html` 分支对转换做了双重防护：一次线性标签扫描预检把嵌套超过 512 层的主体直接原样透传（同步遍历在未闭合嵌套上呈超线性——实测 2 万层需要数秒——期间协作式超时无法触发），扫描看不到的标记若仍让 turndown 抛异常，则由 try/catch 回退为原始 HTML；对提供方已经解码的主体来说，降级页面好过报错。`formatFetchOutput` 对完整输出设上限（`fetchMaxOutputChars` 配置，默认 200,000）：markdown 转义可能把转换后的 HTML 膨胀到提供方主体上限的约 2 倍。`html.ts` 及其转换测试已删除；透传、回退与整体输出上限，连同状态头、截断页脚的格式化，都在 `tests/tool-web.spec.ts` 中有测试覆盖，README 的 Known Limitations 用病态嵌套回退条目替换了正则转换器的警示说明。gfm 插件不带类型声明；`src/turndown-plugin-gfm.d.ts` 基于 `@types/turndown`（devDependency）声明了唯一被导入的导出。 提案标记的依赖体积问题的裁决结果支持替换：`@deepseek-ai/dsh-tool-web` 在单文件可执行文件闭包内（[single-exe 决策记录](../architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.md)），可执行文件的资产 glob 会把这三个包按发布原样打入约 7.9 MB——但其中约 6 MB 是 `@mixmark-io/domino` 的测试语料（`test/**`），运行时 `lib/` 仅约 550 KB，相对约 174 MB 的产物，两种口径都不到 0.5%。 @@ -33,5 +33,5 @@ Status: implemented ## 测试 -- `packages/web/tool-web/tests/tool-web.spec.ts` 通过 `renderBody` 覆盖 turndown 转换面（实体、链接、表格、嵌套、script/style/noscript 移除），并用实测可稳定溢出的 2 万层嵌套输入覆盖原始 HTML 回退；该包 src 的逐文件覆盖率为 100%。 +- `packages/web/tool-web/tests/tool-web.spec.ts` 通过 `renderBody` 覆盖 turndown 转换面（实体、链接、表格、嵌套、script/style/noscript 移除）、2 万层嵌套的快速原样透传、深度扫描的空元素／自闭合／不平衡用例、残余的转换器抛错回退，以及在膨胀、恰好、极小预算下的整体输出上限；该包 src 的逐文件覆盖率为 100%。 - acp-agent 的 `web-fetch` 快照无密钥地端到端固定组装后的行为（真实 Loader 组合、真实 HTTP 抓取、真实转换）。 diff --git a/docs/config-catalog.md b/docs/config-catalog.md index 721c432f73..742d04dd71 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -1656,7 +1656,7 @@ Source: [`packages/tasks/tool-tasks/src/index.ts:23`](../packages/tasks/tool-tas Requires: `tools` · `web` · `systemPrompt` ```ts config-catalog -/** Plugin config: which web tools to register, the source cap, and per-tool budgets. */ +/** Plugin config: which web tools to register, the source cap, per-tool budgets, and the fetch output cap. */ export interface Config { /** Register `web_search`. Defaults to true. */ search?: boolean @@ -1668,10 +1668,12 @@ export interface Config { fetchTimeoutMs?: number /** Cooperative timeout budget (ms) for `web_search`. Defaults to 30000. */ searchTimeoutMs?: number + /** Cap on one `web_fetch` output's characters (header, rendered body, and footer). Defaults to 200000. */ + fetchMaxOutputChars?: number } ``` -Source: [`packages/web/tool-web/src/index.ts:28`](../packages/web/tool-web/src/index.ts) +Source: [`packages/web/tool-web/src/index.ts:37`](../packages/web/tool-web/src/index.ts) ## `@deepseek-ai/dsh-tool-workflow` diff --git a/examples/acp-agent/tests/snapshots/web-fetch/session.jsonl b/examples/acp-agent/tests/snapshots/web-fetch/session.jsonl index c6c34bc8e3..47c97b1cba 100644 --- a/examples/acp-agent/tests/snapshots/web-fetch/session.jsonl +++ b/examples/acp-agent/tests/snapshots/web-fetch/session.jsonl @@ -5,75 +5,9 @@ {"type":"step/start","seq":3,"time":1785078727730,"data":{"turn":1,"step":1}} {"type":"request/header","seq":4,"time":1785078727731,"data":{"header":{"config":{"provider":"deepseek","model":"deepseek-v4-pro"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}} {"type":"assistant/chunk","seq":5,"time":1785078728804,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"reasoning"}}} -{"type":"assistant/chunk","seq":6,"time":1785078728805,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"The"}}} -{"type":"assistant/chunk","seq":7,"time":1785078728943,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" user"}}} -{"type":"assistant/chunk","seq":8,"time":1785078728989,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" wants"}}} -{"type":"assistant/chunk","seq":9,"time":1785078728989,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" me"}}} -{"type":"assistant/chunk","seq":10,"time":1785078728989,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" to"}}} -{"type":"assistant/chunk","seq":11,"time":1785078728990,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" use"}}} -{"type":"assistant/chunk","seq":12,"time":1785078728990,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" the"}}} -{"type":"assistant/chunk","seq":13,"time":1785078728990,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" web"}}} -{"type":"assistant/chunk","seq":14,"time":1785078729038,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"_f"}}} -{"type":"assistant/chunk","seq":15,"time":1785078729038,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"etch"}}} -{"type":"assistant/chunk","seq":16,"time":1785078729039,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" tool"}}} -{"type":"assistant/chunk","seq":17,"time":1785078729039,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" exactly"}}} -{"type":"assistant/chunk","seq":18,"time":1785078729085,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" once"}}} -{"type":"assistant/chunk","seq":19,"time":1785078729086,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" to"}}} -{"type":"assistant/chunk","seq":20,"time":1785078729086,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" fetch"}}} -{"type":"assistant/chunk","seq":21,"time":1785078729086,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" http"}}} -{"type":"assistant/chunk","seq":22,"time":1785078729086,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"://"}}} -{"type":"assistant/chunk","seq":23,"time":1785078729086,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"127"}}} -{"type":"assistant/chunk","seq":24,"time":1785078729132,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"."}}} -{"type":"assistant/chunk","seq":25,"time":1785078729133,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"0"}}} -{"type":"assistant/chunk","seq":26,"time":1785078729133,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"."}}} -{"type":"assistant/chunk","seq":27,"time":1785078729133,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"0"}}} -{"type":"assistant/chunk","seq":28,"time":1785078729133,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"."}}} -{"type":"assistant/chunk","seq":29,"time":1785078729133,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"1"}}} -{"type":"assistant/chunk","seq":30,"time":1785078729182,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":":"}}} -{"type":"assistant/chunk","seq":31,"time":1785078729182,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"431"}}} -{"type":"assistant/chunk","seq":32,"time":1785078729183,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"17"}}} -{"type":"assistant/chunk","seq":33,"time":1785078729183,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"/m"}}} -{"type":"assistant/chunk","seq":34,"time":1785078729183,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"enu"}}} -{"type":"assistant/chunk","seq":35,"time":1785078729183,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":".html"}}} -{"type":"assistant/chunk","seq":36,"time":1785078729230,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":","}}} -{"type":"assistant/chunk","seq":37,"time":1785078729230,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" then"}}} -{"type":"assistant/chunk","seq":38,"time":1785078729230,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" reply"}}} -{"type":"assistant/chunk","seq":39,"time":1785078729230,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" with"}}} -{"type":"assistant/chunk","seq":40,"time":1785078729230,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" exactly"}}} -{"type":"assistant/chunk","seq":41,"time":1785078729231,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" \""}}} -{"type":"assistant/chunk","seq":42,"time":1785078729276,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"D"}}} -{"type":"assistant/chunk","seq":43,"time":1785078729277,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"ONE"}}} -{"type":"assistant/chunk","seq":44,"time":1785078729277,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"\"."}}} -{"type":"assistant/chunk","seq":45,"time":1785078729277,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" Let"}}} -{"type":"assistant/chunk","seq":46,"time":1785078729322,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" me"}}} -{"type":"assistant/chunk","seq":47,"time":1785078729323,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" do"}}} -{"type":"assistant/chunk","seq":48,"time":1785078729323,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":" that"}}} -{"type":"assistant/chunk","seq":49,"time":1785078729323,"data":{"turn":1,"step":1,"chunk":{"type":"reasoning-delta","index":0,"text":"."}}} +{"type":"reasoning-chunks","seq0":6,"time0":1785078728805,"data":{"turn":1,"step":1,"index":0,"dt":[138,46,0,0,1,0,0,48,0,1,0,46,1,0,0,0,0,46,1,0,0,0,0,49,0,1,0,0,0,47,0,0,0,0,1,45,1,0,0,45,1,0,0],"texts":["The"," user"," wants"," me"," to"," use"," the"," web","_f","etch"," tool"," exactly"," once"," to"," fetch"," http","://","127",".","0",".","0",".","1",":","431","17","/m","enu",".html",","," then"," reply"," with"," exactly"," \"","D","ONE","\"."," Let"," me"," do"," that","."]}} {"type":"assistant/chunk","seq":50,"time":1785078729463,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":1,"blockType":"tool-call"}}} -{"type":"assistant/chunk","seq":51,"time":1785078729464,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":""}}} -{"type":"assistant/chunk","seq":52,"time":1785078729511,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"{"}}} -{"type":"assistant/chunk","seq":53,"time":1785078729511,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"\""}}} -{"type":"assistant/chunk","seq":54,"time":1785078729511,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"url"}}} -{"type":"assistant/chunk","seq":55,"time":1785078729511,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"\""}}} -{"type":"assistant/chunk","seq":56,"time":1785078729511,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":": "}}} -{"type":"assistant/chunk","seq":57,"time":1785078729557,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"\""}}} -{"type":"assistant/chunk","seq":58,"time":1785078729557,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"http"}}} -{"type":"assistant/chunk","seq":59,"time":1785078729557,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"://"}}} -{"type":"assistant/chunk","seq":60,"time":1785078729558,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"127"}}} -{"type":"assistant/chunk","seq":61,"time":1785078729604,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"."}}} -{"type":"assistant/chunk","seq":62,"time":1785078729604,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"0"}}} -{"type":"assistant/chunk","seq":63,"time":1785078729604,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"."}}} -{"type":"assistant/chunk","seq":64,"time":1785078729604,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"0"}}} -{"type":"assistant/chunk","seq":65,"time":1785078729604,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"."}}} -{"type":"assistant/chunk","seq":66,"time":1785078729605,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"1"}}} -{"type":"assistant/chunk","seq":67,"time":1785078729651,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":":"}}} -{"type":"assistant/chunk","seq":68,"time":1785078729652,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"431"}}} -{"type":"assistant/chunk","seq":69,"time":1785078729652,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"17"}}} -{"type":"assistant/chunk","seq":70,"time":1785078729652,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"/m"}}} -{"type":"assistant/chunk","seq":71,"time":1785078729652,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"enu"}}} -{"type":"assistant/chunk","seq":72,"time":1785078729652,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":".html"}}} -{"type":"assistant/chunk","seq":73,"time":1785078729697,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"\""}}} -{"type":"assistant/chunk","seq":74,"time":1785078729698,"data":{"turn":1,"step":1,"chunk":{"type":"tool-call-delta","index":1,"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","argumentsDelta":"}"}}} +{"type":"tool-call-chunks","seq0":51,"time0":1785078729464,"data":{"turn":1,"step":1,"index":1,"dt":[47,0,0,0,0,46,0,0,1,46,0,0,0,0,1,46,1,0,0,0,0,45,1],"id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","args":["","{","\"","url","\"",": ","\"","http","://","127",".","0",".","0",".","1",":","431","17","/m","enu",".html","\"","}"]}} {"type":"assistant/chunk","seq":75,"time":1785078729803,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":0,"block":{"type":"reasoning","text":"The user wants me to use the web_fetch tool exactly once to fetch http://127.0.0.1:43117/menu.html, then reply with exactly \"DONE\". Let me do that."}}}} {"type":"assistant/chunk","seq":76,"time":1785078729803,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":1,"block":{"type":"tool-call","id":"call_00_sxjOyfDYN07koiE7jiIa5326","name":"web_fetch","arguments":"{\"url\": \"http://127.0.0.1:43117/menu.html\"}"}}}} {"type":"assistant/chunk","seq":77,"time":1785078729803,"data":{"turn":1,"step":1,"chunk":{"type":"usage","usage":{"inputTokens":5405,"outputTokens":103,"cacheReadTokens":0,"reasoningTokens":44}}}} @@ -84,37 +18,7 @@ {"type":"step/end","seq":82,"time":1785078729847,"data":{"turn":1,"step":1}} {"type":"step/start","seq":83,"time":1785078729848,"data":{"turn":1,"step":2}} {"type":"assistant/chunk","seq":84,"time":1785078730611,"data":{"turn":1,"step":2,"chunk":{"type":"block-start","index":0,"blockType":"reasoning"}}} -{"type":"assistant/chunk","seq":85,"time":1785078730612,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"The"}}} -{"type":"assistant/chunk","seq":86,"time":1785078730770,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" user"}}} -{"type":"assistant/chunk","seq":87,"time":1785078730824,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" asked"}}} -{"type":"assistant/chunk","seq":88,"time":1785078730825,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" me"}}} -{"type":"assistant/chunk","seq":89,"time":1785078730825,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" to"}}} -{"type":"assistant/chunk","seq":90,"time":1785078730825,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" fetch"}}} -{"type":"assistant/chunk","seq":91,"time":1785078730861,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" the"}}} -{"type":"assistant/chunk","seq":92,"time":1785078730862,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" URL"}}} -{"type":"assistant/chunk","seq":93,"time":1785078730909,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":","}}} -{"type":"assistant/chunk","seq":94,"time":1785078730956,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" then"}}} -{"type":"assistant/chunk","seq":95,"time":1785078731002,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" reply"}}} -{"type":"assistant/chunk","seq":96,"time":1785078731003,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" with"}}} -{"type":"assistant/chunk","seq":97,"time":1785078731003,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" exactly"}}} -{"type":"assistant/chunk","seq":98,"time":1785078731003,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" \""}}} -{"type":"assistant/chunk","seq":99,"time":1785078731050,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"D"}}} -{"type":"assistant/chunk","seq":100,"time":1785078731050,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"ONE"}}} -{"type":"assistant/chunk","seq":101,"time":1785078731050,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"\"."}}} -{"type":"assistant/chunk","seq":102,"time":1785078731050,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" I"}}} -{"type":"assistant/chunk","seq":103,"time":1785078731050,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"'ve"}}} -{"type":"assistant/chunk","seq":104,"time":1785078731051,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" fetched"}}} -{"type":"assistant/chunk","seq":105,"time":1785078731097,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" it"}}} -{"type":"assistant/chunk","seq":106,"time":1785078731140,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"."}}} -{"type":"assistant/chunk","seq":107,"time":1785078731141,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" Now"}}} -{"type":"assistant/chunk","seq":108,"time":1785078731141,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" I"}}} -{"type":"assistant/chunk","seq":109,"time":1785078731141,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" just"}}} -{"type":"assistant/chunk","seq":110,"time":1785078731189,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" reply"}}} -{"type":"assistant/chunk","seq":111,"time":1785078731189,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" with"}}} -{"type":"assistant/chunk","seq":112,"time":1785078731235,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":" \""}}} -{"type":"assistant/chunk","seq":113,"time":1785078731235,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"D"}}} -{"type":"assistant/chunk","seq":114,"time":1785078731235,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"ONE"}}} -{"type":"assistant/chunk","seq":115,"time":1785078731235,"data":{"turn":1,"step":2,"chunk":{"type":"reasoning-delta","index":0,"text":"\"."}}} +{"type":"reasoning-chunks","seq0":85,"time0":1785078730612,"data":{"turn":1,"step":2,"index":0,"dt":[158,54,1,0,0,36,1,47,47,46,1,0,0,47,0,0,0,0,1,46,43,1,0,0,48,0,46,0,0,0],"texts":["The"," user"," asked"," me"," to"," fetch"," the"," URL",","," then"," reply"," with"," exactly"," \"","D","ONE","\"."," I","'ve"," fetched"," it","."," Now"," I"," just"," reply"," with"," \"","D","ONE","\"."]}} {"type":"assistant/chunk","seq":116,"time":1785078731235,"data":{"turn":1,"step":2,"chunk":{"type":"block-start","index":1,"blockType":"text"}}} {"type":"assistant/chunk","seq":117,"time":1785078731236,"data":{"turn":1,"step":2,"chunk":{"type":"text-delta","index":1,"text":"D"}}} {"type":"assistant/chunk","seq":118,"time":1785078731282,"data":{"turn":1,"step":2,"chunk":{"type":"text-delta","index":1,"text":"ONE"}}} diff --git a/packages/web/tool-web/README.i18n.yaml b/packages/web/tool-web/README.i18n.yaml index 1e746ed566..44279a66be 100644 --- a/packages/web/tool-web/README.i18n.yaml +++ b/packages/web/tool-web/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -README.md: 5fe48ced81a2cd02197cf8cc10a7d6567b17ffca -README.zh.md: 34ad08e290166ee6db2cd7b836746541d18aad52 +README.md: 44cb1ba2a2f4e1fba7e192d8b6645e0447ebf221 +README.zh.md: 35b390dd5407af16d84ab391dd8351f784c60035 diff --git a/packages/web/tool-web/README.md b/packages/web/tool-web/README.md index 5fe48ced81..44cb1ba2a2 100644 --- a/packages/web/tool-web/README.md +++ b/packages/web/tool-web/README.md @@ -26,8 +26,9 @@ The normalized seam results are also the canonical tool values: `WebSearchResult | `searchMaxResults` | `8` | Upper bound on sources returned by one `web_search` call (the seam truncates a longer provider list and flags it). | | `fetchTimeoutMs` | `30000` | Cooperative tool-call timeout budget (ms) for `web_fetch`. | | `searchTimeoutMs` | `30000` | Cooperative tool-call timeout budget (ms) for `web_search`. | +| `fetchMaxOutputChars` | `200000` | Cap on one `web_fetch` output's characters — header, rendered body, and footer together; a cut body gets the truncation notice. | -`fetchTimeoutMs`/`searchTimeoutMs` declare each tool's cooperative timeout budget (attached as `ToolDefinition.timeoutMs`), enforced by [`@deepseek-ai/dsh-timeout-policy`](../../timeout/timeout-policy/README.md); the model-facing schema exposes no timeout argument. +`fetchTimeoutMs`/`searchTimeoutMs` declare each tool's cooperative timeout budget (attached as `ToolDefinition.timeoutMs`), enforced by [`@deepseek-ai/dsh-timeout-policy`](../../timeout/timeout-policy/README.md); the model-facing schema exposes no timeout argument. `fetchMaxOutputChars` bounds the complete rendered output because markdown escaping can expand converted HTML past a provider's body cap (worst case ~2×); the default is 2× the local provider's default 100,000-character body cap, so it never cuts what that bound already admits. ```yaml - id: tool-web @@ -126,6 +127,6 @@ Append-only; newly visible content follows the reusable request prefix and does ## Known Limitations and Deferred Work -- **HTML→markdown conversion falls back to raw HTML on pathological input** — [turndown](https://github.com/mixmark-io/turndown) (with GFM tables/strikethrough) converts fetched HTML through a real DOM, but its recursive walk overflows on absurdly deep nesting (thousands of levels); such a body passes through unconverted rather than erroring ([Agent Note](../../../.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md)). +- **HTML→markdown conversion falls back to raw HTML on pathological input** — [turndown](https://github.com/mixmark-io/turndown) (with GFM tables/strikethrough) converts fetched HTML through a real DOM, but the synchronous walk is superlinear on deep unclosed nesting, so bodies nested past a fixed 512-level preflight bound pass through unconverted (as does anything that still makes turndown throw) rather than stalling the event loop or erroring ([Agent Note](../../../.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md)). - **The model-facing surface is minimal by design, with promotions deferred** — `max_results` stays a config bound (not a model argument), and `web_fetch` takes only `url` (no `format`/`prompt`/LLM-summarization mode); both are named later steps in [the seam Agent Note](../../../.agents/notes/implemented/architecture/2026-06-24-web-capability-seam.md). - **No web-specific permission policy** — both tools execute without requesting `ctx.approval`; a deployment that needs confirmation must add a `tools/pre-execute` policy, and the package does not define persistent URL/domain grants. diff --git a/packages/web/tool-web/README.zh.md b/packages/web/tool-web/README.zh.md index 34ad08e290..35b390dd54 100644 --- a/packages/web/tool-web/README.zh.md +++ b/packages/web/tool-web/README.zh.md @@ -26,8 +26,9 @@ | `searchMaxResults` | `8` | 一次 `web_search` 调用返回的源数量上限（seam 截断更长的提供方列表并标记）。 | | `fetchTimeoutMs` | `30000` | `web_fetch` 的协作式工具调用超时预算（ms）。 | | `searchTimeoutMs` | `30000` | `web_search` 的协作式工具调用超时预算（ms）。 | +| `fetchMaxOutputChars` | `200000` | 单次 `web_fetch` 输出的字符上限——状态头、渲染后的主体与页脚合并计算；被截断的主体带截断提示。 | -`fetchTimeoutMs`／`searchTimeoutMs` 声明每个工具的协作式超时预算（附加为 `ToolDefinition.timeoutMs`），由 [`@deepseek-ai/dsh-timeout-policy`](../../timeout/timeout-policy/README.md) 强制执行；面向模型的 schema 不公开超时参数。 +`fetchTimeoutMs`／`searchTimeoutMs` 声明每个工具的协作式超时预算（附加为 `ToolDefinition.timeoutMs`），由 [`@deepseek-ai/dsh-timeout-policy`](../../timeout/timeout-policy/README.md) 强制执行；面向模型的 schema 不公开超时参数。`fetchMaxOutputChars` 对完整渲染输出设上限：markdown 转义可能让转换后的 HTML 超出提供方的主体上限（最坏约 2 倍）；默认值取本地提供方默认 100,000 字符主体上限的 2 倍，因此绝不会削减该上限本已允许的内容。 ```yaml - id: tool-web @@ -126,6 +127,6 @@ Use the web_fetch tool to retrieve the content of a specific HTTP(S) URL (for ex ## 已知限制与暂缓事项 -- **HTML→markdown 转换在病态输入上回退为原始 HTML**：[turndown](https://github.com/mixmark-io/turndown)（带 GFM 表格／删除线）通过真实 DOM 转换抓取到的 HTML，但其递归遍历在极深嵌套（数千层）上会栈溢出；此类主体不经转换原样通过，而非报错（[决策记录](../../../.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md)）。 +- **HTML→markdown 转换在病态输入上回退为原始 HTML**：[turndown](https://github.com/mixmark-io/turndown)（带 GFM 表格／删除线）通过真实 DOM 转换抓取到的 HTML，但同步遍历在深层未闭合嵌套上呈超线性，因此嵌套超过固定 512 层预检上限的主体不经转换原样通过（仍让 turndown 抛异常的输入同样如此），而非阻塞事件循环或报错（[决策记录](../../../.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md)）。 - **面向模型的表层有意保持最小，提升项暂缓**：`max_results` 保持为配置上限（不是模型参数），`web_fetch` 只接受 `url`（没有 `format`／`prompt`／LLM 摘要模式）；两项都列为 [seam Agent Note](../../../.agents/notes/implemented/architecture/2026-06-24-web-capability-seam.md) 中的后续步骤。 - **没有 web 专用权限策略**：两个工具都不会请求 `ctx.approval` 就直接执行；需要确认的部署必须添加 `tools/pre-execute` 策略，该包不定义持久 URL／domain 授权。 diff --git a/packages/web/tool-web/src/fetch.ts b/packages/web/tool-web/src/fetch.ts index 60c0f33507..e909ea25be 100644 --- a/packages/web/tool-web/src/fetch.ts +++ b/packages/web/tool-web/src/fetch.ts @@ -44,23 +44,69 @@ export function parseFetchArgs(args: { url: string }): { url: string } { return { url: args.url } } +/** + * Nesting-depth ceiling above which HTML skips conversion and passes through + * raw. Conversion runs synchronously on the event loop, and unclosed-tag + * nesting makes domino's tree (and turndown's walk over it) superlinear — + * measured: depth 512 ≈ 0.15s, 2,000 ≈ 2s, 20,000 ≈ 5s — during which the + * cooperative `fetchTimeoutMs` timer cannot fire. Real pages nest a few dozen + * levels; 512 is far above content and far below weaponizable. A robustness + * invariant, not a tunable. + */ +const MAX_CONVERSION_DEPTH = 512 + +/** Elements that never take a closing tag, so they must not count toward nesting depth. */ +const VOID_ELEMENTS = new Set([ + 'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', + 'link', 'meta', 'param', 'source', 'track', 'wbr', +]) + +/** + * Estimate the maximum element nesting depth of an HTML string with one linear + * tag scan. Overestimates when markup-like text sits inside `script`/`style` + * bodies or comments (the scan does not parse those), which can only cause a + * spurious raw-HTML fallback, never a missed bound. + * + * @param html - the decoded HTML body. + * @returns the deepest open-element count the scan reaches. + */ +export function htmlNestingDepth(html: string): number { + let depth = 0 + let max = 0 + for (const tag of html.matchAll(/<(\/?)([a-zA-Z][a-zA-Z0-9-]*)[^>]*?(\/?)>/g)) { + const [, closing, rawName = '', selfClosing] = tag + const name = rawName.toLowerCase() + if (VOID_ELEMENTS.has(name) || selfClosing === '/') continue + if (closing === '/') { + if (depth > 0) depth -= 1 + } else { + depth += 1 + if (depth > max) max = depth + } + } + return max +} + /** * Render a fetched body to model-facing markdown text. * * @param body - the decoded body; `html` is converted via turndown, `text` - * passes through verbatim. When turndown throws (deeply pathological HTML - * overflows its recursive DOM walk), the raw HTML passes through instead — - * a degraded page beats an error for a body the provider already decoded. + * passes through verbatim. HTML nested beyond {@link MAX_CONVERSION_DEPTH} + * skips conversion up front (the synchronous walk over such trees is + * superlinear and blocks the event loop past the cooperative timeout), and + * when turndown itself throws the raw HTML passes through instead — a + * degraded page beats an error for a body the provider already decoded. * @returns the text for the tool's output block. */ export function renderBody(body: WebFetchBody): string { switch (body.kind) { case 'html': + if (htmlNestingDepth(body.content) > MAX_CONVERSION_DEPTH) return body.content try { return turndown.turndown(body.content) } catch { - // turndown's DOM walk recurses per element; pathological nesting (a - // few thousand levels) throws RangeError. Provider errors stay + // turndown's DOM walk recurses per element; malformed markup the depth + // scan cannot see can still throw RangeError. Provider errors stay // structured WebErrors upstream; conversion failure downgrades to raw HTML. return body.content } @@ -72,17 +118,28 @@ export function renderBody(body: WebFetchBody): string { } } +/** The truncation notice appended when the provider or the output cap cut content. */ +const TRUNCATION_FOOTER = '\n\n(Content truncated. Fetch a more specific URL or section for the full text.)' + /** - * Format a fetch result as one model-facing text block. + * Format a fetch result as one model-facing text block, bounded as a whole. + * Markdown escaping can expand converted HTML (worst case ~2× the provider's + * body cap), so the bound applies here, where the complete output — header, + * rendered body, and footer — is known. * * @param result - the seam's fetch outcome. + * @param maxOutputChars - cap on the complete returned string; a cut body gets + * the same fetch-something-narrower notice as provider-side truncation. * @returns a `Fetched (HTTP )` header, the rendered body, and a - * fetch-something-narrower notice when the provider truncated the content. + * truncation notice when the provider or the cap cut the content. */ -export function formatFetchOutput(result: WebFetchResult): string { - const header = `Fetched ${result.url} (HTTP ${result.statusCode})` - const footer = result.truncated ? '\n\n(Content truncated. Fetch a more specific URL or section for the full text.)' : '' - return `${header}\n\n${renderBody(result.body)}${footer}` +export function formatFetchOutput(result: WebFetchResult, maxOutputChars: number): string { + const header = `Fetched ${result.url} (HTTP ${result.statusCode})\n\n` + const body = renderBody(result.body) + const full = `${header}${body}${result.truncated ? TRUNCATION_FOOTER : ''}` + if (full.length <= maxOutputChars) return full + const budget = Math.max(0, maxOutputChars - header.length - TRUNCATION_FOOTER.length) + return `${header}${body.slice(0, budget)}${TRUNCATION_FOOTER}` } /** @@ -102,8 +159,11 @@ export function presentFetchCall(args: { url: string }): GenericCallView { * registrations; both are effect-scoped and unregister on plugin dispose. * @param timeoutMs - the cooperative tool-call budget (ms) attached as the tool's * `ToolDefinition.timeoutMs` for `@deepseek-ai/dsh-timeout-policy` to enforce. + * @param maxOutputChars - cap on the complete rendered tool output (see + * {@link formatFetchOutput}); markdown escaping can outgrow the provider's + * body cap, so the model-context bound is enforced on the rendered result. */ -export function applyWebFetchTool(ctx: Context, timeoutMs: number): void { +export function applyWebFetchTool(ctx: Context, timeoutMs: number, maxOutputChars: number): void { ctx.systemPrompt.section({ name: 'tool:web_fetch', order: 111, @@ -147,7 +207,7 @@ export function applyWebFetchTool(ctx: Context, timeoutMs: number): void { truncated: { type: 'boolean', required: true }, }, }, - render: (_args, value) => [{ type: 'text', text: formatFetchOutput(value) }], + render: (_args, value) => [{ type: 'text', text: formatFetchOutput(value, maxOutputChars) }], }, timeoutMs, // Provider reads do not mutate parent-agent state. diff --git a/packages/web/tool-web/src/index.ts b/packages/web/tool-web/src/index.ts index e7ac4b2453..4a0ea5202c 100644 --- a/packages/web/tool-web/src/index.ts +++ b/packages/web/tool-web/src/index.ts @@ -13,7 +13,7 @@ import { applyWebSearchTool, WEB_SEARCH_MAX_RESULTS } from './search.ts' import { applyWebFetchTool } from './fetch.ts' export { WEB_SEARCH_MAX_RESULTS, applyWebSearchTool, formatSearchOutput, parseSearchArgs, presentSearchCall } from './search.ts' -export { applyWebFetchTool, formatFetchOutput, parseFetchArgs, presentFetchCall, renderBody } from './fetch.ts' +export { applyWebFetchTool, formatFetchOutput, htmlNestingDepth, parseFetchArgs, presentFetchCall, renderBody } from './fetch.ts' /** Cordis plugin name used by loader diagnostics. */ export const name = 'tool-web' @@ -24,7 +24,16 @@ export const inject = ['tools', 'web', 'systemPrompt'] /** Default cooperative tool-call timeout budget (ms) for the web tools. */ export const DEFAULT_WEB_TOOL_TIMEOUT_MS = 30_000 -/** Plugin config: which web tools to register, the source cap, and per-tool budgets. */ +/** + * Default cap on one `web_fetch` output's characters. Markdown escaping can + * roughly double converted HTML, so this sits at 2× the local provider's + * default 100,000-char body cap: it never cuts what that composition's + * provider bound already admits, while restoring a model-context bound for + * providers with larger or absent body caps. + */ +export const DEFAULT_FETCH_MAX_OUTPUT_CHARS = 200_000 + +/** Plugin config: which web tools to register, the source cap, per-tool budgets, and the fetch output cap. */ export interface Config { /** Register `web_search`. Defaults to true. */ search?: boolean @@ -36,6 +45,8 @@ export interface Config { fetchTimeoutMs?: number /** Cooperative timeout budget (ms) for `web_search`. Defaults to 30000. */ searchTimeoutMs?: number + /** Cap on one `web_fetch` output's characters (header, rendered body, and footer). Defaults to 200000. */ + fetchMaxOutputChars?: number } export const Config: z = z.object({ @@ -44,6 +55,7 @@ export const Config: z = z.object({ searchMaxResults: z.number().default(WEB_SEARCH_MAX_RESULTS), fetchTimeoutMs: z.number().default(DEFAULT_WEB_TOOL_TIMEOUT_MS), searchTimeoutMs: z.number().default(DEFAULT_WEB_TOOL_TIMEOUT_MS), + fetchMaxOutputChars: z.number().default(DEFAULT_FETCH_MAX_OUTPUT_CHARS), }) /** The shape after schemastery applies its defaults to every field. */ @@ -71,6 +83,7 @@ export function apply(ctx: Context, config: Config): void { assertPositiveInteger('searchMaxResults', resolved.searchMaxResults) assertPositiveInteger('fetchTimeoutMs', resolved.fetchTimeoutMs) assertPositiveInteger('searchTimeoutMs', resolved.searchTimeoutMs) + assertPositiveInteger('fetchMaxOutputChars', resolved.fetchMaxOutputChars) if (resolved.search) applyWebSearchTool(ctx, resolved.searchMaxResults, resolved.searchTimeoutMs) - if (resolved.fetch) applyWebFetchTool(ctx, resolved.fetchTimeoutMs) + if (resolved.fetch) applyWebFetchTool(ctx, resolved.fetchTimeoutMs, resolved.fetchMaxOutputChars) } diff --git a/packages/web/tool-web/tests/tool-web.spec.ts b/packages/web/tool-web/tests/tool-web.spec.ts index f9ffb1b5c5..79af9cbd2a 100644 --- a/packages/web/tool-web/tests/tool-web.spec.ts +++ b/packages/web/tool-web/tests/tool-web.spec.ts @@ -1,5 +1,6 @@ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { Context } from 'cordis' +import TurndownService from 'turndown' import { CallId } from '@deepseek-ai/dsh-llm' import SystemPrompt from '@deepseek-ai/dsh-system-prompt' import ToolRegistry, { type ToolExecutionResult } from '@deepseek-ai/dsh-tools' @@ -9,6 +10,7 @@ import * as ToolWeb from '@deepseek-ai/dsh-tool-web' import { formatSearchOutput, formatFetchOutput, + htmlNestingDepth, parseSearchArgs, parseFetchArgs, presentSearchCall, @@ -92,11 +94,13 @@ describe('search formatting', () => { }) describe('fetch formatting', () => { + const NO_CAP = 1_000_000 + it('renders an html body to markdown text with a status header', () => { const out = formatFetchOutput({ url: 'https://a.test', statusCode: 200, truncated: false, body: { kind: 'html', content: '

    Title

    Body text

    ' }, - }) + }, NO_CAP) expect(out).toContain('Fetched https://a.test (HTTP 200)') expect(out).toContain('# Title') expect(out).toContain('Body text') @@ -106,11 +110,37 @@ describe('fetch formatting', () => { const out = formatFetchOutput({ url: 'https://a.test', statusCode: 200, truncated: true, body: { kind: 'text', content: 'plain' }, - }) + }, NO_CAP) expect(out).toContain('plain') expect(out).toContain('Content truncated') }) + it('caps the complete output and notes truncation, even when markdown escaping expands the body', () => { + // 1,000 underscores render as 2,000 escaped characters — conversion can + // outgrow a provider-side body cap, so the bound applies to the output. + const out = formatFetchOutput({ + url: 'https://a.test', statusCode: 200, truncated: false, + body: { kind: 'html', content: `

    ${'_'.repeat(1000)}

    ` }, + }, 500) + expect(out.length).toBeLessThanOrEqual(500) + expect(out).toContain('Fetched https://a.test (HTTP 200)') + expect(out).toContain('\\_\\_') + expect(out).toContain('Content truncated') + // Exact and tiny caps: the complete result is bounded, header and footer included. + const exact = formatFetchOutput({ + url: 'https://a.test', statusCode: 200, truncated: false, + body: { kind: 'text', content: 'abc' }, + }, 'Fetched https://a.test (HTTP 200)\n\nabc'.length) + expect(exact).toBe('Fetched https://a.test (HTTP 200)\n\nabc') + const tiny = formatFetchOutput({ + url: 'https://a.test', statusCode: 200, truncated: true, + body: { kind: 'text', content: 'abcdef' }, + }, 10) + expect(tiny).toContain('Fetched https://a.test (HTTP 200)') + expect(tiny).toContain('Content truncated') + expect(tiny).not.toContain('abcdef') + }) + it('renderBody dispatches on kind', () => { expect(renderBody({ kind: 'text', content: 'x' })).toBe('x') expect(renderBody({ kind: 'html', content: '

    y

    ' })).toBe('y') @@ -129,14 +159,38 @@ describe('fetch formatting', () => { .toBe('**bold _italic_**\n\n> quoted') }) - it('falls back to the raw html body when turndown throws on pathological nesting', { timeout: 60_000 }, () => { - // Nesting past V8's default stack overflows turndown/domino's recursive - // walk with a RangeError (measured: 4k levels throw on the main thread, - // 8k in a worker); 20k adds margin over either stack size. The raw body - // must pass through instead of throwing. + it('passes deeply nested html through raw without attempting conversion', () => { + // Unclosed-tag nesting makes the synchronous conversion superlinear + // (seconds at 20k levels, during which the cooperative timeout cannot + // fire), so the depth preflight skips conversion entirely; this must + // return fast, not merely not-throw. const depth = 20_000 const pathological = '
    '.repeat(depth) + 'x' + '
    '.repeat(depth) + const started = Date.now() expect(renderBody({ kind: 'html', content: pathological })).toBe(pathological) + expect(Date.now() - started).toBeLessThan(2_000) + }) + + it('htmlNestingDepth counts open elements, ignoring void and self-closing tags', () => { + expect(htmlNestingDepth('

    x

    ')).toBe(2) + expect(htmlNestingDepth('

    ')).toBe(1) + expect(htmlNestingDepth('

    x

    ')).toBe(1) + expect(htmlNestingDepth('plain text, no tags')).toBe(0) + expect(htmlNestingDepth('
    '.repeat(600))).toBe(600) + }) + + it('falls back to the raw html when turndown throws despite a shallow depth scan', () => { + // Comments hide markup from the depth scan by design (it may only + // over-count, never under-count real elements); simulate the residual + // turndown failure path with a converter throw instead. + const spy = vi.spyOn(TurndownService.prototype, 'turndown').mockImplementation(() => { + throw new RangeError('Maximum call stack size exceeded') + }) + try { + expect(renderBody({ kind: 'html', content: '

    x

    ' })).toBe('

    x

    ') + } finally { + spy.mockRestore() + } }) it('validates url (non-empty), no timeout parameter', () => { From cff614d37df01efe249bcc4d4bb94d3eb410443a Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Mon, 27 Jul 2026 13:17:12 +0800 Subject: [PATCH 11/14] ci: run the pull-request Windows blocking gates under Wine on hosted Linux The required windows job moves from windows-2025 to ubuntu-latest, running checksum-verified Windows Node under Wine at Linux-job wall clock (2m46s warm vs 7-9min); master's serial-windows native-kernel reference is untouched, and a new master-only wine-apt-cache job seeds the apt cache every pull request restores. The experiment workflow folds into ci.yml, the Agent Note moves to implemented with measured results, and the two CI topology notes update to the shipped facts. --- ...rial-cross-platform-ci-reference.i18n.yaml | 6 +- ...7-21-serial-cross-platform-ci-reference.md | 2 +- ...1-serial-cross-platform-ci-reference.zh.md | 2 +- ...ortable-required-pull-request-ci.i18n.yaml | 6 +- ...07-23-portable-required-pull-request-ci.md | 6 +- ...23-portable-required-pull-request-ci.zh.md | 6 +- ...27-wine-windows-gates-experiment.i18n.yaml | 6 + ...026-07-27-wine-windows-gates-experiment.md | 45 ++++ ...-07-27-wine-windows-gates-experiment.zh.md | 45 ++++ ...27-wine-windows-gates-experiment.i18n.yaml | 6 - ...026-07-27-wine-windows-gates-experiment.md | 51 ---- ...-07-27-wine-windows-gates-experiment.zh.md | 51 ---- .github/AGENTS.md | 2 +- .github/workflows/ci.yml | 227 +++++++++++++++-- .github/workflows/exp-wine-windows.yml | 230 ------------------ 15 files changed, 316 insertions(+), 375 deletions(-) create mode 100644 .agents/notes/implemented/process/2026-07-27-wine-windows-gates-experiment.i18n.yaml create mode 100644 .agents/notes/implemented/process/2026-07-27-wine-windows-gates-experiment.md create mode 100644 .agents/notes/implemented/process/2026-07-27-wine-windows-gates-experiment.zh.md delete mode 100644 .agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.i18n.yaml delete mode 100644 .agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.md delete mode 100644 .agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.zh.md delete mode 100644 .github/workflows/exp-wine-windows.yml diff --git a/.agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.i18n.yaml b/.agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.i18n.yaml index 17edb300cc..553e656805 100644 --- a/.agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.i18n.yaml +++ b/.agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.i18n.yaml @@ -1,6 +1,6 @@ # Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: -# pnpm run verify-translation-pairing --write -2026-07-21-serial-cross-platform-ci-reference.md: 5433d2c51831ce61d06a16ee0b0ed982911f9218 -2026-07-21-serial-cross-platform-ci-reference.zh.md: 041d53d13e14354c995e4b65defce94a97646b0a +# pnpm run verify-translation-pairing --write .agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.md +2026-07-21-serial-cross-platform-ci-reference.md: 5eac1bc1c47c7309942b5615bc98a7fed893f346 +2026-07-21-serial-cross-platform-ci-reference.zh.md: 35fb761023fe7be081bf7d9591a53ed98b6e3abc diff --git a/.agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.md b/.agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.md index 5433d2c518..5eac1bc1c4 100644 --- a/.agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.md +++ b/.agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.md @@ -24,7 +24,7 @@ The macOS reference runs the ordinary Vitest project in forked processes. Node 2 Master reference jobs are diagnostic and do not participate in the pull request's required `all checks passed` result. A pull request runs only its required jobs; a master push runs only the three serial references. Performance is evaluated from completed hosted-job timestamps and reported as a measurement; it is not encoded as a `timeout-minutes` value. -The portable reference uses GitHub's standard `ubuntu-latest`, `macos-latest`, and `windows-2025` labels. Required pull-request jobs use the same portable Linux and Windows capacity under the [required-CI decision](2026-07-23-portable-required-pull-request-ci.md). Higher-core hosted runners remain manual benchmarks because a correctness path must remain runnable without repository-external runner configuration. +The portable reference uses GitHub's standard `ubuntu-latest`, `macos-latest`, and `windows-2025` labels; `serial / windows` is the one remaining native-Windows job, the complete-kernel oracle behind the Wine-hosted pull-request lane ([Wine lane decision](2026-07-27-wine-windows-gates-experiment.md)). Required pull-request jobs use portable standard capacity under the [required-CI decision](2026-07-23-portable-required-pull-request-ci.md). Higher-core hosted runners remain manual benchmarks because a correctness path must remain runnable without repository-external runner configuration. ## Alternatives considered diff --git a/.agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.zh.md b/.agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.zh.md index 041d53d13e..35fb761023 100644 --- a/.agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.zh.md +++ b/.agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.zh.md @@ -24,7 +24,7 @@ macOS 参考流程使用 fork 进程运行常规 Vitest 项目。macOS arm64 上 master 分支的参考作业仅用于诊断，不参与拉取请求所要求的 `all checks passed` 结果。拉取请求只运行其必需作业；向 master 推送时只运行三个串行参考作业。系统根据已完成托管作业的时间戳评估性能，并将其报告为测量结果，而不是写成 `timeout-minutes` 值。 -可移植的参考流程使用 GitHub 标准的 `ubuntu-latest`、`macos-latest` 和 `windows-2025` 标签。依据[必需 CI 决策](2026-07-23-portable-required-pull-request-ci.md)，拉取请求必需作业使用相同的可移植 Linux 和 Windows 容量。更高核心数的托管运行器仍仅用于手动基准测试，因为正确性路径必须无需仓库外部的运行器配置即可运行。 +可移植的参考流程使用 GitHub 标准的 `ubuntu-latest`、`macos-latest` 和 `windows-2025` 标签；`serial / windows` 是仅存的原生 Windows 作业，是 Wine 托管拉取请求通道背后的完整内核标尺（[Wine 通道决策](2026-07-27-wine-windows-gates-experiment.md)）。依据[必需 CI 决策](2026-07-23-portable-required-pull-request-ci.md)，拉取请求必需作业使用可移植的标准容量。更高核心数的托管运行器仍仅用于手动基准测试，因为正确性路径必须无需仓库外部的运行器配置即可运行。 ## 曾考虑的替代方案 diff --git a/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.i18n.yaml b/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.i18n.yaml index 05147cd54a..66131cfe0c 100644 --- a/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.i18n.yaml +++ b/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.i18n.yaml @@ -1,6 +1,6 @@ # Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: -# pnpm run verify-translation-pairing --write -2026-07-23-portable-required-pull-request-ci.md: d1002c7d9db7cd8bbed3bdfda8a773a4b124bf16 -2026-07-23-portable-required-pull-request-ci.zh.md: fedfc6b9c982ace5ece430c52db23c22ec5119d4 +# pnpm run verify-translation-pairing --write .agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.md +2026-07-23-portable-required-pull-request-ci.md: 1a6939e8386e381cba114a7be71993a644457a45 +2026-07-23-portable-required-pull-request-ci.zh.md: cf0af769f9e740a2c9285caf4be05023371578d9 diff --git a/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.md b/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.md index d1002c7d9d..1a6939e838 100644 --- a/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.md +++ b/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.md @@ -12,9 +12,9 @@ Billing health, a runner definition's `Ready` state, and a large autoscaling cei ## Decision -[CI](../../../../.github/workflows/ci.yml) runs the required primary Node 24 jobs, plus the stable `all checks passed` aggregate, on repo-restricted enterprise 32-core pools. The aggregate performs no checkout or repository gate, but sharing the enterprise pool prevents the required verdict from introducing a separate standard-hosted billing dependency after its substantive jobs have already succeeded. The required Windows job runs on standard `windows-2025` with single-worker bounds, keeping the complete Windows contract independent of enterprise Windows allocation. Standard `ubuntu-latest` jobs retain Node 22.19, Node 26, and Python SDK compatibility, and `master` runs complete serial Linux, macOS, and Windows references. Those standard-hosted jobs keep the portable execution boundary observable without duplicating the primary inventory on every pull request. +[CI](../../../../.github/workflows/ci.yml) runs the required primary Node 24 jobs, plus the stable `all checks passed` aggregate, on repo-restricted enterprise 32-core pools. The aggregate performs no checkout or repository gate, but sharing the enterprise pool prevents the required verdict from introducing a separate standard-hosted billing dependency after its substantive jobs have already succeeded. The required Windows job runs Windows Node under Wine on standard `ubuntu-latest` for the blocking surfaces ([Wine lane decision](2026-07-27-wine-windows-gates-experiment.md)), keeping the pull-request Windows contract independent of any Windows runner allocation; the complete native-kernel Windows inventory lives in the master serial reference. Standard `ubuntu-latest` jobs retain Node 22.19, Node 26, and Python SDK compatibility, and `master` runs complete serial Linux, macOS, and Windows references. Those standard-hosted jobs keep the portable execution boundary observable without duplicating the primary inventory on every pull request. -The two Linux primary jobs, Node compatibility, Python SDK, and `windows node 24 / complete` remain dependencies of `all checks passed`; branch protection continues to require `e2e` and `all checks passed`. There is no automatic fallback when a remaining enterprise Linux label cannot allocate: the standard jobs continue to report their own contracts, but they cannot manufacture the missing required result. +The two Linux primary jobs, Node compatibility, Python SDK, and `windows node 24 / wine blocking` remain dependencies of `all checks passed`; branch protection continues to require `e2e` and `all checks passed`. There is no automatic fallback when a remaining enterprise Linux label cannot allocate: the standard jobs continue to report their own contracts, but they cannot manufacture the missing required result. The [larger-runner decision](2026-07-22-evidence-based-larger-hosted-runners.md) owns the current primary topology and its measurements. The [serial cross-platform reference](2026-07-21-serial-cross-platform-ci-reference.md) remains the independent standard-hosted completeness check, and the manual larger-runner suites retain size comparisons without expanding the ordinary required matrix. @@ -30,6 +30,6 @@ The [larger-runner decision](2026-07-22-evidence-based-larger-hosted-runners.md) ## Consequences -Ordinary pull requests spend enterprise capacity on the Linux critical path while standard Windows trades longer runtime for independent allocation. A live exact-head run proves the same commands that branch protection consumes; queue delay is reported separately from each job's `startedAt` to `completedAt` execution interval. +Ordinary pull requests spend enterprise capacity on the Linux critical path while the Wine-hosted Windows job keeps its verdict on standard Linux allocation. A live exact-head run proves the same commands that branch protection consumes; queue delay is reported separately from each job's `startedAt` to `completedAt` execution interval. Standard compatibility and required Windows jobs remain useful when enterprise allocation is degraded, but they do not make a blocked required Linux job or aggregate green. Recovering Linux availability may require restoring the complete standard-hosted topology; changing a pool definition's status alone is insufficient evidence that it can receive work. diff --git a/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.zh.md b/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.zh.md index fedfc6b9c9..cf0af769f9 100644 --- a/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.zh.md +++ b/.agents/notes/implemented/process/2026-07-23-portable-required-pull-request-ci.zh.md @@ -12,9 +12,9 @@ Status: implemented ## 决策 -[CI](../../../../.github/workflows/ci.yml) 在仅限本仓库使用的企业级 32 核运行器池上运行必需的主 Node 24 作业，以及稳定的 `all checks passed` 聚合流程。该聚合流程不执行代码检出或仓库门禁；但让它与所依赖的实质性作业共用企业级运行器池，可以避免这些作业已经成功后，必需判定结果又引入一项单独的标准托管计费依赖。必需的 Windows 作业在标准 `windows-2025` 上运行，并采用单工作线程上限，使完整的 Windows 契约不依赖企业级 Windows 运行器分配。标准 `ubuntu-latest` 作业保留 Node 22.19、Node 26 和 Python SDK 兼容性，`master` 则运行完整的 Linux、macOS 和 Windows 串行参考流程。这些标准托管作业让可移植执行边界保持可观测，而不必在每个拉取请求中重复主清单。 +[CI](../../../../.github/workflows/ci.yml) 在仅限本仓库使用的企业级 32 核运行器池上运行必需的主 Node 24 作业，以及稳定的 `all checks passed` 聚合流程。该聚合流程不执行代码检出或仓库门禁；但让它与所依赖的实质性作业共用企业级运行器池，可以避免这些作业已经成功后，必需判定结果又引入一项单独的标准托管计费依赖。必需的 Windows 作业在标准 `ubuntu-latest` 上通过 Wine 运行 Windows Node 以覆盖阻断表面（[Wine 通道决策](2026-07-27-wine-windows-gates-experiment.md)），使拉取请求的 Windows 契约不依赖任何 Windows 运行器分配；完整的原生内核 Windows 清单归 master 串行参考流程所有。标准 `ubuntu-latest` 作业保留 Node 22.19、Node 26 和 Python SDK 兼容性，`master` 则运行完整的 Linux、macOS 和 Windows 串行参考流程。这些标准托管作业让可移植执行边界保持可观测，而不必在每个拉取请求中重复主清单。 -两项 Linux 主作业、Node 兼容性、Python SDK 和 `windows node 24 / complete` 继续作为 `all checks passed` 的依赖项；分支保护继续要求 `e2e` 和 `all checks passed`。剩余的企业级 Linux 运行器标签无法分配运行器时没有自动后备机制：标准作业会继续报告各自的契约，但无法产出缺失的必需结果。 +两项 Linux 主作业、Node 兼容性、Python SDK 和 `windows node 24 / wine blocking` 继续作为 `all checks passed` 的依赖项；分支保护继续要求 `e2e` 和 `all checks passed`。剩余的企业级 Linux 运行器标签无法分配运行器时没有自动后备机制：标准作业会继续报告各自的契约，但无法产出缺失的必需结果。 当前主拓扑及其测量结果由[大型运行器决策](2026-07-22-evidence-based-larger-hosted-runners.md)记录。[跨平台串行参考流程](2026-07-21-serial-cross-platform-ci-reference.md)继续作为独立的标准托管完整性检查，手动大型运行器套件则保留规格比较，同时不扩大普通必需矩阵。 @@ -30,6 +30,6 @@ Status: implemented ## 后果 -普通拉取请求会将企业级运行器容量用于 Linux 关键路径，而标准托管 Windows 作业则以更长的运行时间换取不依赖企业池的运行器分配。一次实际的分支头精确运行能够证明分支保护使用的同一组命令；排队延迟与每个作业从 `startedAt` 到 `completedAt` 的执行区间分开报告。 +普通拉取请求会将企业级运行器容量用于 Linux 关键路径，而 Wine 托管的 Windows 作业让其判定保持在标准 Linux 运行器分配上。一次实际的分支头精确运行能够证明分支保护使用的同一组命令；排队延迟与每个作业从 `startedAt` 到 `completedAt` 的执行区间分开报告。 企业级运行器分配能力下降时，标准兼容性作业和必需的 Windows 作业仍能提供有用证据，但无法让受阻的必需 Linux 作业或聚合流程变绿。恢复 Linux 可用性时，可能需要恢复完整的标准托管拓扑；仅改变运行器池定义的状态，不足以证明它可以接收作业。 diff --git a/.agents/notes/implemented/process/2026-07-27-wine-windows-gates-experiment.i18n.yaml b/.agents/notes/implemented/process/2026-07-27-wine-windows-gates-experiment.i18n.yaml new file mode 100644 index 0000000000..8b8b736a99 --- /dev/null +++ b/.agents/notes/implemented/process/2026-07-27-wine-windows-gates-experiment.i18n.yaml @@ -0,0 +1,6 @@ +# Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each +# side as of the last confirmed-consistent state. Both languages carry equal authority; +# after editing either side, bring the other along and re-record with: +# pnpm run verify-translation-pairing --write .agents/notes/implemented/process/2026-07-27-wine-windows-gates-experiment.md +2026-07-27-wine-windows-gates-experiment.md: aab8aecdfca06c1f15641044a071015f543a84b6 +2026-07-27-wine-windows-gates-experiment.zh.md: 5239b185e1e0c63aa626ee3f20f3f298c0c8579d diff --git a/.agents/notes/implemented/process/2026-07-27-wine-windows-gates-experiment.md b/.agents/notes/implemented/process/2026-07-27-wine-windows-gates-experiment.md new file mode 100644 index 0000000000..aab8aecdfc --- /dev/null +++ b/.agents/notes/implemented/process/2026-07-27-wine-windows-gates-experiment.md @@ -0,0 +1,45 @@ +# Agent Note: Wine-run Windows blocking gates on Linux runners + +Status: implemented + +English | [中文](2026-07-27-wine-windows-gates-experiment.zh.md) + +## Problem + +The pull-request Windows lane exists to prove the two blocking win32 surfaces — the workspace build and the production site — and it ran on hosted `windows-2025`, the slowest job in the required matrix: 7–9 minutes against 1.5–2.5 for the Linux jobs, so the Windows VM's boot, setup, and filesystem costs dominated every pull request's critical path. + +The question the experiment answered: can a plain Linux runner produce an equivalent win32 signal for the blocking surfaces at Linux wall clock, so no Windows VM sits on the pull-request path at all? + +## Decision + +The required pull-request `windows` job in [ci.yml](../../../../.github/workflows/ci.yml) (`windows node 24 / wine blocking`) runs the blocking gate commands on `ubuntu-latest` under Wine with real Windows binaries: a checksum-verified win-x64 Node.js executes `tsc -b`, `tsdown`, and the VitePress production build, so the win32 branches of the toolchain — backslash path handling, `CreateProcess` spawn semantics, PE loading of `@esbuild/win32-x64`, and the rolldown/rollup MSVC `.node` addons — actually execute. The master `serial-windows` job is untouched: the complete native-kernel inventory, including the observational portability gates this lane does not run, still executes on real `windows-2025` on every master push. + +Dependencies install natively on Linux with `supportedArchitectures` extended to win32-x64, which materializes the Windows platform packages in the same store; the cmd-shim layer is bypassed by invoking each tool's JavaScript entrypoint directly, the same processes `run-gates` ultimately spawns. `nodeLinker: hoisted` is load-bearing, not stylistic: the independent prototype in [PR #689](https://github.com/deepseek-harness/deepseek-harness/pull/689) kept pnpm's default isolated layout — including a faithful offline Windows-pnpm re-install over a Linux-prefetched store — and Windows Node under Wine still could not resolve `@esbuild/win32-x64` or load the koffi prebuild through the isolated symlink chain, failing before any repository gate ran. A flat layout with real files is what makes the gates reachable at all; #689's checksum pinning is adopted, while its Windows-pnpm-installs-the-tree goal is explicitly given up (the install contract stays Linux-tested here). + +The lane holds the wall clock of the Linux CI jobs through four levers: the master-refreshed pnpm store cache (restore-only, same key as the Linux jobs), Wine provisioning (apt install, Windows Node download, `wineboot`) running concurrently with `pnpm install`, the two blocking surfaces running concurrently — the same shape `run-gates` gives them on native Windows — and an apt-archive cache keyed on the runner image, seeded from master by the `wine apt cache` job so every pull request restores from the default-branch scope. + +Four environment constraints shape the job, each found as a red run: Ubuntu's `wine64` package alone puts nothing on PATH (install `wine`, the dispatcher); Node under Wine cannot attach stdio to the Actions runner's pipes (`Socket open EBADF` at bootstrap — every invocation routes stdio through a file); Wine does not realpath pnpm's isolated-layout Unix symlinks (the hoisted layout above); and Wine cannot create Windows symlinks (`ENOTSUP` from VitePress's `linkVue` — the `vue` link is laid down host-side before the gate). + +## Measured results + +Measured on 2026-07-27, warm caches, pull-request trigger, standard 2-core `ubuntu-latest`: 2m46s end-to-end — setup and cache restores ~17s, concurrent install+provision 33s, concurrent gates 110s — against 1.5–2.5 minutes for the Linux CI jobs and 7–9 minutes for the replaced `windows-2025` job. A cold-cache run pays roughly one extra minute. An 8-core benchmark leg was defined during the experiment but never left the restricted `dsh-ubuntu-*` pool's queue; the standard-runner number met the target, so no larger box is used. + +## Alternatives considered + +**Keep the hosted `windows-2025` pull-request job (status quo).** Nothing wrong with its signal, only its latency: 7–9 minutes for two build commands, the slowest required job in the matrix. It survives as the master serial reference, where completeness matters more than latency. + +**A full Windows guest under QEMU/KVM inside the Linux runner.** Real NT kernel, so full fidelity including case-insensitive NTFS and ConPTY — but tens of minutes of image download and unattended install before the first gate runs (40m19s measured end-to-end on the sibling experiment branch `exp/kvm-windows-ci`). Promotable only with disk-image caching that pressures the Actions cache budget. + +**Windows pnpm performing the install under Wine ([PR #689](https://github.com/deepseek-harness/deepseek-harness/pull/689)).** The higher-fidelity variant of this same idea: MinGit and pnpm staged into the prefix, a Linux prefetch filling the store, then `pnpm install --offline` run by Windows Node so the install contract itself executes as win32. It reached the install but not the gates — Wine's networking could not reach the registry directly, and the isolated `node_modules` layout defeated resolution of the Windows platform packages even after a clean offline install. This lane trades that fidelity away (hoisted layout, Linux-side install) to reach the gates; the two records are complementary halves of the same verdict. + +**Filesystem-semantics lanes on Linux (casefolded ext4, filename lint).** Catches the highest-frequency Windows breakage class for near-zero cost but proves nothing about win32 binaries. Explored as the sibling experiment branch `exp/casefold-windows-ci`; complementary to, not competitive with, this lane. + +**Windows containers.** Not possible: Windows containers require a Windows host kernel; a hosted Linux runner cannot run them. + +**Dropping the Windows lane.** Rejected — win32 is a first-class product target: the koffi-backed DACL and durable-namespace modules, ConPTY-based PTY sessions, and Windows path policy all ship in `packages/`. + +## Consequences + +Every pull request's Windows verdict now arrives in Linux-job time on free standard capacity, and no Windows VM allocation sits on the pull-request critical path; `all checks passed` consumes the same `windows` job id it always did. + +What the trade costs: Wine reimplements Win32 over a case-sensitive ext4 — NTFS case-insensitivity, real DACLs, ConPTY, and crash-durability semantics are not proved here, and the observational portability inventory (duplication, publint, node-next types, built-package invariants on win32) no longer runs on pull requests at all. The master `serial-windows` reference owns all of that: a Wine-green pull request can still fail the native-kernel master run, and that failure mode is accepted as post-merge. The lane also inherits Wine-specific divergences as permanent job structure — file-routed stdio, the host-side `vue` link, the hoisted layout — so a future toolchain change that depends on isolated-layout semantics or in-process symlink creation will surface here first as a Wine failure rather than a product failure, and triage must classify it as such. If Wine reds ever recur without product cause, the recorded fallback is reverting the `windows` job to the pre-Wine `windows-2025` definition preserved in git history. diff --git a/.agents/notes/implemented/process/2026-07-27-wine-windows-gates-experiment.zh.md b/.agents/notes/implemented/process/2026-07-27-wine-windows-gates-experiment.zh.md new file mode 100644 index 0000000000..5239b185e1 --- /dev/null +++ b/.agents/notes/implemented/process/2026-07-27-wine-windows-gates-experiment.zh.md @@ -0,0 +1,45 @@ +# Agent Note: 在 Linux runner 上用 Wine 运行 Windows 阻断门禁 + +Status: implemented + +[English](2026-07-27-wine-windows-gates-experiment.md) | 中文 + +## 问题 + +Pull request 的 Windows 通道存在的意义是证明两个阻断性 win32 表面——workspace 构建与生产站点——它此前运行在托管 `windows-2025` 上，是必需矩阵中最慢的作业：7–9 分钟，对照 Linux 作业的 1.5–2.5 分钟，因此 Windows VM 的启动、准备与文件系统开销主导了每个 pull request 的关键路径。 + +实验回答的问题是：一台普通 Linux runner 能否以 Linux 墙钟为阻断表面产出等效的 win32 信号，让 pull request 路径上完全没有 Windows VM？ + +## 决策 + +[ci.yml](../../../../.github/workflows/ci.yml) 中必需的 pull request `windows` 作业（`windows node 24 / wine blocking`）在 `ubuntu-latest` 上通过 Wine 用真实 Windows 二进制运行阻断门禁命令：校验和验证过的 win-x64 Node.js 执行 `tsc -b`、`tsdown` 与 VitePress 生产构建，因此工具链的 win32 分支——反斜杠路径处理、`CreateProcess` 派生语义、`@esbuild/win32-x64` 的 PE 加载、以及 rolldown/rollup 的 MSVC `.node` 插件——都真正执行。master 的 `serial-windows` 作业原封不动：完整的原生内核清单，包括本通道不运行的观察性可移植性门禁，仍在每次 master push 时于真实 `windows-2025` 上执行。 + +依赖在 Linux 上原生安装，`supportedArchitectures` 扩展到 win32-x64，使 Windows 平台包物化进同一个 store；通过直接调用各工具的 JavaScript 入口绕开 cmd-shim 层，这正是 `run-gates` 最终派生的那些进程。`nodeLinker: hoisted` 是承重的，不是风格问题：[PR #689](https://github.com/deepseek-harness/deepseek-harness/pull/689) 的独立原型保留了 pnpm 默认的 isolated 布局——包括在 Linux 预取的 store 上忠实地用 Windows pnpm 离线重装——而 Wine 下的 Windows Node 依然无法穿过 isolated 符号链接链解析 `@esbuild/win32-x64` 或加载 koffi 预编译产物，在任何仓库门禁运行前就失败了。扁平的真实文件布局才让门禁变得可达；本通道采纳了 #689 的校验和固定，同时明确放弃其"Windows pnpm 安装依赖树"的目标（安装契约在此仍由 Linux 侧验证）。 + +该通道靠四个杠杆保持 Linux CI 作业的墙钟：master 刷新的 pnpm store 缓存（只恢复，与 Linux 作业同键）、Wine 供给（apt 安装、Windows Node 下载、`wineboot`）与 `pnpm install` 并发运行、两个阻断表面并发运行——与 `run-gates` 在原生 Windows 上给它们的形状相同——以及按 runner 镜像为键的 apt 归档缓存，由 master 的 `wine apt cache` 作业播种，使每个 pull request 都能从默认分支作用域恢复。 + +四条环境约束塑造了该作业，每条都以一次红色运行被发现：Ubuntu 的 `wine64` 包本身不往 PATH 放任何东西（要装 `wine` 调度器）；Wine 下的 Node 无法把 stdio 接到 Actions runner 的管道上（引导期 `Socket open EBADF`——所有调用都经文件中转 stdio）；Wine 不对 pnpm isolated 布局的 Unix 符号链接做 realpath（即上文的 hoisted 布局）；Wine 无法创建 Windows 符号链接（VitePress 的 `linkVue` 报 `ENOTSUP`——`vue` 链接在门禁前由宿主侧铺好）。 + +## 实测结果 + +2026-07-27 实测，热缓存，pull request 触发，标准 2 核 `ubuntu-latest`：端到端 2 分 46 秒——准备与缓存恢复约 17 秒，并发安装+供给 33 秒，并发门禁 110 秒——对照 Linux CI 作业的 1.5–2.5 分钟与被替换的 `windows-2025` 作业的 7–9 分钟。冷缓存约多付一分钟。实验期间定义过 8 核基准腿，但它从未离开受限 `dsh-ubuntu-*` 池的队列；标准 runner 的数字已达标，故不使用更大的机器。 + +## 考虑过的替代方案 + +**保留托管 `windows-2025` 的 pull request 作业（现状）。** 其信号没有问题，问题只在延迟：为两条构建命令花 7–9 分钟，是必需矩阵中最慢的作业。它作为 master 串行参照存续——在那里完整性比延迟更重要。 + +**在 Linux runner 内用 QEMU/KVM 跑完整 Windows 客户机。** 真实 NT 内核，保真度完整，包括大小写不敏感的 NTFS 与 ConPTY——但首个门禁运行前要花数十分钟下载镜像并做无人值守安装（兄弟实验分支 `exp/kvm-windows-ci` 实测端到端 40 分 19 秒）。只有配上会挤压 Actions 缓存预算的磁盘镜像缓存才可晋升。 + +**在 Wine 下由 Windows pnpm 执行安装（[PR #689](https://github.com/deepseek-harness/deepseek-harness/pull/689)）。** 同一想法的更高保真度变体：把 MinGit 与 pnpm 放进 prefix，用 Linux 预取填充 store，再由 Windows Node 运行 `pnpm install --offline`，让安装契约本身以 win32 身份执行。它到达了安装但没到达门禁——Wine 的网络无法直接访问 registry，且 isolated 的 `node_modules` 布局即便在干净的离线安装后也挫败了 Windows 平台包的解析。本通道用掉这份保真度（hoisted 布局、Linux 侧安装）来换取门禁可达；两份记录是同一裁决互补的两半。 + +**Linux 上的文件系统语义通道（casefold ext4、文件名 lint）。** 以近零成本捕获最高频的 Windows 破坏类别，但对 win32 二进制什么也证明不了。作为兄弟实验分支 `exp/casefold-windows-ci` 探索；与本通道互补而非竞争。 + +**Windows 容器。** 不可行：Windows 容器要求 Windows 宿主内核；托管 Linux runner 无法运行。 + +**砍掉 Windows 通道。** 已否决——win32 是一等产品目标：基于 koffi 的 DACL 与持久命名空间模块、基于 ConPTY 的 PTY 会话、以及 Windows 路径策略都随 `packages/` 交付。 + +## 结果 + +每个 pull request 的 Windows 裁决现在以 Linux 作业的时间在免费标准容量上到达，pull request 关键路径上不再有任何 Windows VM 分配；`all checks passed` 消费的仍是原来的 `windows` 作业 id。 + +这笔交易的代价：Wine 在大小写敏感的 ext4 之上重实现 Win32——NTFS 大小写不敏感、真实 DACL、ConPTY 与崩溃持久性语义在此都未被证明，且观察性可移植性清单（duplication、publint、node-next 类型、win32 上的构建包不变量）完全不再于 pull request 上运行。master 的 `serial-windows` 参照拥有这一切：Wine 绿灯的 pull request 仍可能在原生内核的 master 运行上失败，该失败模式被接受为合并后处理。该通道还把 Wine 特有的分歧继承为永久的作业结构——文件中转的 stdio、宿主侧的 `vue` 链接、hoisted 布局——因此未来依赖 isolated 布局语义或进程内符号链接创建的工具链变更会先在这里以 Wine 失败而非产品失败的形式浮现，分诊必须如此归类。若 Wine 红灯在无产品原因的情况下反复出现，记录在案的退路是把 `windows` 作业还原为 git 历史中保存的 Wine 之前的 `windows-2025` 定义。 diff --git a/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.i18n.yaml b/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.i18n.yaml deleted file mode 100644 index c39841966d..0000000000 --- a/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.i18n.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each -# side as of the last confirmed-consistent state. Both languages carry equal authority; -# after editing either side, bring the other along and re-record with: -# pnpm run verify-translation-pairing --write .agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.md -2026-07-27-wine-windows-gates-experiment.md: 47a37ddb48f4321f916c7f7a0cb96ae80b133103 -2026-07-27-wine-windows-gates-experiment.zh.md: 3a912861110a06b39bfb2c37395fc6a061bdfbe6 diff --git a/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.md b/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.md deleted file mode 100644 index 47a37ddb48..0000000000 --- a/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.md +++ /dev/null @@ -1,51 +0,0 @@ -# Agent Note: Wine-run Windows blocking gates on Linux runners - -Status: proposed - -English | [中文](2026-07-27-wine-windows-gates-experiment.zh.md) - -## Problem - -The pull-request Windows lane exists to prove the two blocking win32 surfaces — the workspace build and the production site — plus an observational portability inventory, and it runs on a dedicated paid Windows larger-runner pool; the master serial reference adds a second hosted Windows job. That pool is the only reason a Windows VM exists anywhere in this pipeline, and its provisioning, pricing, and slow setup dominate the lane's cost. - -The open question: can a plain Linux runner produce an equivalent win32 signal for the blocking surfaces, so the dedicated Windows pool can shrink to a master-only reference or disappear from the pull-request path entirely? - -## Proposal - -[exp-wine-windows.yml](../../../../.github/workflows/exp-wine-windows.yml) (self-path-filtered, plus manual dispatch) runs the blocking gate commands on `ubuntu-latest` under Wine with real Windows binaries: a checksum-verified win-x64 Node.js executes `tsc -b`, `tsdown`, and the VitePress production build, so the win32 branches of the toolchain — backslash path handling, `CreateProcess` spawn semantics, PE loading of `@esbuild/win32-x64`, and the rolldown/rollup MSVC `.node` addons — actually execute. - -Dependencies install natively on Linux with `supportedArchitectures` extended to win32-x64, which materializes the Windows platform packages in the same store; the cmd-shim layer is bypassed by invoking each tool's JavaScript entrypoint directly, the same processes `run-gates` ultimately spawns. `nodeLinker: hoisted` is load-bearing, not stylistic: the independent prototype in [PR #689](https://github.com/deepseek-harness/deepseek-harness/pull/689) kept pnpm's default isolated layout — including a faithful offline Windows-pnpm re-install over a Linux-prefetched store — and Windows Node under Wine still could not resolve `@esbuild/win32-x64` or load the koffi prebuild through the isolated symlink chain, failing before any repository gate ran. A flat layout with real files is what makes the gates reachable at all; #689's checksum pinning is adopted, while its Windows-pnpm-installs-the-tree goal is explicitly given up (the install contract stays Linux-tested here). - -The lane targets the wall clock of the Linux CI jobs (about two minutes), from four levers: the master-refreshed pnpm store cache (restore-only, same key as ci.yml), Wine provisioning (apt install, Windows Node download, `wineboot`) running concurrently with `pnpm install`, the two blocking surfaces running concurrently — the same shape `run-gates` gives them on native Windows — and an apt-archive cache keyed on the runner image so Wine's package downloads are paid once per image version. - -Measured on 2026-07-27: 2m46s end-to-end on a warm-cache pull-request run (setup and cache restores ~17s, concurrent install+provision 33s, concurrent gates 110s), against 1.5–2.5 minutes for the Linux CI jobs and 7–9 minutes for the paid Windows lane; a cold-cache run pays roughly one extra minute. The 8-core benchmark leg never left the queue — the restricted `dsh-ubuntu-*` pools were also observed queueing indefinitely from the sibling KVM experiment — so the standard-runner number stands as the result, and no larger box is needed to hit the target. - -This is deliberately a fidelity probe, not a drop-in replacement: Wine reimplements the Win32 API over a case-sensitive ext4 (NTFS case-insensitivity is not emulated by default), provides no ConPTY, and substitutes its own security-descriptor and `MoveFileExW` semantics — exactly the surfaces the repo's `win32.ts` modules and PTY backend care about. The experiment measures which blocking gates pass, which fail for Wine reasons rather than product reasons, and the wall-clock cost relative to the recorded Windows benchmark lanes. - -Promotion, if the verdict is positive: fold the Wine lane in as the pull-request Windows signal for blocking gates and demote the real-Windows pool to the master serial reference; otherwise record the failure class here and keep the pool. - -## Alternatives considered - -**Keep the dedicated Windows pool (status quo).** It is the baseline being priced; nothing is wrong with its signal, only with paying for a Windows VM pool whose blocking surface is two build commands. - -**A full Windows guest under QEMU/KVM inside the Linux runner.** Real NT kernel, so full fidelity including case-insensitive NTFS and ConPTY — but tens of minutes of image download and unattended install before the first gate runs. Explored as the sibling experiment branch `exp/kvm-windows-ci`; the two experiments price fidelity against latency. - -**Windows pnpm performing the install under Wine ([PR #689](https://github.com/deepseek-harness/deepseek-harness/pull/689)).** The higher-fidelity variant of this same idea: MinGit and pnpm staged into the prefix, a Linux prefetch filling the store, then `pnpm install --offline` run by Windows Node so the install contract itself executes as win32. It reached the install but not the gates — Wine's networking could not reach the registry directly, and the isolated `node_modules` layout defeated resolution of the Windows platform packages even after a clean offline install. This lane trades that fidelity away (hoisted layout, Linux-side install) to reach the gates; the two records are complementary halves of the same verdict. - -**Filesystem-semantics lanes on Linux (casefolded ext4, filename lint).** Catches the highest-frequency Windows breakage class for near-zero cost but proves nothing about win32 binaries. Explored as the sibling experiment branch `exp/casefold-windows-ci`. - -**Windows containers.** Not possible: Windows containers require a Windows host kernel; a hosted Linux runner cannot run them. - -**Dropping the Windows lane.** Rejected — win32 is a first-class product target: the koffi-backed DACL and durable-namespace modules, ConPTY-based PTY sessions, and Windows path policy all ship in `packages/`. - -## Acceptance criteria - -- The workflow completes on `ubuntu-latest` with an independent pass/fail verdict per blocking surface (build, production site) and a recorded wall-clock comparison against both the paid Windows lane and the Linux CI jobs. -- End-to-end wall clock lands in the same band as the Linux CI jobs (minutes, not tens of minutes), demonstrating the pool-replacement case on cost as well as signal. -- A decision is recorded here: promote the lane, keep it as a non-blocking canary, or reject it with the observed failure class. - -## Risks - -- False greens: Wine's case-sensitive filesystem and permissive path handling can pass code that breaks on real NTFS, so this lane can complement but never fully replace a real-kernel check for release qualification. -- False reds: missing or stubbed Win32 APIs under Wine fail gates for non-product reasons, and each such failure costs triage time to classify. -- Throughput: Wine's syscall translation on the 2-core standard runner may push the blocking gates past the paid Windows lane's wall clock, erasing the cost argument; the run records the numbers either way. diff --git a/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.zh.md b/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.zh.md deleted file mode 100644 index 3a91286111..0000000000 --- a/.agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.zh.md +++ /dev/null @@ -1,51 +0,0 @@ -# Agent Note: 在 Linux runner 上用 Wine 运行 Windows 阻断门禁 - -Status: proposed - -[English](2026-07-27-wine-windows-gates-experiment.md) | 中文 - -## 问题 - -Pull request 的 Windows 通道存在的意义是证明两个阻断性 win32 表面——workspace 构建与生产站点——外加一份观察性可移植性清单，它运行在一个专用的付费 Windows larger-runner 池上；master 串行参照又增加一个托管 Windows 作业。该池是这条流水线中唯一需要 Windows VM 的理由，而其供给、计价与缓慢的准备阶段主导了该通道的成本。 - -悬而未决的问题是：一台普通 Linux runner 能否为阻断表面产出等效的 win32 信号，让专用 Windows 池收缩为仅 master 的参照、甚至完全退出 pull request 路径？ - -## 提案 - -[exp-wine-windows.yml](../../../../.github/workflows/exp-wine-windows.yml)（自身路径过滤，外加手动触发）在 `ubuntu-latest` 上通过 Wine 用真实 Windows 二进制运行阻断门禁命令：校验和验证过的 win-x64 Node.js 执行 `tsc -b`、`tsdown` 与 VitePress 生产构建，因此工具链的 win32 分支——反斜杠路径处理、`CreateProcess` 派生语义、`@esbuild/win32-x64` 的 PE 加载、以及 rolldown/rollup 的 MSVC `.node` 插件——都真正执行。 - -依赖在 Linux 上原生安装，`supportedArchitectures` 扩展到 win32-x64，使 Windows 平台包物化进同一个 store；通过直接调用各工具的 JavaScript 入口绕开 cmd-shim 层，这正是 `run-gates` 最终派生的那些进程。`nodeLinker: hoisted` 是承重的，不是风格问题：[PR #689](https://github.com/deepseek-harness/deepseek-harness/pull/689) 的独立原型保留了 pnpm 默认的 isolated 布局——包括在 Linux 预取的 store 上忠实地用 Windows pnpm 离线重装——而 Wine 下的 Windows Node 依然无法穿过 isolated 符号链接链解析 `@esbuild/win32-x64` 或加载 koffi 预编译产物，在任何仓库门禁运行前就失败了。扁平的真实文件布局才让门禁变得可达；本通道采纳了 #689 的校验和固定，同时明确放弃其"Windows pnpm 安装依赖树"的目标（安装契约在此仍由 Linux 侧验证）。 - -该通道以 Linux CI 作业的墙钟（约两分钟）为目标，靠四个杠杆：master 刷新的 pnpm store 缓存（只恢复，与 ci.yml 同键）、Wine 供给（apt 安装、Windows Node 下载、`wineboot`）与 `pnpm install` 并发运行、两个阻断表面并发运行——与 `run-gates` 在原生 Windows 上给它们的形状相同——以及按 runner 镜像为键的 apt 归档缓存，使 Wine 的包下载每个镜像版本只付一次。 - -2026-07-27 实测：热缓存 pull request 运行端到端 2 分 46 秒（准备与缓存恢复约 17 秒，并发安装+供给 33 秒，并发门禁 110 秒），对照 Linux CI 作业的 1.5–2.5 分钟与付费 Windows 通道的 7–9 分钟；冷缓存约多付一分钟。8 核基准腿从未离开队列——受限的 `dsh-ubuntu-*` 池在兄弟 KVM 实验中也被观察到无限排队——因此标准 runner 的数字即为结果，达标不需要更大的机器。 - -这刻意是一次保真度探针，而非直接替换：Wine 在大小写敏感的 ext4 之上重实现 Win32 API（默认不模拟 NTFS 的大小写不敏感）、不提供 ConPTY、并用自己的安全描述符与 `MoveFileExW` 语义替代——恰是本仓库 `win32.ts` 模块与 PTY 后端关心的表面。实验度量哪些阻断门禁通过、哪些因 Wine 原因而非产品原因失败，以及相对已记录 Windows 基准通道的墙钟成本。 - -若结论为正则晋升：把 Wine 通道并入为 pull request 的阻断门禁 Windows 信号，将真实 Windows 池降级为 master 串行参照；否则在此记录失败类别并保留该池。 - -## 考虑过的替代方案 - -**保留专用 Windows 池（现状）。** 它正是被计价的基线；其信号没有问题，问题只在于为一个阻断表面仅是两条构建命令的 Windows VM 池付费。 - -**在 Linux runner 内用 QEMU/KVM 跑完整 Windows 客户机。** 真实 NT 内核，保真度完整，包括大小写不敏感的 NTFS 与 ConPTY——但首个门禁运行前要花数十分钟下载镜像并做无人值守安装。作为兄弟实验分支 `exp/kvm-windows-ci` 探索；两个实验共同为保真度与延迟定价。 - -**在 Wine 下由 Windows pnpm 执行安装（[PR #689](https://github.com/deepseek-harness/deepseek-harness/pull/689)）。** 同一想法的更高保真度变体：把 MinGit 与 pnpm 放进 prefix，用 Linux 预取填充 store，再由 Windows Node 运行 `pnpm install --offline`，让安装契约本身以 win32 身份执行。它到达了安装但没到达门禁——Wine 的网络无法直接访问 registry，且 isolated 的 `node_modules` 布局即便在干净的离线安装后也挫败了 Windows 平台包的解析。本通道用掉这份保真度（hoisted 布局、Linux 侧安装）来换取门禁可达；两份记录是同一裁决互补的两半。 - -**Linux 上的文件系统语义通道（casefold ext4、文件名 lint）。** 以近零成本捕获最高频的 Windows 破坏类别，但对 win32 二进制什么也证明不了。作为兄弟实验分支 `exp/casefold-windows-ci` 探索。 - -**Windows 容器。** 不可行：Windows 容器要求 Windows 宿主内核；托管 Linux runner 无法运行。 - -**砍掉 Windows 通道。** 已否决——win32 是一等产品目标：基于 koffi 的 DACL 与持久命名空间模块、基于 ConPTY 的 PTY 会话、以及 Windows 路径策略都随 `packages/` 交付。 - -## 验收标准 - -- 该 workflow 在 `ubuntu-latest` 上完成，对每个阻断表面（构建、生产站点）给出独立的通过/失败裁决，并记录与付费 Windows 通道及 Linux CI 作业两者的墙钟对比。 -- 端到端墙钟落在 Linux CI 作业的同一档位（分钟级，而非数十分钟），从成本与信号两方面共同论证替换池的理由。 -- 在此记录一项决定：晋升该通道、保留为非阻断金丝雀、或以观察到的失败类别否决。 - -## 风险 - -- 假绿：Wine 的大小写敏感文件系统与宽松路径处理可能放过在真实 NTFS 上会坏的代码，因此该通道可以补充、但永远无法完全替代发布资格所需的真实内核检查。 -- 假红：Wine 下缺失或桩化的 Win32 API 会因非产品原因让门禁失败，每次此类失败都要花分诊时间归类。 -- 吞吐：Wine 的系统调用翻译在 2 核标准 runner 上可能让阻断门禁的墙钟超过付费 Windows 通道，抹掉成本论点；无论结果如何，运行都会记录数字。 diff --git a/.github/AGENTS.md b/.github/AGENTS.md index 5f03c8617d..ff4fd4e6b2 100644 --- a/.github/AGENTS.md +++ b/.github/AGENTS.md @@ -1,3 +1,3 @@ # AGENTS.md — GitHub Actions -Run Windows jobs under native `pwsh`. +Run jobs on Windows runners (`windows-*` labels) under native `pwsh`. The pull-request `windows` job is not one of them: it runs Windows Node under Wine on hosted Linux, so its steps are bash — see the [Wine lane Agent Note](../.agents/notes/implemented/process/2026-07-27-wine-windows-gates-experiment.md). diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f02d30a563..94c97fd0be 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -281,41 +281,224 @@ jobs: - name: Run complete keyless Python suite run: uv run --python 3.10 --group test --project python/sdk pytest - # One standard Windows box shares setup across the required build/site checks - # and the observational portability inventory. Serial worker bounds keep this - # recovery path portable; Linux owns duplicate lint, coverage, and snapshots. + # The required pull-request Windows signal: the two blocking win32 surfaces + # (workspace build, production site) execute with real, checksum-verified + # Windows Node under Wine on standard hosted Linux. The master + # serial-windows job below keeps the complete native-kernel inventory — + # including the observational portability gates this lane does not run — + # on real windows-2025. Direct tool entrypoints stand in for pnpm's cmd + # shims, which a Linux-side install does not create; layout, fidelity + # limits, and measured timings live in + # .agents/notes/implemented/process/2026-07-27-wine-windows-gates-experiment.md windows: if: github.event_name == 'pull_request' - runs-on: windows-2025 - name: windows node 24 / complete + runs-on: ubuntu-latest + name: windows node 24 / wine blocking + timeout-minutes: 15 env: - DSH_COVERAGE_MAX_WORKERS: '1' - DSH_GATE_CONCURRENCY: '1' - DSH_PUBLINT_CONCURRENCY: '1' + WINEDEBUG: '-all' + WINEARCH: win64 + # Skip Wine Mono / Gecko installers: Node needs neither. + WINEDLLOVERRIDES: 'mscoree,mshtml=' steps: - uses: actions/checkout@v6 - - - name: Enable Developer Mode (symlink support) - shell: pwsh - run: >- - reg add "HKLM\SOFTWARE\Microsoft\Windows\CurrentVersion\AppModelUnlock" - /t REG_DWORD /f /v "AllowDevelopmentWithoutDevLicense" /d "1" + with: + persist-credentials: false - uses: actions/setup-node@v6 with: node-version: ${{ env.PRIMARY_NODE_VERSION }} - # Extracting the many-file pnpm store cache is slower than a clean install, - # and saving it adds more latency after gates. - - name: Enable corepack and install (immutable) - shell: pwsh + - uses: actions/cache/restore@v4 + with: + path: /home/runner/.local/share/pnpm/store/v11 + key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm- + + # Master's wine-apt-cache job seeds the default-branch scope every pull + # request can read; a save from this job only reaches reruns of the + # same merge ref. + - name: Compose Wine apt cache key + id: wine-cache-key + run: echo "key=wine-debs-${ImageOS:-linux}-${ImageVersion:-v0}" >> "$GITHUB_OUTPUT" + + - uses: actions/cache@v4 + with: + path: ~/wine-debs + key: ${{ steps.wine-cache-key.outputs.key }} + + - name: Install dependencies and provision Wine concurrently run: | corepack enable - pnpm install --frozen-lockfile - - name: Run blocking and observational Windows gates concurrently - shell: pwsh - run: pnpm run check:ci:windows-complete + # Windows-lane install-time overrides. supportedArchitectures + # additionally materializes the win32-x64 platform packages + # (@esbuild/win32-x64, rolldown and rollup MSVC bindings) the + # Windows toolchain resolves at runtime; nodeLinker: hoisted lays + # node_modules out flat with real files because Windows Node under + # Wine does not realpath pnpm's isolated-layout symlinks. Neither + # override is recorded in the lockfile, so --frozen-lockfile stays + # valid. --ignore-scripts skips Linux lifecycle scripts no gate in + # this lane loads; the win32 binaries ship prebuilt. + cat >> pnpm-workspace.yaml <<'EOF' + + nodeLinker: hoisted + supportedArchitectures: + os: [current, win32] + cpu: [current, x64] + EOF + + pnpm install --frozen-lockfile --ignore-scripts & + install_pid=$! + + provision_wine() { + set -euo pipefail + # Wine from the apt cache when present; else download the full + # dependency closure once and keep it for the next run. The + # `wine` dispatcher package (not bare `wine64`) is what puts a + # binary on PATH. + if compgen -G "$HOME/wine-debs/*.deb" > /dev/null; then + sudo apt-get install -y --no-install-recommends "$HOME"/wine-debs/*.deb + else + sudo apt-get update + sudo apt-get install -y --no-install-recommends --download-only wine + mkdir -p "$HOME/wine-debs" + cp /var/cache/apt/archives/*.deb "$HOME/wine-debs/" 2>/dev/null || true + sudo apt-get install -y --no-install-recommends wine + fi + WINE_BIN='' + for candidate in "$(command -v wine || true)" "$(command -v wine64 || true)" /usr/lib/wine/wine64; do + if [ -n "$candidate" ] && [ -x "$candidate" ]; then WINE_BIN="$candidate"; break; fi + done + [ -n "$WINE_BIN" ] || { echo '::error::no wine binary found after install'; exit 1; } + echo "WINE_BIN=$WINE_BIN" >> "$GITHUB_ENV" + + # Windows Node for the repo's primary line, checksum-verified + # against the same dist directory. + version=$(curl -fsSL https://nodejs.org/dist/index.json \ + | jq -r --arg p "v${PRIMARY_NODE_VERSION}." '[.[] | select(.version | startswith($p))][0].version') + echo "Windows Node: $version" + curl -fsSL -o "$RUNNER_TEMP/node-win.zip" \ + "https://nodejs.org/dist/${version}/node-${version}-win-x64.zip" + curl -fsSL "https://nodejs.org/dist/${version}/SHASUMS256.txt" \ + | awk -v a="node-${version}-win-x64.zip" '$2 == a { print $1 " '"$RUNNER_TEMP"'/node-win.zip" }' \ + | sha256sum --check - + unzip -q "$RUNNER_TEMP/node-win.zip" -d "$RUNNER_TEMP/node-win" + echo "NODE_WIN=$RUNNER_TEMP/node-win/node-${version}-win-x64/node.exe" >> "$GITHUB_ENV" + + "$WINE_BIN" wineboot --init || true + wineserver -w || true + } + provision_wine & + wine_pid=$! + + install_status=0 + wait "$install_pid" || install_status=$? + wine_status=0 + wait "$wine_pid" || wine_status=$? + if (( install_status != 0 )); then exit "$install_status"; fi + exit "$wine_status" + + - name: Resolve entrypoints, link vue, smoke Windows Node + run: | + # Node under Wine cannot attach stdio to the Actions runner's pipes + # (Socket open EBADF at bootstrap), so every invocation runs through + # this wrapper: stdio to a regular file, replayed after exit. + cat > "$RUNNER_TEMP/wine-node.sh" <<'SH' + #!/usr/bin/env bash + set -u + log="$1"; shift + "$WINE_BIN" "$NODE_WIN" "$@" < /dev/null > "$log" 2>&1 + status=$? + tail -n 300 "$log" + exit "$status" + SH + chmod +x "$RUNNER_TEMP/wine-node.sh" + + resolve() { + local name="$1"; shift + for p in "$@"; do + if [ -f "$p" ]; then echo "$name=$PWD/$p" >> "$GITHUB_ENV"; return 0; fi + done + echo "::error::$name not found at any of: $*"; return 1 + } + resolve TSC_JS node_modules/typescript/bin/tsc + resolve TSDOWN_JS node_modules/tsdown/dist/run.mjs + resolve VITEPRESS_JS website/node_modules/vitepress/bin/vitepress.js node_modules/vitepress/bin/vitepress.js + + # VitePress links vue into the site's node_modules at build time; + # Wine cannot CREATE Windows symlinks (ENOTSUP) but follows + # pre-existing Unix ones, so lay the link down host-side. + if [ -d node_modules/vue ] && [ ! -e website/node_modules/vue ]; then + mkdir -p website/node_modules + ln -s ../../node_modules/vue website/node_modules/vue + fi + + "$RUNNER_TEMP/wine-node.sh" "$RUNNER_TEMP/smoke.log" -p "'smoke: ' + process.platform + ' ' + process.arch + ' ' + process.version" + + # The two blocking surfaces run concurrently, the same shape run-gates + # gives ci-windows-blocking on native Windows: `build` = tsc -b then + # tsdown, `production site` = the VitePress build. Both statuses are + # captured so one failure cannot hide the other's result. + - name: Run blocking Windows gates concurrently under Wine + run: | + build_gate() { + "$RUNNER_TEMP/wine-node.sh" "$RUNNER_TEMP/tsc.log" "$TSC_JS" -b --pretty false || return $? + "$RUNNER_TEMP/wine-node.sh" "$RUNNER_TEMP/tsdown.log" "$TSDOWN_JS" + } + site_gate() { + cd website + "$RUNNER_TEMP/wine-node.sh" "$RUNNER_TEMP/site.log" "$VITEPRESS_JS" build . + } + start=$SECONDS + build_gate > "$RUNNER_TEMP/build-gate.out" 2>&1 & + build_pid=$! + site_gate > "$RUNNER_TEMP/site-gate.out" 2>&1 & + site_pid=$! + build_status=0 + wait "$build_pid" || build_status=$? + site_status=0 + wait "$site_pid" || site_status=$? + echo "== build gate (exit $build_status, $((SECONDS - start))s elapsed) ==" + tail -n 120 "$RUNNER_TEMP/build-gate.out" + echo "== production site gate (exit $site_status, $((SECONDS - start))s elapsed) ==" + tail -n 120 "$RUNNER_TEMP/site-gate.out" + if (( build_status != 0 )); then exit "$build_status"; fi + exit "$site_status" + + - name: Shut down wineserver + if: always() + run: wineserver -k 2>/dev/null || true + + # Master seeds the Wine apt-archive cache in the default-branch scope, + # which every pull request's windows job can restore; saves from + # pull-request runs are scoped to their own merge ref and help nobody + # else. Runs in seconds when the image version already has a cache. + wine-apt-cache: + if: github.event_name == 'push' && github.ref == 'refs/heads/master' + name: wine apt cache + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Compose Wine apt cache key + id: wine-cache-key + run: echo "key=wine-debs-${ImageOS:-linux}-${ImageVersion:-v0}" >> "$GITHUB_OUTPUT" + + - uses: actions/cache@v4 + id: wine-cache + with: + path: ~/wine-debs + key: ${{ steps.wine-cache-key.outputs.key }} + + - name: Download the Wine dependency closure + if: steps.wine-cache.outputs.cache-hit != 'true' + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends --download-only wine + mkdir -p "$HOME/wine-debs" + cp /var/cache/apt/archives/*.deb "$HOME/wine-debs/" + du -sh "$HOME/wine-debs" # Master pushes run only the serial reference jobs below. # Each host executes the complete, unsharded primary Node aggregate with one diff --git a/.github/workflows/exp-wine-windows.yml b/.github/workflows/exp-wine-windows.yml deleted file mode 100644 index e9a79a18a7..0000000000 --- a/.github/workflows/exp-wine-windows.yml +++ /dev/null @@ -1,230 +0,0 @@ -# EXPERIMENT: run the blocking Windows CI gates on a Linux runner through -# Wine with a real Windows Node.js binary, at roughly the wall clock of the -# Linux CI jobs (~2 min). Speed comes from four levers: the master-refreshed -# pnpm store cache, provisioning Wine concurrently with the dependency -# install, running the two blocking surfaces concurrently (the same shape -# run-gates gives them on native Windows), and an apt package cache for Wine -# itself. Dependency provisioning happens natively on Linux with -# `supportedArchitectures` extended to win32-x64 so the Windows -# esbuild/rolldown/rollup binaries are present, and `nodeLinker: hoisted` -# because Windows Node under Wine does not realpath pnpm's isolated-layout -# Unix symlinks — the sibling prototype in PR #689 kept the isolated layout -# and failed on exactly that. The pnpm-run/cmd shim layer is deliberately -# bypassed; each gate invokes its tool's JavaScript entrypoint directly — the -# same commands run-gates ultimately spawns. Owning rationale and promotion -# criteria: -# .agents/notes/proposed/process/2026-07-27-wine-windows-gates-experiment.md -name: Experiment Wine Windows gates - -on: - workflow_dispatch: - pull_request: - paths: - - .github/workflows/exp-wine-windows.yml - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -permissions: - contents: read - -env: - PRIMARY_NODE_VERSION: '24' - -jobs: - wine-blocking-gates: - name: wine / blocking windows gates (${{ matrix.runner }}) - # Pull requests run the free standard runner only; a manual dispatch adds - # the 8-core benchmark pool for a like-for-like core-count comparison. - # The larger leg stays dispatch-only because those restricted pools can - # queue indefinitely (observed on the sibling KVM experiment). - runs-on: ${{ matrix.runner }} - strategy: - fail-fast: false - matrix: - runner: ${{ fromJSON(github.event_name == 'workflow_dispatch' && '["ubuntu-latest", "dsh-ubuntu-24-04-8core"]' || '["ubuntu-latest"]') }} - timeout-minutes: 30 - env: - WINEDEBUG: '-all' - WINEARCH: win64 - # Skip Wine Mono / Gecko installers: Node needs neither. - WINEDLLOVERRIDES: 'mscoree,mshtml=' - steps: - - uses: actions/checkout@v6 - with: - persist-credentials: false - - - uses: actions/setup-node@v6 - with: - node-version: ${{ env.PRIMARY_NODE_VERSION }} - - # The default-branch pnpm store cache ci.yml maintains; restore-only, - # same key, so this lane rides the cache master already refreshes. - - uses: actions/cache/restore@v4 - with: - path: /home/runner/.local/share/pnpm/store/v11 - key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm- - - # Keyed on the runner image so a new image version re-downloads once. - # Cache scoping: each trigger seeds its own scope (pull_request → the - # PR merge ref, dispatch → the branch); only same-scope reruns hit. - # Promotion to ci.yml would let master seed the shared default-branch - # scope every trigger reads, as the pnpm store cache already does. - - name: Compose Wine apt cache key - id: wine-cache-key - run: echo "key=wine-debs-${ImageOS:-linux}-${ImageVersion:-v0}" >> "$GITHUB_OUTPUT" - - - uses: actions/cache@v4 - with: - path: ~/wine-debs - key: ${{ steps.wine-cache-key.outputs.key }} - - - name: Install dependencies and provision Wine concurrently - run: | - corepack enable - - # Experiment-only install-time overrides. supportedArchitectures - # additionally materializes the win32-x64 platform packages - # (@esbuild/win32-x64, rolldown and rollup MSVC bindings) the - # Windows toolchain resolves at runtime; nodeLinker: hoisted lays - # node_modules out flat with real files because Windows Node under - # Wine does not realpath pnpm's isolated-layout symlinks (PR #689's - # failure mode). Neither override is recorded in the lockfile, so - # --frozen-lockfile stays valid. --ignore-scripts skips the Linux - # esbuild/node-pty/lefthook lifecycle scripts: no gate in this lane - # loads them, and the win32 binaries ship prebuilt in their - # packages. - cat >> pnpm-workspace.yaml <<'EOF' - - nodeLinker: hoisted - supportedArchitectures: - os: [current, win32] - cpu: [current, x64] - EOF - - pnpm install --frozen-lockfile --ignore-scripts & - install_pid=$! - - provision_wine() { - set -euo pipefail - # Wine from the apt cache when present; else download the full - # dependency closure once and keep it for the next run. The - # `wine` dispatcher package (not bare `wine64`) is what puts a - # binary on PATH. - if compgen -G "$HOME/wine-debs/*.deb" > /dev/null; then - sudo apt-get install -y --no-install-recommends "$HOME"/wine-debs/*.deb - else - sudo apt-get update - sudo apt-get install -y --no-install-recommends --download-only wine - mkdir -p "$HOME/wine-debs" - cp /var/cache/apt/archives/*.deb "$HOME/wine-debs/" 2>/dev/null || true - sudo apt-get install -y --no-install-recommends wine - fi - WINE_BIN='' - for candidate in "$(command -v wine || true)" "$(command -v wine64 || true)" /usr/lib/wine/wine64; do - if [ -n "$candidate" ] && [ -x "$candidate" ]; then WINE_BIN="$candidate"; break; fi - done - [ -n "$WINE_BIN" ] || { echo '::error::no wine binary found after install'; exit 1; } - echo "WINE_BIN=$WINE_BIN" >> "$GITHUB_ENV" - - # Windows Node for the repo's primary line, checksum-verified - # against the same dist directory (adopted from PR #689). - version=$(curl -fsSL https://nodejs.org/dist/index.json \ - | jq -r --arg p "v${PRIMARY_NODE_VERSION}." '[.[] | select(.version | startswith($p))][0].version') - echo "Windows Node: $version" - curl -fsSL -o "$RUNNER_TEMP/node-win.zip" \ - "https://nodejs.org/dist/${version}/node-${version}-win-x64.zip" - curl -fsSL "https://nodejs.org/dist/${version}/SHASUMS256.txt" \ - | awk -v a="node-${version}-win-x64.zip" '$2 == a { print $1 " '"$RUNNER_TEMP"'/node-win.zip" }' \ - | sha256sum --check - - unzip -q "$RUNNER_TEMP/node-win.zip" -d "$RUNNER_TEMP/node-win" - echo "NODE_WIN=$RUNNER_TEMP/node-win/node-${version}-win-x64/node.exe" >> "$GITHUB_ENV" - - "$WINE_BIN" wineboot --init || true - wineserver -w || true - } - provision_wine & - wine_pid=$! - - install_status=0 - wait "$install_pid" || install_status=$? - wine_status=0 - wait "$wine_pid" || wine_status=$? - if (( install_status != 0 )); then exit "$install_status"; fi - exit "$wine_status" - - - name: Resolve entrypoints, link vue, smoke Windows Node - run: | - # Node under Wine cannot attach stdio to the Actions runner's pipes - # (Socket open EBADF at bootstrap), so every invocation runs through - # this wrapper: stdio to a regular file, replayed after exit. - cat > "$RUNNER_TEMP/wine-node.sh" <<'SH' - #!/usr/bin/env bash - set -u - log="$1"; shift - "$WINE_BIN" "$NODE_WIN" "$@" < /dev/null > "$log" 2>&1 - status=$? - tail -n 300 "$log" - exit "$status" - SH - chmod +x "$RUNNER_TEMP/wine-node.sh" - - resolve() { - local name="$1"; shift - for p in "$@"; do - if [ -f "$p" ]; then echo "$name=$PWD/$p" >> "$GITHUB_ENV"; return 0; fi - done - echo "::error::$name not found at any of: $*"; return 1 - } - resolve TSC_JS node_modules/typescript/bin/tsc - resolve TSDOWN_JS node_modules/tsdown/dist/run.mjs - resolve VITEPRESS_JS website/node_modules/vitepress/bin/vitepress.js node_modules/vitepress/bin/vitepress.js - - # VitePress links vue into the site's node_modules at build time; - # Wine cannot CREATE Windows symlinks (ENOTSUP) but follows - # pre-existing Unix ones, so lay the link down host-side. - if [ -d node_modules/vue ] && [ ! -e website/node_modules/vue ]; then - mkdir -p website/node_modules - ln -s ../../node_modules/vue website/node_modules/vue - fi - - "$RUNNER_TEMP/wine-node.sh" "$RUNNER_TEMP/smoke.log" -p "'smoke: ' + process.platform + ' ' + process.arch + ' ' + process.version" - - # The two blocking surfaces run concurrently, the same shape run-gates - # gives ci-windows-blocking on native Windows (DSH_GATE_CONCURRENCY): - # `build` = tsc -b then tsdown, `production site` = the VitePress - # build. Both statuses are captured so one failure cannot hide the - # other's result. - - name: Run blocking Windows gates concurrently under Wine - timeout-minutes: 20 - run: | - build_gate() { - "$RUNNER_TEMP/wine-node.sh" "$RUNNER_TEMP/tsc.log" "$TSC_JS" -b --pretty false || return $? - "$RUNNER_TEMP/wine-node.sh" "$RUNNER_TEMP/tsdown.log" "$TSDOWN_JS" - } - site_gate() { - cd website - "$RUNNER_TEMP/wine-node.sh" "$RUNNER_TEMP/site.log" "$VITEPRESS_JS" build . - } - start=$SECONDS - build_gate > "$RUNNER_TEMP/build-gate.out" 2>&1 & - build_pid=$! - site_gate > "$RUNNER_TEMP/site-gate.out" 2>&1 & - site_pid=$! - build_status=0 - wait "$build_pid" || build_status=$? - site_status=0 - wait "$site_pid" || site_status=$? - echo "== build gate (exit $build_status, $((SECONDS - start))s elapsed) ==" - tail -n 120 "$RUNNER_TEMP/build-gate.out" - echo "== production site gate (exit $site_status, $((SECONDS - start))s elapsed) ==" - tail -n 120 "$RUNNER_TEMP/site-gate.out" - if (( build_status != 0 )); then exit "$build_status"; fi - exit "$site_status" - - - name: Shut down wineserver - if: always() - run: wineserver -k 2>/dev/null || true From 31073cc60f50fa08e098170061e3949d2c2e7eba Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Mon, 27 Jul 2026 16:36:08 +0800 Subject: [PATCH 12/14] test(acp): refresh web-fetch tool schema snapshot --- .../tests/snapshots/web-fetch/tool-schemas.expected.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/acp-agent/tests/snapshots/web-fetch/tool-schemas.expected.json b/examples/acp-agent/tests/snapshots/web-fetch/tool-schemas.expected.json index 1ee86b38ba..70940f8907 100644 --- a/examples/acp-agent/tests/snapshots/web-fetch/tool-schemas.expected.json +++ b/examples/acp-agent/tests/snapshots/web-fetch/tool-schemas.expected.json @@ -288,7 +288,7 @@ "description": "The COMPLETE task list, replacing any previous list.", "items": { "type": "object", - "additionalProperties": true, + "additionalProperties": false, "properties": { "content": { "type": "string", From 9801a0d10221818f361cfa753d6bd15aff0d01d0 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Mon, 27 Jul 2026 16:40:02 +0800 Subject: [PATCH 13/14] fix(tool-web): bound HTML conversion work --- ...ndown-for-tool-web-html-markdown.i18n.yaml | 6 +- ...-26-turndown-for-tool-web-html-markdown.md | 8 +- ...-turndown-for-tool-web-html-markdown.zh.md | 8 +- docs/config-catalog.md | 4 +- .../acp-agent/web-fetch-fixture-server.mjs | 9 +- packages/web/tool-web/README.i18n.yaml | 6 +- packages/web/tool-web/README.md | 6 +- packages/web/tool-web/README.zh.md | 6 +- packages/web/tool-web/src/fetch.ts | 207 ++++++++++++++---- packages/web/tool-web/src/index.ts | 14 +- packages/web/tool-web/tests/tool-web.spec.ts | 140 +++++++++--- 11 files changed, 314 insertions(+), 100 deletions(-) diff --git a/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.i18n.yaml b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.i18n.yaml index a114e32885..41f59f644f 100644 --- a/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.i18n.yaml +++ b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.i18n.yaml @@ -1,6 +1,6 @@ # Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: -# pnpm run verify-translation-pairing --write -2026-07-26-turndown-for-tool-web-html-markdown.md: c7ef4bf538cc949eec8463c8a2ac750685d1a715 -2026-07-26-turndown-for-tool-web-html-markdown.zh.md: 3104dac3cd6db516396b6773f5d2185f3da22ca3 +# pnpm run verify-translation-pairing --write .agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md +2026-07-26-turndown-for-tool-web-html-markdown.md: 0e387021e3d3be3011cc0d64d37864b30aec4fdf +2026-07-26-turndown-for-tool-web-html-markdown.zh.md: 44e7d08c1db40a1203e8cda955b521774335e774 diff --git a/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md index c7ef4bf538..0e387021e3 100644 --- a/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md +++ b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md @@ -10,7 +10,7 @@ English | [中文](2026-07-26-turndown-for-tool-web-html-markdown.zh.md) ## Decision -`packages/web/tool-web/src/fetch.ts` owns a module-level [`turndown`](https://github.com/mixmark-io/turndown) instance (`headingStyle: 'atx'`, `codeBlockStyle: 'fenced'`, `bulletListMarker: '-'` — fixed model-facing presentation, not deployment tunables) with `@joplin/turndown-plugin-gfm`'s composite `gfm` plugin for tables/strikethrough and `remove(['script', 'style', 'noscript'])` replacing the old wholesale drops. `renderBody`'s `html` arm guards the conversion twice: a linear tag-scan preflight passes bodies nested past 512 levels through raw (the synchronous walk is superlinear on unclosed nesting — measured seconds at 20k levels — during which the cooperative timeout cannot fire), and a try/catch falls back to the raw HTML when turndown still throws on markup the scan cannot see; a degraded page beats an error for a body the provider already decoded. `formatFetchOutput` bounds the complete output (`fetchMaxOutputChars` config, default 200,000) because markdown escaping can expand converted HTML to ~2× a provider's body cap. `html.ts` and its conversion tests are deleted; the fallback and the status-header/truncation-footer formatting are tested in `tests/tool-web.spec.ts`, and the README's Known Limitations trades the regex-converter caveat for the pathological-nesting fallback. The gfm plugin ships no types; `src/turndown-plugin-gfm.d.ts` declares the one imported export over `@types/turndown` (a devDependency). +`packages/web/tool-web/src/fetch.ts` owns a module-level [`turndown`](https://github.com/mixmark-io/turndown) instance (`headingStyle: 'atx'`, `codeBlockStyle: 'fenced'`, `bulletListMarker: '-'` — fixed model-facing presentation, not deployment tunables) with `@joplin/turndown-plugin-gfm`'s composite `gfm` plugin for tables/strikethrough and `remove(['script', 'style', 'noscript'])` replacing the old wholesale drops. `formatFetchOutput` limits both the source prefix converted synchronously and the complete rendered output with `fetchMaxOutputChars` (default 200,000), so a custom provider cannot make conversion work unbounded before the output cap applies. The HTML arm then guards conversion twice: a conservative linear lexical pass treats comment contents conservatively, skips raw-text bodies, honors quoted tag text, and passes a body through raw when its stack crosses 512 levels; a try/catch also falls back to raw HTML when turndown rejects markup the guard cannot model. The GFM cell rule is overridden to ignore `colspan`, which Markdown cannot represent, rather than letting an untrusted numeric attribute synthesize arbitrary empty cells. `html.ts` and its conversion tests are deleted; the source/output bounds, fallback, and status-header/truncation-footer formatting are tested in `tests/tool-web.spec.ts`, and the README's Known Limitations trades the regex-converter caveat for the bounded degradation cases. The gfm plugin ships no types; `src/turndown-plugin-gfm.d.ts` declares the one imported export over `@types/turndown` (a devDependency). The dependency-weight question the proposal flagged resolves in favor of the swap: `@deepseek-ai/dsh-tool-web` is in the single-file-executable closure ([single-exe note](../architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.md)), and the exe's asset globs would pack ~7.9 MB of the three packages as published — but ~6 MB of that is `@mixmark-io/domino`'s test corpus (`test/**`), with runtime `lib/` at ~550 KB against a ~174 MB artifact, under 0.5% either way. @@ -27,11 +27,11 @@ The previously-missing keyless `web_fetch` snapshot ships with the change as the ## Consequences -- **Bought**: full-fidelity model-visible markdown — tables, images, strikethrough, nested emphasis, fenced code blocks, and the complete named-entity set — plus the deletion of the bespoke converter and its entity tables, with the README's regex-converter caveat narrowed to one degenerate case. -- **Paid**: two runtime dependencies (`turndown` → `@mixmark-io/domino`) enter tool-web and therefore the exe closure (~550 KB of runtime code as measured above), and a new failure mode — pathological nesting — is handled by falling back to raw HTML rather than converting. +- **Bought**: standards-based model-visible markdown — ordinary tables, images, strikethrough, nested emphasis, fenced code blocks, and the complete named-entity set — plus the deletion of the bespoke converter and its entity tables. +- **Paid**: two runtime dependencies (`turndown` → `@mixmark-io/domino`) enter tool-web and therefore the exe closure (~550 KB of runtime code as measured above); overlong input is converted only through a bounded prefix, pathological nesting falls back to raw HTML, and spanning table cells are flattened because GFM has no corresponding syntax. - Model-visible output changed on every fetched HTML page; nothing pinned the old output, and the new snapshot pins the new one. ## Testing -- `packages/web/tool-web/tests/tool-web.spec.ts` covers the turndown conversion surface (entities, links, tables, nesting, script/style/noscript removal) through `renderBody`, the fast raw-HTML passthrough for 20k-level nesting, the depth scan's void/self-closing/unbalanced cases, the residual converter-throw fallback, and the whole-output cap at expanding, exact, and tiny budgets; per-file coverage on the package src is 100%. +- `packages/web/tool-web/tests/tool-web.spec.ts` covers the turndown conversion surface (entities, links, tables, nesting, script/style/noscript removal), ignored table spans, source-prefix and complete-output bounds, fast raw-HTML passthrough for deep or deceptively closed nesting, linear handling of malformed tags, the residual converter-throw fallback, and exact and tiny output budgets; per-file coverage on the package src is 100%. - The `web-fetch` acp-agent snapshot pins the assembled behavior keylessly end to end (real Loader composition, real HTTP fetch, real conversion). diff --git a/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.zh.md b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.zh.md index 3104dac3cd..44e7d08c1d 100644 --- a/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.zh.md +++ b/.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.zh.md @@ -10,7 +10,7 @@ Status: implemented ## 决策 -`packages/web/tool-web/src/fetch.ts` 持有一个模块级 [`turndown`](https://github.com/mixmark-io/turndown) 实例（`headingStyle: 'atx'`、`codeBlockStyle: 'fenced'`、`bulletListMarker: '-'`——固定的面向模型呈现方式，不是部署可调项），配合 `@joplin/turndown-plugin-gfm` 的组合 `gfm` 插件提供表格／删除线支持，并用 `remove(['script', 'style', 'noscript'])` 替代旧实现的整体剥离。`renderBody` 的 `html` 分支对转换做了双重防护：一次线性标签扫描预检把嵌套超过 512 层的主体直接原样透传（同步遍历在未闭合嵌套上呈超线性——实测 2 万层需要数秒——期间协作式超时无法触发），扫描看不到的标记若仍让 turndown 抛异常，则由 try/catch 回退为原始 HTML；对提供方已经解码的主体来说，降级页面好过报错。`formatFetchOutput` 对完整输出设上限（`fetchMaxOutputChars` 配置，默认 200,000）：markdown 转义可能把转换后的 HTML 膨胀到提供方主体上限的约 2 倍。`html.ts` 及其转换测试已删除；透传、回退与整体输出上限，连同状态头、截断页脚的格式化，都在 `tests/tool-web.spec.ts` 中有测试覆盖，README 的 Known Limitations 用病态嵌套回退条目替换了正则转换器的警示说明。gfm 插件不带类型声明；`src/turndown-plugin-gfm.d.ts` 基于 `@types/turndown`（devDependency）声明了唯一被导入的导出。 +`packages/web/tool-web/src/fetch.ts` 持有一个模块级 [`turndown`](https://github.com/mixmark-io/turndown) 实例（`headingStyle: 'atx'`、`codeBlockStyle: 'fenced'`、`bulletListMarker: '-'`——固定的面向模型呈现方式，不是部署可调项），配合 `@joplin/turndown-plugin-gfm` 的组合 `gfm` 插件提供表格／删除线支持，并用 `remove(['script', 'style', 'noscript'])` 替代旧实现的整体剥离。`formatFetchOutput` 通过 `fetchMaxOutputChars`（默认 200,000）同时限制同步转换的源前缀和完整渲染输出，因此自定义提供方无法在输出上限生效前造成无界的转换工作。随后，HTML 分支对转换做双重防护：保守的线性词法扫描会保守处理注释内容，跳过原始文本元素的内容，正确处理标签内的引号文本，并在栈深超过 512 层时将主体作为原始 HTML 直接透传；当 turndown 拒绝守卫无法建模的标记时，try/catch 同样回退为原始 HTML。GFM 单元格规则被覆写为忽略 `colspan`；Markdown 无法表示它，这也避免了不受信任的数值属性凭空合成任意数量的空单元格。`html.ts` 及其转换测试已删除；源／输出上限、回退以及状态头／截断页脚格式化均在 `tests/tool-web.spec.ts` 中有测试覆盖，README 的 Known Limitations 用有界降级情形替换了正则转换器警示。gfm 插件不带类型声明；`src/turndown-plugin-gfm.d.ts` 基于 `@types/turndown`（devDependency）声明了唯一被导入的导出。 提案标记的依赖体积问题的裁决结果支持替换：`@deepseek-ai/dsh-tool-web` 在单文件可执行文件闭包内（[single-exe 决策记录](../architecture/2026-07-10-single-file-executable-sdk-runtime-distribution.md)），可执行文件的资产 glob 会把这三个包按发布原样打入约 7.9 MB——但其中约 6 MB 是 `@mixmark-io/domino` 的测试语料（`test/**`），运行时 `lib/` 仅约 550 KB，相对约 174 MB 的产物，两种口径都不到 0.5%。 @@ -27,11 +27,11 @@ Status: implemented ## 后果 -- **收益**：模型可见的完整保真 markdown——表格、图片、删除线、嵌套强调、围栏代码块以及完整的命名实体集——并删除了自制转换器及其实体表，README 中的正则转换器警示收窄为一个退化用例。 -- **代价**：两个运行时依赖（`turndown` → `@mixmark-io/domino`）进入 tool-web 进而进入可执行文件闭包（如上实测约 550 KB 运行时代码），并新增一种失败模式——病态嵌套改为回退原始 HTML 而非转换。 +- **收益**：基于标准的模型可见 markdown——普通表格、图片、删除线、嵌套强调、围栏代码块以及完整的命名实体集——并删除了自制转换器及其实体表。 +- **代价**：两个运行时依赖（`turndown` → `@mixmark-io/domino`）进入 tool-web 进而进入可执行文件闭包（如上实测约 550 KB 运行时代码）；超长输入只转换有界前缀，病态嵌套回退为原始 HTML，跨列表格单元格会被展平，因为 GFM 没有对应语法。 - 每个抓取到的 HTML 页面上模型可见的输出都已变化；旧输出本无任何固定，新快照固定了新输出。 ## 测试 -- `packages/web/tool-web/tests/tool-web.spec.ts` 通过 `renderBody` 覆盖 turndown 转换面（实体、链接、表格、嵌套、script/style/noscript 移除）、2 万层嵌套的快速原样透传、深度扫描的空元素／自闭合／不平衡用例、残余的转换器抛错回退，以及在膨胀、恰好、极小预算下的整体输出上限；该包 src 的逐文件覆盖率为 100%。 +- `packages/web/tool-web/tests/tool-web.spec.ts` 覆盖 turndown 转换面（实体、链接、表格、嵌套、script/style/noscript 移除）、被忽略的表格跨列、源前缀与完整输出上限、深层或带欺骗性闭合嵌套的快速原始 HTML 透传、畸形标签的线性处理、残余的转换器抛错回退，以及恰好达到上限和极小的输出预算；该包 src 的逐文件覆盖率为 100%。 - acp-agent 的 `web-fetch` 快照无密钥地端到端固定组装后的行为（真实 Loader 组合、真实 HTTP 抓取、真实转换）。 diff --git a/docs/config-catalog.md b/docs/config-catalog.md index 471a5dce96..fb96ff101a 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -1669,12 +1669,12 @@ export interface Config { fetchTimeoutMs?: number /** Cooperative timeout budget (ms) for `web_search`. Defaults to 30000. */ searchTimeoutMs?: number - /** Cap on one `web_fetch` output's characters (header, rendered body, and footer). Defaults to 200000. */ + /** Cap on source characters converted and complete `web_fetch` output characters. Defaults to 200000. */ fetchMaxOutputChars?: number } ``` -Source: [`packages/web/tool-web/src/index.ts:37`](../packages/web/tool-web/src/index.ts) +Source: [`packages/web/tool-web/src/index.ts:35`](../packages/web/tool-web/src/index.ts) ## `@deepseek-ai/dsh-tool-workflow` diff --git a/examples/acp-agent/web-fetch-fixture-server.mjs b/examples/acp-agent/web-fetch-fixture-server.mjs index 34a45fdd15..1667562faf 100644 --- a/examples/acp-agent/web-fetch-fixture-server.mjs +++ b/examples/acp-agent/web-fetch-fixture-server.mjs @@ -45,8 +45,11 @@ export async function apply(ctx) { }) // The fixture must never hold the process open past protocol shutdown. server.unref() - ctx.effect(() => () => { - server.close() - server.closeAllConnections() + ctx.effect(() => async () => { + await new Promise((resolve, reject) => { + server.close(error => error ? reject(error) : resolve(undefined)) + // Stop accepting first so a connection cannot arrive after the forced close. + server.closeAllConnections() + }) }, 'web-fetch-fixture-server') } diff --git a/packages/web/tool-web/README.i18n.yaml b/packages/web/tool-web/README.i18n.yaml index 44279a66be..58641384b9 100644 --- a/packages/web/tool-web/README.i18n.yaml +++ b/packages/web/tool-web/README.i18n.yaml @@ -1,6 +1,6 @@ # Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: -# pnpm run verify-translation-pairing --write -README.md: 44cb1ba2a2f4e1fba7e192d8b6645e0447ebf221 -README.zh.md: 35b390dd5407af16d84ab391dd8351f784c60035 +# pnpm run verify-translation-pairing --write packages/web/tool-web/README.md +README.md: 9b78920b1b6c611118294421dec1e75e381ed5d6 +README.zh.md: 2152c40f1ccac2272fa0b2681514a712417c0ad3 diff --git a/packages/web/tool-web/README.md b/packages/web/tool-web/README.md index 44cb1ba2a2..9b78920b1b 100644 --- a/packages/web/tool-web/README.md +++ b/packages/web/tool-web/README.md @@ -26,9 +26,9 @@ The normalized seam results are also the canonical tool values: `WebSearchResult | `searchMaxResults` | `8` | Upper bound on sources returned by one `web_search` call (the seam truncates a longer provider list and flags it). | | `fetchTimeoutMs` | `30000` | Cooperative tool-call timeout budget (ms) for `web_fetch`. | | `searchTimeoutMs` | `30000` | Cooperative tool-call timeout budget (ms) for `web_search`. | -| `fetchMaxOutputChars` | `200000` | Cap on one `web_fetch` output's characters — header, rendered body, and footer together; a cut body gets the truncation notice. | +| `fetchMaxOutputChars` | `200000` | Cap on source characters converted synchronously and on one complete `web_fetch` output (header, rendered body, and footer); a cut body gets the truncation notice when it fits. | -`fetchTimeoutMs`/`searchTimeoutMs` declare each tool's cooperative timeout budget (attached as `ToolDefinition.timeoutMs`), enforced by [`@deepseek-ai/dsh-timeout-policy`](../../timeout/timeout-policy/README.md); the model-facing schema exposes no timeout argument. `fetchMaxOutputChars` bounds the complete rendered output because markdown escaping can expand converted HTML past a provider's body cap (worst case ~2×); the default is 2× the local provider's default 100,000-character body cap, so it never cuts what that bound already admits. +`fetchTimeoutMs`/`searchTimeoutMs` declare each tool's cooperative timeout budget (attached as `ToolDefinition.timeoutMs`), enforced by [`@deepseek-ai/dsh-timeout-policy`](../../timeout/timeout-policy/README.md); the model-facing schema exposes no timeout argument. `fetchMaxOutputChars` bounds both synchronous conversion work and the complete rendered result: only that many source characters are converted, and the header, converted prefix, and truncation notice are then capped together. The default leaves headroom above the local provider's 100,000-character body cap, but rendered expansion can still make the final bound truncate the result. ```yaml - id: tool-web @@ -127,6 +127,6 @@ Append-only; newly visible content follows the reusable request prefix and does ## Known Limitations and Deferred Work -- **HTML→markdown conversion falls back to raw HTML on pathological input** — [turndown](https://github.com/mixmark-io/turndown) (with GFM tables/strikethrough) converts fetched HTML through a real DOM, but the synchronous walk is superlinear on deep unclosed nesting, so bodies nested past a fixed 512-level preflight bound pass through unconverted (as does anything that still makes turndown throw) rather than stalling the event loop or erroring ([Agent Note](../../../.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md)). +- **HTML→markdown conversion degrades on inputs GFM cannot safely represent** — [turndown](https://github.com/mixmark-io/turndown) (with GFM tables/strikethrough) converts at most `fetchMaxOutputChars` source characters through a real DOM. A conservative 512-level lexical guard passes deeply or ambiguously nested bodies through as raw HTML, conversion exceptions do the same, and table `colspan` is ignored because GFM has no spanning-cell representation; these bounds avoid blocking the event loop or expanding output from an untrusted numeric attribute ([Agent Note](../../../.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md)). - **The model-facing surface is minimal by design, with promotions deferred** — `max_results` stays a config bound (not a model argument), and `web_fetch` takes only `url` (no `format`/`prompt`/LLM-summarization mode); both are named later steps in [the seam Agent Note](../../../.agents/notes/implemented/architecture/2026-06-24-web-capability-seam.md). - **No web-specific permission policy** — both tools execute without requesting `ctx.approval`; a deployment that needs confirmation must add a `tools/pre-execute` policy, and the package does not define persistent URL/domain grants. diff --git a/packages/web/tool-web/README.zh.md b/packages/web/tool-web/README.zh.md index 35b390dd54..2152c40f1c 100644 --- a/packages/web/tool-web/README.zh.md +++ b/packages/web/tool-web/README.zh.md @@ -26,9 +26,9 @@ | `searchMaxResults` | `8` | 一次 `web_search` 调用返回的源数量上限（seam 截断更长的提供方列表并标记）。 | | `fetchTimeoutMs` | `30000` | `web_fetch` 的协作式工具调用超时预算（ms）。 | | `searchTimeoutMs` | `30000` | `web_search` 的协作式工具调用超时预算（ms）。 | -| `fetchMaxOutputChars` | `200000` | 单次 `web_fetch` 输出的字符上限——状态头、渲染后的主体与页脚合并计算；被截断的主体带截断提示。 | +| `fetchMaxOutputChars` | `200000` | 同步转换的源字符数与单次完整 `web_fetch` 输出的上限（状态头、渲染后的主体与页脚合并计算）；主体被截断时，在能容纳的情况下附带截断提示。 | -`fetchTimeoutMs`／`searchTimeoutMs` 声明每个工具的协作式超时预算（附加为 `ToolDefinition.timeoutMs`），由 [`@deepseek-ai/dsh-timeout-policy`](../../timeout/timeout-policy/README.md) 强制执行；面向模型的 schema 不公开超时参数。`fetchMaxOutputChars` 对完整渲染输出设上限：markdown 转义可能让转换后的 HTML 超出提供方的主体上限（最坏约 2 倍）；默认值取本地提供方默认 100,000 字符主体上限的 2 倍，因此绝不会削减该上限本已允许的内容。 +`fetchTimeoutMs`／`searchTimeoutMs` 声明每个工具的协作式超时预算（附加为 `ToolDefinition.timeoutMs`），由 [`@deepseek-ai/dsh-timeout-policy`](../../timeout/timeout-policy/README.md) 强制执行；面向模型的 schema 不公开超时参数。`fetchMaxOutputChars` 同时限制同步转换工作量和完整渲染结果：只转换至多该数量的源字符，随后对状态头、转换后的前缀和截断提示合并设限。默认值为本地提供方的 100,000 字符主体上限留出余量，但渲染膨胀仍可能使最终上限截断结果。 ```yaml - id: tool-web @@ -127,6 +127,6 @@ Use the web_fetch tool to retrieve the content of a specific HTTP(S) URL (for ex ## 已知限制与暂缓事项 -- **HTML→markdown 转换在病态输入上回退为原始 HTML**：[turndown](https://github.com/mixmark-io/turndown)（带 GFM 表格／删除线）通过真实 DOM 转换抓取到的 HTML，但同步遍历在深层未闭合嵌套上呈超线性，因此嵌套超过固定 512 层预检上限的主体不经转换原样通过（仍让 turndown 抛异常的输入同样如此），而非阻塞事件循环或报错（[决策记录](../../../.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md)）。 +- **HTML→markdown 转换会在 GFM 无法安全表示的输入上降级**：[turndown](https://github.com/mixmark-io/turndown)（带 GFM 表格／删除线）通过真实 DOM 转换至多 `fetchMaxOutputChars` 个源字符。保守的 512 层词法守卫会将深层或嵌套有歧义的主体作为原始 HTML 直接透传，转换异常也会如此处理；表格的 `colspan` 会被忽略，因为 GFM 无法表示跨列单元格。这些限制可避免阻塞事件循环，也避免不受信任的数值属性使输出膨胀（[决策记录](../../../.agents/notes/implemented/simplification/2026-07-26-turndown-for-tool-web-html-markdown.md)）。 - **面向模型的表层有意保持最小，提升项暂缓**：`max_results` 保持为配置上限（不是模型参数），`web_fetch` 只接受 `url`（没有 `format`／`prompt`／LLM 摘要模式）；两项都列为 [seam Agent Note](../../../.agents/notes/implemented/architecture/2026-06-24-web-capability-seam.md) 中的后续步骤。 - **没有 web 专用权限策略**：两个工具都不会请求 `ctx.approval` 就直接执行；需要确认的部署必须添加 `tools/pre-execute` 策略，该包不定义持久 URL／domain 授权。 diff --git a/packages/web/tool-web/src/fetch.ts b/packages/web/tool-web/src/fetch.ts index e909ea25be..75108f663c 100644 --- a/packages/web/tool-web/src/fetch.ts +++ b/packages/web/tool-web/src/fetch.ts @@ -30,6 +30,52 @@ const turndown = new TurndownService({ turndown.use(gfm) turndown.remove(['script', 'style', 'noscript']) +/** Render one GFM table cell without interpreting HTML span counts. */ +function renderTableCell(content: string, index: number): string { + const prefix = index === 0 ? '| ' : ' ' + const escaped = content.trim().replace(/\n\r/g, '
    ').replace(/\n/g, '
    ').replace(/\|+/g, '\\|').padEnd(3, ' ') + return `${prefix}${escaped} |` +} + +/** Whether a row is the table's Markdown heading row. */ +function isTableHeadingRow(row: HTMLTableRowElement): boolean { + const cells = Array.from(row.cells) + const section = row.parentElement as HTMLTableSectionElement + const table = section.parentElement as HTMLTableElement + return (section.nodeName === 'THEAD' || table.rows[0] === row) + && cells.every(cell => cell.nodeName === 'TH') +} + +/** Map an HTML table-cell alignment to the GFM separator marker. */ +function tableBorder(cell: HTMLTableCellElement): string { + const alignment = (cell.getAttribute('align') || cell.style.textAlign || '').toLowerCase() + if (alignment === 'left') return ':---' + if (alignment === 'right') return '---:' + if (alignment === 'center') return ':---:' + return '---' +} + +turndown.addRule('tableCellWithoutSpanExpansion', { + filter: ['th', 'td'], + replacement(content, node) { + const cell = node as HTMLTableCellElement + const row = cell.parentNode as HTMLTableRowElement + // GFM cannot represent spanning cells. Ignoring colspan keeps conversion + // work and output proportional to the source instead of the numeric attribute. + return renderTableCell(content, Array.prototype.indexOf.call(row.childNodes, cell)) + }, +}) +turndown.addRule('tableRowWithoutSpanExpansion', { + filter: 'tr', + replacement(content, node) { + const row = node as HTMLTableRowElement + const border = isTableHeadingRow(row) + ? Array.from(row.cells, (cell, index) => renderTableCell(tableBorder(cell), index)).join('') + : '' + return `\n${content}${border.length > 0 ? `\n${border}` : ''}` + }, +}) + /** * Validate value constraints the schema DSL can't express: a non-blank `url`. * Throws a plain `Error` otherwise. No timeout parameter — the tool-call budget @@ -55,63 +101,142 @@ export function parseFetchArgs(args: { url: string }): { url: string } { */ const MAX_CONVERSION_DEPTH = 512 -/** Elements that never take a closing tag, so they must not count toward nesting depth. */ +/** Elements that never take a closing tag, so they do not grow the lexical stack. */ const VOID_ELEMENTS = new Set([ 'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'link', 'meta', 'param', 'source', 'track', 'wbr', ]) +/** Elements whose contents HTML parses as text until their matching end tag. */ +const RAW_TEXT_ELEMENTS = new Set(['script', 'style', 'noscript']) + +/** Whether a character can occur after a raw-text end-tag name. */ +function isTagBoundary(char: string | undefined): boolean { + return char === undefined || char === '>' || char === '/' || /\s/.test(char) +} + +/** Find the matching raw-text end tag without interpreting markup-like body text. */ +function findRawTextEnd(lowerHtml: string, name: string, from: number): number { + const prefix = `` characters, and only accepts a closing + * tag for the current element; malformed input therefore over-counts rather + * than hiding nesting. * * @param html - the decoded HTML body. - * @returns the deepest open-element count the scan reaches. + * @returns whether the body crosses {@link MAX_CONVERSION_DEPTH}. */ -export function htmlNestingDepth(html: string): number { - let depth = 0 - let max = 0 - for (const tag of html.matchAll(/<(\/?)([a-zA-Z][a-zA-Z0-9-]*)[^>]*?(\/?)>/g)) { - const [, closing, rawName = '', selfClosing] = tag - const name = rawName.toLowerCase() - if (VOID_ELEMENTS.has(name) || selfClosing === '/') continue - if (closing === '/') { - if (depth > 0) depth -= 1 - } else { - depth += 1 - if (depth > max) max = depth +function exceedsConversionDepth(html: string): boolean { + const lowerHtml = html.toLowerCase() + const openElements: string[] = [] + let offset = 0 + let inComment = false + + while (offset < html.length) { + const start = html.indexOf('<', offset) + if (inComment) { + const end = html.indexOf('-->', offset) + if (end !== -1 && (start === -1 || end < start)) { + inComment = false + offset = end + 3 + continue + } } + if (start === -1) break + if (!inComment && html.startsWith(''.repeat(600) + 'x' + expect(formatFetchOutput({ + url: 'https://a.test', statusCode: 200, truncated: false, + body: { kind: 'html', content: pathological }, + }, NO_CAP)).toBe(`${HEADER}${pathological}`) + const abruptlyClosedComments = '
    '.repeat(600) + 'x' + expect(formatFetchOutput({ + url: 'https://a.test', statusCode: 200, truncated: false, + body: { kind: 'html', content: abruptlyClosedComments }, + }, NO_CAP)).toBe(`${HEADER}${abruptlyClosedComments}`) + }) + + it('the preflight accepts ordinary closed, void, self-closing, quoted, and raw-text markup', () => { + const paragraphs = '

    \'>x

    '.repeat(600) + const script = `` + expect(renderHtml(`<1bad>${paragraphs}${script}`)) + .not.toContain('x