From f1202d0dd883d7f7ee34814f86c4ea20bb1ced15 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Tue, 21 Jul 2026 20:15:09 +0800 Subject: [PATCH 01/49] ci: enforce bounded build lanes --- ...kage-invariant-runtime-contracts.i18n.yaml | 4 +- ...-19-package-invariant-runtime-contracts.md | 2 +- ...-package-invariant-runtime-contracts.zh.md | 2 +- .../2026-07-06-parallel-github-ci-gates.md | 39 ++-- .../2026-07-06-parallel-pre-push-gates.md | 8 +- .github/workflows/ci.yml | 192 +++++++++++++++--- .../skill-local/tests/skill-local.spec.ts | 3 + scripts/coverage-shards.spec.ts | 43 ++++ scripts/coverage-shards.ts | 68 +++++++ scripts/publint-all.spec.ts | 61 ++++++ scripts/publint-all.ts | 160 ++++++++++----- scripts/run-gates.ts | 61 ++++-- scripts/static-shards.spec.ts | 16 ++ scripts/static-shards.ts | 77 +++++++ scripts/verify-built-package-invariants.mjs | 136 +++++++------ .../verify-built-package-invariants.spec.ts | 88 ++++++++ 16 files changed, 769 insertions(+), 191 deletions(-) create mode 100644 scripts/coverage-shards.spec.ts create mode 100644 scripts/coverage-shards.ts create mode 100644 scripts/publint-all.spec.ts create mode 100644 scripts/static-shards.spec.ts create mode 100644 scripts/static-shards.ts create mode 100644 scripts/verify-built-package-invariants.spec.ts diff --git a/.agents/notes/implemented/architecture/2026-07-19-package-invariant-runtime-contracts.i18n.yaml b/.agents/notes/implemented/architecture/2026-07-19-package-invariant-runtime-contracts.i18n.yaml index 60f33a0e1e..0379a79e52 100644 --- a/.agents/notes/implemented/architecture/2026-07-19-package-invariant-runtime-contracts.i18n.yaml +++ b/.agents/notes/implemented/architecture/2026-07-19-package-invariant-runtime-contracts.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -2026-07-19-package-invariant-runtime-contracts.md: 7d1fb1ad5a2e7563bdddffde1f49368b9f0c13f7 -2026-07-19-package-invariant-runtime-contracts.zh.md: 669eb02221aea4b0654497bb81327d725648dabe +2026-07-19-package-invariant-runtime-contracts.md: 40d152b2320ac65f9ea7d8732b1a667236d2780a +2026-07-19-package-invariant-runtime-contracts.zh.md: bd2f440d5dce15b352e7bcea0d1243400d290f11 diff --git a/.agents/notes/implemented/architecture/2026-07-19-package-invariant-runtime-contracts.md b/.agents/notes/implemented/architecture/2026-07-19-package-invariant-runtime-contracts.md index 7d1fb1ad5a..40d152b232 100644 --- a/.agents/notes/implemented/architecture/2026-07-19-package-invariant-runtime-contracts.md +++ b/.agents/notes/implemented/architecture/2026-07-19-package-invariant-runtime-contracts.md @@ -59,7 +59,7 @@ Session-backed companions validate existing durable events when they load, using `verify-package-invariants` discovers every workspace package and enforces companion source, exact-name registration, named-only Loader shape, `./invariant` exports, publication files, dependencies, TypeScript references, and bundle entries. Its AST rule rejects generated markers, default exports, and unexplained empty installers. A non-empty installer must accept and use the failure reporter, and registration must pass that checked local `install` function. The gate deliberately does not infer semantic quality from method names or helper calls. -Vitest mounts `InvariantService` with `{ enabled: true }` for every package test topology and loads the owning companion. The invariant subpath path mapping resolves source companions instead of stale built output. Focused suites cover every executable companion's valid and invalid observations, and the exhaustive topology runs every source companion through the real Loader namespace normalization. An artifact gate stages each package's exact `npm pack` file inventory, imports its compiled `./invariant` self-reference under plain Node, and repeats that Loader-shape check, so an unpublished shared runtime chunk fails before release. Tests that synthesize event streams must produce a valid surrounding lifecycle unless the test is intentionally asserting a violation. +Vitest mounts `InvariantService` with `{ enabled: true }` for every package test topology and loads the owning companion. The invariant subpath path mapping resolves source companions instead of stale built output. Focused suites cover every executable companion's valid and invalid observations, and the exhaustive topology runs every source companion through the real Loader namespace normalization. After the structural gate validates each publication map, an artifact gate stages its manifest-declared `lib/` files, imports the compiled `./invariant` self-reference under plain Node, and repeats that Loader-shape check, so a companion that imports an undeclared runtime chunk fails before release. Tests that synthesize event streams must produce a valid surrounding lifecycle unless the test is intentionally asserting a violation. ## Alternatives considered diff --git a/.agents/notes/implemented/architecture/2026-07-19-package-invariant-runtime-contracts.zh.md b/.agents/notes/implemented/architecture/2026-07-19-package-invariant-runtime-contracts.zh.md index 669eb02221..bd2f440d5d 100644 --- a/.agents/notes/implemented/architecture/2026-07-19-package-invariant-runtime-contracts.zh.md +++ b/.agents/notes/implemented/architecture/2026-07-19-package-invariant-runtime-contracts.zh.md @@ -59,7 +59,7 @@ Status: implemented `verify-package-invariants` 发现每个 workspace 包，并强制 companion 源文件、完整名称注册、仅含具名 export 的 Loader 形状、`./invariant` export、发布文件、依赖、TypeScript reference 和 bundle entry 完整。其 AST 规则拒绝生成标记、默认导出和没有解释的空安装器。非空安装器必须接收并使用失败报告器，注册时还必须传入该经检查的本地 `install` 函数。门禁不会通过方法名或 helper 调用推断语义质量。 -Vitest 为每个包测试拓扑使用 `{ enabled: true }` 挂载 `InvariantService`，并加载所有者 companion。不变量 subpath 的 path mapping 会解析源 companion，而不是陈旧的构建输出。聚焦 suite 覆盖每个可执行 companion 的有效和无效观测；穷举拓扑通过真实 Loader 命名空间归一化运行每个源 companion。产物门禁会按每个包的精确 `npm pack` 文件清单暂存文件，在 plain Node 下导入该包已编译的 `./invariant` 自引用，并重复执行该 Loader 形状检查；这样，未发布的共享运行时分片会在正式发布前导致门禁失败。合成事件流的测试必须构造有效的外围生命周期，除非测试本身就是在断言违规。 +Vitest 为每个包测试拓扑使用 `{ enabled: true }` 挂载 `InvariantService`，并加载所有者 companion。不变量 subpath 的 path mapping 会解析源 companion，而不是陈旧的构建输出。聚焦 suite 覆盖每个可执行 companion 的有效和无效观测；穷举拓扑通过真实 Loader 命名空间归一化运行每个源 companion。结构门禁验证每个包的发布映射后，产物门禁会暂存其 manifest（元数据清单）声明的 `lib/` 文件，在 plain Node 下导入已编译的 `./invariant` 自引用，并重复执行该 Loader 形状检查；这样，若 companion 导入未声明的运行时分片，门禁就会在发布前失败。合成事件流的测试必须构造有效的外围生命周期，除非测试本身就是在断言违规。 ## 考虑过的替代方案 diff --git a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md index 65ad437ca9..0058a30660 100644 --- a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md +++ b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md @@ -4,36 +4,43 @@ Status: implemented ## Problem -The keyless GitHub CI gates are mostly orthogonal: typecheck, lint, documentation freshness, coverage, snapshot replay, build, package-publication hygiene, demo smoke, and built-bin smoke fail for different reasons and do not need each other's runtime state. Running them as one ordered command chain makes the workflow wall clock equal the sum of those gates, while splitting every leaf gate into its own GitHub job repeats checkout, Node setup, pnpm restore, and install work until orchestration overhead becomes the bottleneck. +The keyless GitHub CI gates are mostly orthogonal: typecheck, lint, documentation freshness, coverage, snapshot replay, build, package-publication hygiene, demo smoke, and built-bin smoke fail for different reasons and do not need each other's runtime state. Running them as one ordered command chain makes the workflow wall clock equal the sum of those gates, while splitting every short leaf into its own GitHub job repeats checkout, Node setup, pnpm restore, and install work until orchestration overhead becomes the bottleneck. -The hard part is the artifact boundary. `publint`, `verify-node-next-types`, and built-bin smoke tests need the built `lib/` outputs, while most gates only need source and dependencies. A blind fan-out either races those artifact consumers before `pnpm run build` has emitted declarations and bundles, or repeats the build in every artifact-dependent job. +The original broad-lane split stopped meeting that balance as the workspace grew. On the merge of PR #404, Linux static, coverage, snapshot, and artifact jobs took 148, 195, 94, and 230 seconds; Windows static and artifacts took 251 and 482 seconds. Package-manager packing once per package dominated both artifact validators, coverage needlessly rebuilt output before a source-only suite, and CPU-heavy gates contended inside the static and coverage lanes. + +The artifact boundary remains load-bearing. `publint`, `verify-node-next-types`, compiled invariant loading, and built-bin smoke tests need emitted `lib/` output. Sharding cannot race those consumers ahead of build or replace their published-artifact signal with source execution. ## Decision -[CI](../../../../.github/workflows/ci.yml) groups keyless checks into broad primary-runtime lanes plus a compatibility matrix. The workflow file owns the current lane and runtime inventory. +[CI](../../../../.github/workflows/ci.yml) bounds every non-Windows job to one minute and every Windows job to three minutes. The timeout is an executable regression ceiling; the lane design leaves headroom below it rather than treating a timeout as normal control flow. -Each lane delegates to [scripts/run-gates.ts](../../../../scripts/run-gates.ts), which schedules independent gates with bounded concurrency and prints an attributable result block for each one. Artifact consumers depend on one build within their lane, while compatibility jobs combine typechecking with a real unbuilt worker launch to cover runtime-specific loader behavior. +[scripts/run-gates.ts](../../../../scripts/run-gates.ts) remains the common bounded scheduler, but GitHub supplies explicit shard names for the expensive gate families. [scripts/static-shards.ts](../../../../scripts/static-shards.ts) partitions static gates into foundation, API-contract, catalog, prose, and documentation-site lanes and rejects a missing or duplicate gate assignment. [scripts/coverage-shards.ts](../../../../scripts/coverage-shards.ts) assigns every workspace package to exactly one source-coverage lane; its test expands the live package tree, so a new package makes CI red until it has an owner. Each coverage lane includes only its owned source files, repeats the exhaustive companion topology test, and runs without a preceding build because the complete coverage suite passes from a tree with every generated `lib/` removed. -Generated `.sessions/` logs and `.doc-typecheck-*` temp directories are ignored by lint. The aggregate local CI mode still runs demo smoke after lint, while the split GitHub static lane can run demo smoke directly because lint is isolated in its own lane. +Snapshot replay is four Vitest file shards. Each snapshot job builds the shipped runtime while its Linux runner installs bubblewrap, then runs only its assigned replay files. Static, coverage, and snapshot sharding changes only GitHub scheduling: the ordinary local package scripts still run their complete suites. -Build output is produced once inside the Node 24 artifact lane. The artifact consumers (`publint`, `verify-node-next-types`, and built-bin smoke) declare a dependency on `build`, so there is no upload/download handoff and no consumer can race ahead of declarations or bundles. The CI coverage reporter is text-only while local coverage keeps the HTML report. +Artifacts use three lanes: one metadata lane for `publint`, NodeNext declarations, and compiled invariant loading, plus two Vitest shards for built-bin smoke. Each lane produces its own build before its consumers. Repeating the short build costs runner minutes but avoids an upload/download dependency and keeps each job's critical path bounded. -Both workflows cache the pnpm store. The real-API workflow uses the shared bounded Vitest file pool rather than a separate job per test group. +[scripts/publint-all.ts](../../../../scripts/publint-all.ts) calls publint's supported API in-process against an in-memory publication view made from each manifest's declared files and npm's mandatory metadata files. This preserves the distinction between workspace files and published files without spawning a package-manager pack command 103 times. [scripts/verify-built-package-invariants.mjs](../../../../scripts/verify-built-package-invariants.mjs) stages those structurally validated manifest-declared `lib/` files below the real package, then imports the compiled self-reference through plain Node and Cordis Loader normalization. A companion that reaches an undeclared runtime chunk still fails. + +Compatibility lanes run the source worker and Zstandard runtime smokes on every advertised Node line. TypeScript checks the source graph once on the primary Node 24 lane; repeating the same compiler analysis on Node 22 and 26 added time without runtime-specific signal. + +The workflow caches the pnpm store, preserves native PowerShell for Windows measurements, and retains one aggregate `all checks passed` status for branch protection. Windows build remains blocking; the wider Windows static, lint, and artifact matrix remains observational while carrying the same three-minute ceiling. ## Alternatives considered -- **Keep the full serial chain in a Node matrix** - simplest to reason about, but it duplicates repo-wide gates that do not produce Node-version-specific signal and leaves every PR waiting for the sum of all gates. -- **Run every gate as a separate GitHub job** - maximizes GitHub-visible fan-out, but it creates too many checks and pays repeated setup/install overhead for gates whose runtime is shorter than the runner preparation. -- **Upload build artifacts to artifact-dependent jobs** - preserves correctness across many jobs, but it adds artifact upload/download time and keeps the workflow wide when the artifact consumers can run behind a local dependency in the primary job. -- **Run `typecheck` and `build` concurrently** - exposes more work to the scheduler, but both commands invoke `tsc -b`; sharing incremental build state between them is a needless race for a small wall-clock gain. -- **Use unbounded real-API e2e parallelism** - rejected because the suite includes many live model/tool scenarios; the worker pool needs an explicit `DSH_E2E_MAX_WORKERS` cap so CI and local runs can fan out without hiding quota or resource problems behind flaky rate-limit failures. +- **Keep the broad lanes and raise timeouts** - minimizes workflow YAML, but it preserves the measured multi-minute feedback loop and offers no regression budget. +- **Run every leaf gate as a separate GitHub job** - maximizes fan-out, but short generators and prose checks would spend more time preparing a runner than checking the repository. +- **Upload one build to artifact consumers** - avoids repeated compilation, but upload/download and dependency scheduling lengthen wall time; the clean build is short enough to repeat inside bounded lanes. +- **Keep package-manager packing in both publication gates** - delegates inventory selection to pnpm, but repeats more than 200 package-manager processes. The manifest structural gate plus publication-view fixtures make the optimized inventory contract explicit and fail on an on-disk but unpublished dependency. +- **Keep build before coverage** - provides emitted output the source suite no longer consumes; a clean-tree coverage proof showed it was pure latency. +- **Typecheck on every Node version** - repeats compiler work while the compatibility smokes already exercise actual Node-specific loading and compression behavior. ## Consequences -PR feedback arrives as a few GitHub checks with structured per-gate log blocks inside each broad job. That keeps runner setup overhead bounded and the Actions UI compact, at the cost of losing one status check per leaf gate. +The Actions UI contains more matrix checks and total runner time can exceed a serial workflow, but PR wall time is the slowest bounded lane instead of the sum of unrelated work. Repeated setup and builds are the deliberate price of sub-minute non-Windows feedback and sub-three-minute Windows feedback. -The broad-lane split repeats checkout, setup, and install more often than a single primary job. That setup cost is intentional: on GitHub's hosted runner, running lint, coverage, and snapshot replay in one process pool oversubscribes CPU badly enough that the single-job critical path is longer than the repeated setup. +Shard inventories are repository contracts. Static selection validates the complete live gate list at runtime, coverage tests validate exhaustive package ownership, and Vitest owns deterministic file sharding for snapshots and built-bin smokes. Adding a gate or package therefore requires an explicit scheduling decision instead of silently lengthening an existing lane. -The split introduces a maintenance obligation: when `package.json` adds or removes a gate that belongs in CI, [scripts/run-gates.ts](../../../../scripts/run-gates.ts) needs the matching leaf. That obligation is intentional because the runner is the parallel execution plan for the same gate vocabulary, not a separate quality policy. +The optimized publication validators rely on the manifest `files` contract enforced by `verify-package-invariants`. If publication rules grow beyond that contract, the structural gate and both staged views must change together. -The compatibility signal is narrower than the primary Node 24 signal. It proves that the source graph typechecks and that the real unbuilt workflow-worker launch path executes on every advertised runtime line without doubling documentation, coverage, publication, snapshot replay, and unrelated smoke checks whose failures are not expected to vary by Node version. +Compatibility jobs no longer claim that TypeScript itself was exercised under every Node runtime. They prove runtime-sensitive source loading on Node 22, 24, and 26, while the primary runtime owns the single source-graph typecheck. diff --git a/.agents/notes/implemented/process/2026-07-06-parallel-pre-push-gates.md b/.agents/notes/implemented/process/2026-07-06-parallel-pre-push-gates.md index 60472e51ec..1e7639ad1f 100644 --- a/.agents/notes/implemented/process/2026-07-06-parallel-pre-push-gates.md +++ b/.agents/notes/implemented/process/2026-07-06-parallel-pre-push-gates.md @@ -8,7 +8,7 @@ The pre-push hook is the last local checkpoint before a branch leaves the machin Flattening those members directly into `lefthook.yml` solves the local hook only. CI has the same scheduling problem, and duplicating a long leaf list in YAML gives future script changes two places to drift. -`publint` has the same shape one level lower. Each package is linted independently against its own manifest and built output, but the runner loops through every package in order. On this repo that makes one package-publication gate consume time proportional to the number of packages even though the checks do not share mutable state. +`publint` has the same shape one level lower. Each package is linted independently against its own manifest and built output, but invoking the CLI separately also asks the package manager to compute the same manifest-bounded publication view once per package. On this repo process and packing overhead dominate the publication checks. ## Decision @@ -18,7 +18,7 @@ The `pre-push` mode expands into leaf gates for the unit suite, snapshot suite, The build gate makes the hook self-contained from a clean worktree. `publint`, `verify-node-next-types`, and the pre-push form of `doc-typecheck` wait for that build output, while source-only gates continue in parallel. -[scripts/publint-all.ts](../../../../scripts/publint-all.ts) discovers the package list from `packages//` and runs `publint` with a worker pool sized from `availableParallelism()`. `DSH_PUBLINT_CONCURRENCY` can cap or raise the worker count for local machines and CI runners with different resource profiles. Results are buffered per package and printed in deterministic package order, so parallel execution does not scramble each package's log block. +[scripts/publint-all.ts](../../../../scripts/publint-all.ts) discovers the package list from `packages//` and calls publint's supported API against an in-memory view of each manifest's declared publication files plus npm's mandatory metadata files. That keeps unpublished workspace files invisible to publint without a package-manager subprocess per package. A worker pool sized from `availableParallelism()` bounds parallel file loading and linting; `DSH_PUBLINT_CONCURRENCY` can cap or raise it, and results print in deterministic package order. The aggregate package scripts remain the source of truth for ad hoc local runs. The scheduler is a parallel execution plan over their member gates, not a replacement vocabulary. @@ -29,7 +29,7 @@ The aggregate package scripts remain the source of truth for ad hoc local runs. - **Require developers to build before pushing** - avoids one hook gate, but it makes `publint` fail in a clean worktree and turns the final local checkpoint into a convention instead of a runnable check. - **Background subcommands inside shell scripts** - can parallelize work, but it loses lefthook's job names, per-job timing, and failure grouping, and makes signal handling harder to reason about. - **Declare one publint lefthook job per package** - exposes maximum parallelism, but it turns the hook into a hand-maintained package inventory that drifts exactly when new packages are added. -- **Run publint with unbounded concurrency** - minimizes elapsed time on small machines only by gambling with process count, memory pressure, package tarball creation, and readable logs. +- **Run publint with unbounded concurrency** - minimizes elapsed time on small machines only by gambling with file descriptors, memory pressure, and readable logs. ## Consequences @@ -37,4 +37,4 @@ The hook's critical path becomes the slowest real gate instead of the sum of hid The hook file stays short, and the duplicated member list lives in [scripts/run-gates.ts](../../../../scripts/run-gates.ts), where CI and pre-push can share it. The cost is a custom scheduler script instead of pure lefthook configuration, plus a build in the local pre-push path. -`publint-all.ts` becomes asynchronous code and buffers command output instead of inheriting stdio live. The payoff is package-level parallelism with stable output order and one environment variable for resource tuning. +`publint-all.ts` becomes asynchronous code and formats API results after each package completes. The payoff is package-level parallelism with stable output order, one environment variable for resource tuning, and no repeated package-manager packing. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bac25b07cb..9be2932c11 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,45 +19,128 @@ jobs: node-24: runs-on: ubuntu-latest name: node 24 / ${{ matrix.lane }} + timeout-minutes: 1 env: DSH_GATE_CONCURRENCY: ${{ matrix.gate_concurrency }} DSH_PUBLINT_CONCURRENCY: ${{ matrix.publint_concurrency }} + DSH_ARTIFACT_SHARD: ${{ matrix.artifact_shard }} DSH_COVERAGE_MAX_WORKERS: ${{ matrix.coverage_max_workers }} + DSH_COVERAGE_SHARD: ${{ matrix.coverage_shard }} + DSH_STATIC_SHARD: ${{ matrix.static_shard }} + DSH_SNAPSHOT_PREBUILT: ${{ matrix.snapshot_prebuilt }} + DSH_SNAPSHOT_SHARD: ${{ matrix.snapshot_shard }} DSH_ESLINT_CACHE: ${{ matrix.eslint_cache }} strategy: fail-fast: false matrix: include: - - lane: static + - lane: static-foundation command: pnpm run check:ci:static gate_concurrency: '4' - publint_concurrency: '8' - coverage_max_workers: '' - eslint_cache: '' + static_shard: foundation + - lane: static-api-contracts + command: pnpm run check:ci:static + gate_concurrency: '4' + static_shard: api-contracts + - lane: static-catalogs + command: pnpm run check:ci:static + gate_concurrency: '4' + static_shard: catalogs + - lane: static-prose + command: pnpm run check:ci:static + gate_concurrency: '4' + static_shard: prose + - lane: static-site + command: pnpm run check:ci:static + gate_concurrency: '1' + static_shard: site - lane: lint command: pnpm run check:ci:lint gate_concurrency: '1' - publint_concurrency: '8' - coverage_max_workers: '' eslint_cache: '1' - - lane: coverage + - lane: coverage-spine command: pnpm run check:ci:coverage gate_concurrency: '1' - publint_concurrency: '8' coverage_max_workers: '4' - eslint_cache: '' - - lane: snapshot + coverage_shard: spine + - lane: coverage-sdk + command: pnpm run check:ci:coverage + gate_concurrency: '1' + coverage_max_workers: '4' + coverage_shard: sdk + - lane: coverage-interfaces + command: pnpm run check:ci:coverage + gate_concurrency: '1' + coverage_max_workers: '4' + coverage_shard: interfaces + - lane: coverage-execution + command: pnpm run check:ci:coverage + gate_concurrency: '1' + coverage_max_workers: '4' + coverage_shard: execution + - lane: coverage-orchestration + command: pnpm run check:ci:coverage + gate_concurrency: '1' + coverage_max_workers: '4' + coverage_shard: orchestration + - lane: coverage-infrastructure + command: pnpm run check:ci:coverage + gate_concurrency: '1' + coverage_max_workers: '4' + coverage_shard: infrastructure + - lane: coverage-session-state + command: pnpm run check:ci:coverage + gate_concurrency: '1' + coverage_max_workers: '4' + coverage_shard: session-state + - lane: coverage-hooks-claude + command: pnpm run check:ci:coverage + gate_concurrency: '1' + coverage_max_workers: '4' + coverage_shard: hooks-claude + - lane: coverage-hooks-codex + command: pnpm run check:ci:coverage + gate_concurrency: '1' + coverage_max_workers: '4' + coverage_shard: hooks-codex + - lane: coverage-capabilities + command: pnpm run check:ci:coverage + gate_concurrency: '1' + coverage_max_workers: '4' + coverage_shard: capabilities + - lane: snapshot-1 command: pnpm run check:ci:snapshot gate_concurrency: '1' - publint_concurrency: '8' - coverage_max_workers: '' - eslint_cache: '' - - lane: artifacts + snapshot_prebuilt: '1' + snapshot_shard: '1/4' + - lane: snapshot-2 + command: pnpm run check:ci:snapshot + gate_concurrency: '1' + snapshot_prebuilt: '1' + snapshot_shard: '2/4' + - lane: snapshot-3 + command: pnpm run check:ci:snapshot + gate_concurrency: '1' + snapshot_prebuilt: '1' + snapshot_shard: '3/4' + - lane: snapshot-4 + command: pnpm run check:ci:snapshot + gate_concurrency: '1' + snapshot_prebuilt: '1' + snapshot_shard: '4/4' + - lane: artifacts-metadata command: pnpm run check:ci:artifacts gate_concurrency: '3' publint_concurrency: '8' - coverage_max_workers: '' - eslint_cache: '' + artifact_shard: metadata + - lane: artifacts-smoke-1 + command: pnpm run check:ci:artifacts + gate_concurrency: '1' + artifact_shard: smoke-1 + - lane: artifacts-smoke-2 + command: pnpm run check:ci:artifacts + gate_concurrency: '1' + artifact_shard: smoke-2 steps: - uses: actions/checkout@v6 @@ -82,19 +165,30 @@ jobs: - name: Install (immutable) run: pnpm install --frozen-lockfile - # The snapshot lane REPLAYS the sandbox example's recorded scenarios, + # The snapshot lanes REPLAY the sandbox example's recorded scenarios, # re-executing their bash calls under a real runner. ubuntu-latest has # no bubblewrap preinstalled and no built Landlock launcher, so without # this the confined executions fail closed (SANDBOX_UNAVAILABLE). Same # install as sandbox.yml's bwrap leg (incl. the Ubuntu 24.04 AppArmor - # userns knob). - - name: Install bubblewrap (unrestrict userns) - if: matrix.lane == 'snapshot' + # userns knob). Building does not depend on bubblewrap, so overlap them. + - name: Prepare built snapshot runtime and bubblewrap + if: startsWith(matrix.lane, 'snapshot-') run: | - sudo apt-get update -q - sudo apt-get install -yq bubblewrap - sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 \ - || echo "apparmor userns knob absent — the functional probe decides" + pnpm run build & + build_pid=$! + ( + sudo apt-get update -q + sudo apt-get install -yq bubblewrap + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 \ + || echo "apparmor userns knob absent — the functional probe decides" + ) & + sandbox_pid=$! + build_status=0 + sandbox_status=0 + wait "$build_pid" || build_status=$? + wait "$sandbox_pid" || sandbox_status=$? + if (( build_status != 0 )); then exit "$build_status"; fi + exit "$sandbox_status" - uses: actions/cache@v4 if: matrix.lane == 'lint' @@ -110,12 +204,20 @@ jobs: node-compat: runs-on: ubuntu-latest name: node ${{ matrix.node }} + timeout-minutes: 1 env: DSH_GATE_CONCURRENCY: '2' + DSH_NODE_COMPAT_SKIP_TYPECHECK: ${{ matrix.skip_typecheck }} strategy: fail-fast: false matrix: - node: ['22.19', 24, 26] + include: + - node: '22.19' + skip_typecheck: '1' + - node: 24 + skip_typecheck: '' + - node: 26 + skip_typecheck: '1' steps: - uses: actions/checkout@v6 @@ -146,6 +248,7 @@ jobs: python-sdk: runs-on: ubuntu-latest name: python 3.10 / keyless SDK + timeout-minutes: 1 steps: - uses: actions/checkout@v6 @@ -166,6 +269,7 @@ jobs: windows-build: runs-on: windows-2025 name: windows / build + timeout-minutes: 3 steps: - uses: actions/checkout@v6 @@ -191,29 +295,54 @@ jobs: continue-on-error: true runs-on: windows-2025 name: windows node 24 / ${{ matrix.lane }} + timeout-minutes: 3 env: DSH_GATE_CONCURRENCY: ${{ matrix.gate_concurrency }} DSH_PUBLINT_CONCURRENCY: ${{ matrix.publint_concurrency }} + DSH_ARTIFACT_SHARD: ${{ matrix.artifact_shard }} + DSH_STATIC_SHARD: ${{ matrix.static_shard }} DSH_ESLINT_CACHE: ${{ matrix.eslint_cache }} strategy: fail-fast: false matrix: include: - - lane: static + - lane: static-foundation command: pnpm run check:ci:static gate_concurrency: '4' - publint_concurrency: '8' - eslint_cache: '' + static_shard: foundation + - lane: static-api-contracts + command: pnpm run check:ci:static + gate_concurrency: '4' + static_shard: api-contracts + - lane: static-catalogs + command: pnpm run check:ci:static + gate_concurrency: '4' + static_shard: catalogs + - lane: static-prose + command: pnpm run check:ci:static + gate_concurrency: '4' + static_shard: prose + - lane: static-site + command: pnpm run check:ci:static + gate_concurrency: '1' + static_shard: site - lane: lint command: pnpm run check:ci:lint gate_concurrency: '1' - publint_concurrency: '8' eslint_cache: '1' - - lane: artifacts + - lane: artifacts-metadata command: pnpm run check:ci:artifacts gate_concurrency: '3' publint_concurrency: '8' - eslint_cache: '' + artifact_shard: metadata + - lane: artifacts-smoke-1 + command: pnpm run check:ci:artifacts + gate_concurrency: '1' + artifact_shard: smoke-1 + - lane: artifacts-smoke-2 + command: pnpm run check:ci:artifacts + gate_concurrency: '1' + artifact_shard: smoke-2 steps: - uses: actions/checkout@v6 @@ -271,6 +400,7 @@ jobs: all-checks-passed: name: all checks passed runs-on: ubuntu-latest + timeout-minutes: 1 needs: [node-24, node-compat, python-sdk, windows-build] if: always() steps: diff --git a/packages/skill/skill-local/tests/skill-local.spec.ts b/packages/skill/skill-local/tests/skill-local.spec.ts index 43cc70c7ad..0c7d473b13 100644 --- a/packages/skill/skill-local/tests/skill-local.spec.ts +++ b/packages/skill/skill-local/tests/skill-local.spec.ts @@ -391,6 +391,9 @@ describe('LocalSkillProvider', () => { await empty.plugin(SkillService) SkillLocal.apply(empty, {}) expect(await empty.skills.list()).toEqual([]) + + delete process.env.DSH_AGENTS_HOME + expect(new SkillLocal.LocalSkillProvider(empty, { dshHome: join(envHome, 'empty-dsh') }).name).toBe('local') } finally { if (previousDshHome === undefined) { delete process.env.DSH_HOME diff --git a/scripts/coverage-shards.spec.ts b/scripts/coverage-shards.spec.ts new file mode 100644 index 0000000000..8db6ac1aba --- /dev/null +++ b/scripts/coverage-shards.spec.ts @@ -0,0 +1,43 @@ +import { readdirSync } from 'node:fs' +import { resolve } from 'node:path' +import { describe, expect, it } from 'vitest' +import { coverageArgs, coverageShards } from './coverage-shards.ts' + +const repositoryRoot = resolve(import.meta.dirname, '..') + +describe('coverage shards', () => { + it('assigns every workspace package to exactly one lane', () => { + const packagesRoot = resolve(repositoryRoot, 'packages') + const workspacePackages = readdirSync(packagesRoot, { withFileTypes: true }) + .filter(group => group.isDirectory()) + .flatMap(group => readdirSync(resolve(packagesRoot, group.name), { withFileTypes: true }) + .filter(entry => entry.isDirectory()) + .map(entry => `${group.name}/${entry.name}`)) + .sort() + const assignedPackages = coverageShards.flatMap(shard => shard.packageRoots.flatMap((packageRoot) => { + if (packageRoot.includes('/')) return [packageRoot] + return readdirSync(resolve(packagesRoot, packageRoot), { withFileTypes: true }) + .filter(entry => entry.isDirectory()) + .map(entry => `${packageRoot}/${entry.name}`) + })) + + expect([...assignedPackages].sort()).toEqual(workspacePackages) + expect(new Set(assignedPackages).size).toBe(assignedPackages.length) + }) + + it.each(coverageShards)('selects tests and source includes for $name', (shard) => { + const args = coverageArgs(shard.name) + for (const packageRoot of shard.packageRoots) { + expect(args).toContain(`packages/${packageRoot}`) + expect(args).toContain(packageRoot.includes('/') + ? `--coverage.include=packages/${packageRoot}/src/**/*.ts` + : `--coverage.include=packages/${packageRoot}/*/src/**/*.ts`) + } + expect(args).toContain('scripts/test-invariants.spec.ts') + expect(new Set(args).size).toBe(args.length) + }) + + it('rejects an unknown lane', () => { + expect(() => coverageArgs('missing')).toThrow('unknown DSH_COVERAGE_SHARD') + }) +}) diff --git a/scripts/coverage-shards.ts b/scripts/coverage-shards.ts new file mode 100644 index 0000000000..18c360401c --- /dev/null +++ b/scripts/coverage-shards.ts @@ -0,0 +1,68 @@ +/** Coverage shard definitions for the GitHub Actions source-test lanes. */ + +/** A coverage lane that owns complete package roots and optional cross-package tests. */ +export interface CoverageShard { + /** Stable lane identifier passed through `DSH_COVERAGE_SHARD`. */ + name: string + /** Group or package paths below `packages/` whose tests and source coverage belong to the lane. */ + packageRoots: readonly string[] + /** Additional test roots needed for cross-package behavior or repository scripts. */ + extraTestRoots?: readonly string[] +} + +/** Exhaustive, non-overlapping ownership of workspace packages in coverage CI. */ +export const coverageShards = [ + { + name: 'spine', + packageRoots: ['core', 'llm', 'compact', 'context'], + extraTestRoots: ['packages/examples/cli-demo/tests'], + }, + { name: 'sdk', packageRoots: ['sdk'] }, + { + name: 'interfaces', + packageRoots: ['ui', 'examples', 'goal'], + extraTestRoots: ['examples'], + }, + { name: 'execution', packageRoots: ['fs', 'bash', 'sandbox', 'code-runtime'] }, + { name: 'orchestration', packageRoots: ['workflow', 'subagent', 'tasks'] }, + { + name: 'infrastructure', + packageRoots: ['cordis', 'support', 'lsp', 'mcp'], + extraTestRoots: ['scripts'], + }, + { + name: 'session-state', + packageRoots: ['session-persistence', 'session-query'], + }, + { name: 'hooks-claude', packageRoots: ['hooks/hook-protocol', 'hooks/hooks-claude'] }, + { name: 'hooks-codex', packageRoots: ['hooks/hooks-codex'] }, + { + name: 'capabilities', + packageRoots: ['web', 'skill', 'spill', 'util', 'guard', 'todo', 'timeout'], + }, +] as const satisfies readonly CoverageShard[] + +/** + * Build Vitest filters and coverage include globs for one source-test lane. + * + * @param name Stable shard name from {@link coverageShards}. + * @returns Positional test roots followed by per-group coverage include flags. + */ +export function coverageArgs(name: string): string[] { + const shard = coverageShards.find(candidate => candidate.name === name) + if (shard === undefined) { + throw new Error(`run-gates: unknown DSH_COVERAGE_SHARD ${JSON.stringify(name)}.`) + } + + const testRoots = new Set([ + ...shard.packageRoots.map(packageRoot => `packages/${packageRoot}`), + ...('extraTestRoots' in shard ? shard.extraTestRoots : []), + 'scripts/test-invariants.spec.ts', + ]) + return [ + ...testRoots, + ...shard.packageRoots.map(packageRoot => packageRoot.includes('/') + ? `--coverage.include=packages/${packageRoot}/src/**/*.ts` + : `--coverage.include=packages/${packageRoot}/*/src/**/*.ts`), + ] +} diff --git a/scripts/publint-all.spec.ts b/scripts/publint-all.spec.ts new file mode 100644 index 0000000000..22dd80d6b0 --- /dev/null +++ b/scripts/publint-all.spec.ts @@ -0,0 +1,61 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { spawnSync } from 'node:child_process' +import { afterEach, describe, expect, it } from 'vitest' + +const repositoryRoot = fileURLToPath(new URL('..', import.meta.url)) +const runner = fileURLToPath(new URL('./publint-all.ts', import.meta.url)) +const roots: string[] = [] + +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }) +}) + +function fixture(exportPath = './lib/index.js'): string { + const root = mkdtempSync(join(tmpdir(), 'dsh-publint-all-')) + roots.push(root) + const packageDir = join(root, 'packages/core/probe') + mkdirSync(join(packageDir, 'lib'), { recursive: true }) + writeFileSync(join(packageDir, 'package.json'), `${JSON.stringify({ + name: '@deepseek-ai/dsh-probe', + version: '0.0.1', + type: 'module', + license: 'MIT', + engines: { node: '>=22.19' }, + sideEffects: false, + files: ['lib'], + exports: { '.': { default: exportPath } }, + }, null, 2)}\n`) + writeFileSync(join(packageDir, 'README.md'), '# Probe\n') + writeFileSync(join(packageDir, 'lib/index.js'), 'export const probe = true\n') + writeFileSync(join(packageDir, 'unpublished.js'), 'export const hidden = true\n') + return root +} + +function run(root: string) { + return spawnSync(process.execPath, [ + '--import', 'tsx', runner, + '--packages-root', root, + ], { + cwd: repositoryRoot, + encoding: 'utf8', + timeout: 5_000, + }) +} + +describe('publint package runner', () => { + it('lints recursively declared files from an in-memory publication view', () => { + const result = run(fixture()) + expect(result.status, result.stderr).toBe(0) + expect(result.stdout).toContain('linting 1 package(s)') + expect(result.stdout).toContain('All good!') + }) + + it('rejects an export that exists in the workspace but is not published', () => { + const result = run(fixture('./unpublished.js')) + expect(result.status).toBe(1) + expect(result.stdout).toContain('unpublished.js') + }) +}) diff --git a/scripts/publint-all.ts b/scripts/publint-all.ts index 0911316f18..2ed1906763 100644 --- a/scripts/publint-all.ts +++ b/scripts/publint-all.ts @@ -1,46 +1,53 @@ -import { execFile } from 'node:child_process' -import { existsSync, readdirSync } from 'node:fs' +/** Run publint over the exact manifest-declared publication view of every package. */ + +import { + globSync, + readFileSync, + readdirSync, + statSync, +} from 'node:fs' import { availableParallelism } from 'node:os' -import { resolve } from 'node:path' -import { promisify } from 'node:util' +import { dirname, relative, resolve, sep } from 'node:path' +import { publint, type Message, type PackFile } from 'publint' +import { formatMessage } from 'publint/utils' -const execFileAsync = promisify(execFile) const CONCURRENCY_ENV = 'DSH_PUBLINT_CONCURRENCY' +const repositoryRoot = resolve(import.meta.dirname, '..') +const options = parseOptions(process.argv.slice(2)) +const packagesRoot = resolve(options.get('--packages-root') ?? repositoryRoot) -// Discover harness packages at packages//; group containers, -// examples, and private vendored sources are not package targets. -const root = resolve(import.meta.dirname, '..') -const packagesRoot = resolve(root, 'packages') +interface PackageTarget { + path: string + directory: string + manifest: PackageManifest +} -// Run publint's JS CLI through the current node, not the .bin shim: the -// extensionless shim isn't spawnable on Windows (CVE-2024-27980) and the .cmd -// variant needs shell:true, which space-joins args UNESCAPED (DEP0190) and -// breaks when the repo path contains spaces. The JS entry is identical on every -// platform (`bin` is `./src/cli.js` per publint's package.json). -const publintCli = resolve(root, 'node_modules/publint/src/cli.js') +interface PackageManifest { + name?: string + files?: unknown +} type PublintResult = - | { path: string; status: 'passed'; stdout: string; stderr: string } - | { path: string; status: 'failed'; stdout: string; stderr: string; message: string } + | { path: string; status: 'passed'; messages: Message[]; manifest: Record } + | { path: string; status: 'failed'; messages: Message[]; manifest: Record; failure?: string } -function workspacePackages(): string[] { - return readdirSync(packagesRoot, { withFileTypes: true }) - .filter(group => group.isDirectory()) - .flatMap(group => - readdirSync(resolve(packagesRoot, group.name), { withFileTypes: true }) - .filter(pkg => pkg.isDirectory()) - .filter(pkg => existsSync(resolve(packagesRoot, group.name, pkg.name, 'package.json'))) - .map(pkg => `packages/${group.name}/${pkg.name}`), - ) +function workspacePackages(): PackageTarget[] { + return globSync('packages/*/*/package.json', { cwd: packagesRoot }) + .sort() + .map((manifestPath) => { + const absoluteManifestPath = resolve(packagesRoot, manifestPath) + const manifest = JSON.parse(readFileSync(absoluteManifestPath, 'utf8')) as PackageManifest + return { path: dirname(manifestPath), directory: dirname(absoluteManifestPath), manifest } + }) } function publintConcurrency(total: number): number { if (total === 0) return 0 const raw = process.env[CONCURRENCY_ENV] - if (raw !== undefined) { + if (raw !== undefined && raw !== '') { const parsed = Number.parseInt(raw, 10) - if (!Number.isSafeInteger(parsed) || parsed < 1) { + if (!Number.isSafeInteger(parsed) || parsed < 1 || String(parsed) !== raw) { throw new Error(`publint-all: ${CONCURRENCY_ENV} must be a positive integer, got ${JSON.stringify(raw)}.`) } return Math.min(total, parsed) @@ -49,57 +56,106 @@ function publintConcurrency(total: number): number { return Math.min(total, availableParallelism()) } -function outputText(value: unknown): string { - if (typeof value === 'string') return value - if (Buffer.isBuffer(value)) return value.toString() - return '' +function publicationFiles(target: PackageTarget): PackFile[] { + const paths = new Set() + addPath(resolve(target.directory, 'package.json'), paths) + const declared = Array.isArray(target.manifest.files) + ? target.manifest.files.filter((value): value is string => typeof value === 'string') + : [] + for (const pattern of [ + ...declared, + 'README*', + 'LICENSE*', + 'LICENCE*', + 'CHANGELOG*', + 'CHANGES*', + 'HISTORY*', + 'NOTICE*', + ]) { + for (const match of globSync(pattern, { cwd: target.directory })) { + addPath(resolve(target.directory, match), paths) + } + } + + return [...paths] + .sort() + .map(path => ({ + name: `package/${relative(target.directory, path).split(sep).join('/')}`, + data: readFileSync(path), + })) } -async function runPublint(path: string): Promise { +function addPath(path: string, paths: Set): void { + const stat = statSync(path) + if (stat.isDirectory()) { + for (const entry of readdirSync(path)) addPath(resolve(path, entry), paths) + } else if (stat.isFile()) { + paths.add(path) + } +} + +async function runPublint(target: PackageTarget): Promise { try { - const { stdout, stderr } = await execFileAsync(process.execPath, [publintCli, path], { - cwd: root, - encoding: 'utf8', - maxBuffer: 10 * 1024 * 1024, + const result = await publint({ + pkgDir: 'package', + pack: { files: publicationFiles(target) }, }) - return { path, status: 'passed', stdout, stderr } + const manifest = result.pkg as Record + return result.messages.some(message => message.type === 'error') + ? { path: target.path, status: 'failed', messages: result.messages, manifest } + : { path: target.path, status: 'passed', messages: result.messages, manifest } } catch (error: unknown) { - const failed = error as { stdout?: unknown; stderr?: unknown; message?: string } return { - path, + path: target.path, status: 'failed', - stdout: outputText(failed.stdout), - stderr: outputText(failed.stderr), - message: failed.message ?? 'publint failed', + messages: [], + manifest: target.manifest as Record, + failure: error instanceof Error ? error.message : String(error), } } } -async function runAll(paths: string[], concurrency: number): Promise { +async function runAll(targets: PackageTarget[], concurrency: number): Promise { let next = 0 const results: Array = [] await Promise.all(Array.from({ length: concurrency }, async () => { for (;;) { const index = next next += 1 - const path = paths[index] - if (path === undefined) return - results[index] = await runPublint(path) + const target = targets[index] + if (target === undefined) return + results[index] = await runPublint(target) } })) - return paths.map((path, index) => { + return targets.map((target, index) => { const result = results[index] - if (result === undefined) throw new Error(`publint-all: missing result for ${path}.`) + if (result === undefined) throw new Error(`publint-all: missing result for ${target.path}.`) return result }) } function printResult(result: PublintResult): void { console.log(`Running publint for ${result.path}...`) - process.stdout.write(result.stdout) - process.stderr.write(result.stderr) - if (result.status === 'failed') console.error(result.message) + if ('failure' in result) console.error(result.failure) + for (const message of result.messages) { + console.log(formatMessage(message, result.manifest, { color: false }) ?? message.code) + } + if (result.status === 'passed' && result.messages.length === 0) console.log('All good!') +} + +function parseOptions(args: string[]): Map { + const parsed = new Map() + for (let index = 0; index < args.length; index += 2) { + const name = args[index] + const value = args[index + 1] + if (name !== '--packages-root' || value === undefined || value.startsWith('--')) { + throw new Error(`publint-all: expected [--packages-root PATH], got ${JSON.stringify(args)}.`) + } + if (parsed.has(name)) throw new Error(`publint-all: duplicate option ${name}.`) + parsed.set(name, value) + } + return parsed } const packages = workspacePackages() diff --git a/scripts/run-gates.ts b/scripts/run-gates.ts index 582972f778..7df9d8c4c5 100644 --- a/scripts/run-gates.ts +++ b/scripts/run-gates.ts @@ -8,6 +8,8 @@ import { spawn } from 'node:child_process' import { availableParallelism } from 'node:os' import { resolve } from 'node:path' import { performance } from 'node:perf_hooks' +import { coverageArgs } from './coverage-shards.ts' +import { selectStaticGates } from './static-shards.ts' type Mode = | 'ci-primary' @@ -161,20 +163,16 @@ function gatesForMode(selected: Mode): Gate[] { pnpmScript('duplication', 'duplication'), ] case 'ci-coverage': - return [ - pnpmScript('build', 'build'), - coverageGate(), - ] + return [coverageGate()] case 'ci-snapshot': - return [ - pnpmScript('build', 'build'), - snapshotGate(), - ] + return flagEnabled('DSH_SNAPSHOT_PREBUILT') + ? [snapshotGate([])] + : [pnpmScript('build', 'build'), snapshotGate()] case 'ci-artifacts': return ciArtifactGates() case 'node-compat': return [ - pnpmScript('typecheck', 'typecheck'), + ...flagEnabled('DSH_NODE_COMPAT_SKIP_TYPECHECK') ? [] : [pnpmScript('typecheck', 'typecheck')], pnpmExec('source-worker-smoke', [ 'vitest', 'run', @@ -230,7 +228,7 @@ function ciPrimaryGates(): Gate[] { } function ciStaticGates(): Gate[] { - return [ + const gates = [ pnpmScript('runtime-closure', 'verify-runtime-closure', { label: 'runtime closure' }), pnpmScript('constraints', 'constraints'), pnpmScript('package-invariants', 'verify-package-invariants', { label: 'package invariants' }), @@ -239,10 +237,12 @@ function ciStaticGates(): Gate[] { pnpmScript('module-graph', 'verify-module-graph', { label: 'module graph' }), pnpmScript('knip', 'knip'), ] + return selectStaticGates(gates, process.env.DSH_STATIC_SHARD) } function ciArtifactGates(): Gate[] { - return [ + const shard = process.env.DSH_ARTIFACT_SHARD + const metadataGates = [ pnpmScript('build', 'build'), pnpmScript('publint', 'publint', { needs: ['build'] }), pnpmScript('node-next-types', 'verify-node-next-types', { @@ -250,8 +250,14 @@ function ciArtifactGates(): Gate[] { needs: ['build'], }), builtPackageInvariantsGate(['build']), - builtBinSmokeGate(), ] + if (shard === 'metadata') return metadataGates + if (shard === 'smoke-1') return [pnpmScript('build', 'build'), builtBinSmokeGate('1/2')] + if (shard === 'smoke-2') return [pnpmScript('build', 'build'), builtBinSmokeGate('2/2')] + if (shard !== undefined && shard !== '') { + throw new Error(`run-gates: unknown DSH_ARTIFACT_SHARD ${JSON.stringify(shard)}.`) + } + return [...metadataGates, builtBinSmokeGate()] } function lintGate(): Gate { @@ -275,25 +281,36 @@ function lintGate(): Gate { } function coverageGate(): Gate { + const shard = process.env.DSH_COVERAGE_SHARD return pnpmExec('coverage', [ 'vitest', 'run', '--coverage', + ...(shard === undefined || shard === '' ? [] : coverageArgs(shard)), ...positiveIntArg('DSH_COVERAGE_MAX_WORKERS', '--maxWorkers'), ], { label: 'test:coverage', - env: { DSH_EXAMPLE_MODE: 'lib' }, - needs: ['build'], }) } // The snapshot suite boots the example bins in `lib` mode (built artifact under plain Node, // plugins via real exports) — CI and pre-push already build, so they exercise what ships rather // than the tsx/source path dev uses. It therefore waits on `build`. -function snapshotGate(): Gate { - return pnpmScript('snapshot', 'test:snapshot', { +function snapshotGate(needs: string[] = ['build']): Gate { + const shard = process.env.DSH_SNAPSHOT_SHARD + if (shard !== undefined && shard !== '' && !/^\d+\/\d+$/.test(shard)) { + throw new Error(`run-gates: DSH_SNAPSHOT_SHARD must be INDEX/TOTAL, got ${JSON.stringify(shard)}.`) + } + return pnpmExec('snapshot', [ + 'vitest', + 'run', + '--config', + 'vitest.snapshot.config.ts', + ...(shard === undefined || shard === '' ? [] : [`--shard=${shard}`]), + ], { + label: 'test:snapshot', env: { DSH_EXAMPLE_MODE: 'lib' }, - needs: ['build'], + ...needs.length === 0 ? {} : { needs }, }) } @@ -314,6 +331,13 @@ function positiveIntArg(envName: string, flag: string): string[] { return [`${flag}=${raw}`] } +function flagEnabled(envName: string): boolean { + const raw = process.env[envName] + if (raw === undefined || raw === '') return false + if (raw !== '1') throw new Error(`run-gates: ${envName} must be 1 when set, got ${JSON.stringify(raw)}.`) + return true +} + function hygieneLeafGates(options: { artifactNeeds?: string[] } = {}): Gate[] { const artifactOptions = options.artifactNeeds === undefined ? {} : { needs: options.artifactNeeds } return [ @@ -363,7 +387,7 @@ function docSyncLeafGates(options: { ] } -function builtBinSmokeGate(): Gate { +function builtBinSmokeGate(shard?: string): Gate { return pnpmExec('built-bin-smoke', [ 'vitest', 'run', @@ -379,6 +403,7 @@ function builtBinSmokeGate(): Gate { // (the e2e lane runs unbuilt, so these files self-skip there). 'packages/workflow/workflow-workerthread/tests/built-worker.e2e.ts', 'packages/code-runtime/code-runtime-worker/tests/built-lib.e2e.ts', + ...(shard === undefined ? [] : [`--shard=${shard}`]), ], { label: 'built-bin smoke', needs: ['build'], diff --git a/scripts/static-shards.spec.ts b/scripts/static-shards.spec.ts new file mode 100644 index 0000000000..2c57be6dd5 --- /dev/null +++ b/scripts/static-shards.spec.ts @@ -0,0 +1,16 @@ +import { describe, expect, it } from 'vitest' +import { selectStaticGates, staticShards } from './static-shards.ts' + +const completeInventory = staticShards.flatMap(shard => shard.gateIds).map(id => ({ id })) + +describe('static gate shards', () => { + it.each(staticShards)('selects only the gates owned by $name', (shard) => { + expect(selectStaticGates(completeInventory, shard.name).map(gate => gate.id)).toEqual(shard.gateIds) + }) + + it('rejects missing, duplicate, and unknown assignments', () => { + expect(() => selectStaticGates(completeInventory.slice(1))).toThrow('assign every static gate exactly once') + expect(() => selectStaticGates([...completeInventory, completeInventory[0]!])).toThrow('static gate IDs must be unique') + expect(() => selectStaticGates(completeInventory, 'missing')).toThrow('unknown DSH_STATIC_SHARD') + }) +}) diff --git a/scripts/static-shards.ts b/scripts/static-shards.ts new file mode 100644 index 0000000000..4d7a63b8c9 --- /dev/null +++ b/scripts/static-shards.ts @@ -0,0 +1,77 @@ +/** Static-gate shard definitions for GitHub Actions. */ + +/** A static CI lane identified by the gate IDs it owns. */ +export interface StaticShard { + /** Stable lane identifier passed through `DSH_STATIC_SHARD`. */ + name: string + /** Gate IDs selected from the static gate inventory. */ + gateIds: readonly string[] +} + +/** Exhaustive, non-overlapping ownership of static CI gates. */ +export const staticShards = [ + { + name: 'foundation', + gateIds: [ + 'runtime-closure', + 'constraints', + 'package-invariants', + 'cordis-config', + 'module-graph', + 'knip', + ], + }, + { + name: 'api-contracts', + gateIds: ['doc-typecheck', 'export-jsdoc', 'scoped-events', 'type-equivalence'], + }, + { + name: 'catalogs', + gateIds: ['cordis-catalog', 'tool-catalog', 'config-catalog', 'persistence-catalog', 'doc-graphs'], + }, + { + name: 'prose', + gateIds: [ + 'markdown-wrap', + 'markdown-links', + 'doc-refs', + 'package-paths', + 'package-readme-model-experience', + 'mermaid', + 'agent-note-classification', + 'agent-note-format', + 'translation-prompt', + 'translation-pairing', + 'doc-budgets', + 'package-readme-limitations', + ], + }, + { name: 'site', gateIds: ['docs-site'] }, +] as const satisfies readonly StaticShard[] + +/** + * Validate the complete gate partition and optionally select one lane. + * + * @param gates Complete static gate inventory. + * @param name Optional stable shard name. + * @returns All gates when no shard is requested, otherwise the selected lane. + */ +export function selectStaticGates(gates: readonly T[], name?: string): T[] { + const gateIds = gates.map(gate => gate.id) + const assignedIds = staticShards.flatMap(shard => shard.gateIds) + const uniqueGateIds = new Set(gateIds) + const uniqueAssignedIds = new Set(assignedIds) + if (uniqueGateIds.size !== gateIds.length) throw new Error('run-gates: static gate IDs must be unique.') + if (uniqueAssignedIds.size !== assignedIds.length) throw new Error('run-gates: static shard gate IDs must be unique.') + if (gateIds.length !== assignedIds.length + || gateIds.some(id => !uniqueAssignedIds.has(id)) + || assignedIds.some(id => !uniqueGateIds.has(id))) { + throw new Error('run-gates: static shards must assign every static gate exactly once.') + } + if (name === undefined || name === '') return [...gates] + + const shard = staticShards.find(candidate => candidate.name === name) + if (shard === undefined) throw new Error(`run-gates: unknown DSH_STATIC_SHARD ${JSON.stringify(name)}.`) + const selectedIds = new Set(shard.gateIds) + return gates.filter(gate => selectedIds.has(gate.id)) +} diff --git a/scripts/verify-built-package-invariants.mjs b/scripts/verify-built-package-invariants.mjs index 4b298946d1..2d5a496354 100644 --- a/scripts/verify-built-package-invariants.mjs +++ b/scripts/verify-built-package-invariants.mjs @@ -1,102 +1,106 @@ -/** Verify every packed companion through its package self-reference under plain Node. */ +/** Verify every compiled companion through its staged package self-reference under plain Node. */ -import { spawnSync } from 'node:child_process' import { copyFileSync, + cpSync, + existsSync, globSync, mkdirSync, mkdtempSync, readFileSync, rmSync, + writeFileSync, } from 'node:fs' import { dirname, resolve } from 'node:path' import { pathToFileURL } from 'node:url' -const root = resolve(import.meta.dirname, '..') -const loaderUrl = pathToFileURL(resolve(root, 'vendor/loader/lib/index.js')).href +const repositoryRoot = resolve(import.meta.dirname, '..') +const options = parseOptions(process.argv.slice(2)) +const packagesRoot = resolve(options.get('--packages-root') ?? repositoryRoot) +const loaderUrl = options.get('--loader-url') + ?? pathToFileURL(resolve(repositoryRoot, 'vendor/loader/lib/index.js')).href const failures = [] -const manifests = globSync('packages/*/*/package.json', { cwd: root }).sort() -const packArgs = ['pack', '--dry-run', '--json', '--ignore-scripts'] -// Windows cannot spawn npm's .cmd shim directly; setup-node installs this JS -// entrypoint beside node.exe, so the probe stays shell-free on every runner. -const npmInvocation = process.platform === 'win32' - ? [process.execPath, [resolve(dirname(process.execPath), 'node_modules/npm/bin/npm-cli.js'), ...packArgs]] - : ['npm', packArgs] +const manifests = globSync('packages/*/*/package.json', { cwd: packagesRoot }).sort() +const { default: Loader } = await import(loaderUrl) +const loader = Object.create(Loader.prototype) for (const manifestPath of manifests) { - const packageDir = dirname(resolve(root, manifestPath)) - const manifest = JSON.parse(readFileSync(resolve(root, manifestPath), 'utf8')) + const packageDir = dirname(resolve(packagesRoot, manifestPath)) + const manifest = JSON.parse(readFileSync(resolve(packagesRoot, manifestPath), 'utf8')) const packageName = manifest.name if (typeof packageName !== 'string' || packageName.length === 0) { failures.push(`${manifestPath}: missing package name`) continue } - - const pack = spawnSync(npmInvocation[0], npmInvocation[1], { - cwd: packageDir, - encoding: 'utf8', - }) - if (pack.status !== 0) { - const detail = pack.error?.message - ?? (pack.stderr.trim() || pack.stdout.trim() || `npm pack exited ${pack.status}`) - failures.push(`${packageName}: ${detail}`) + const invariantExport = manifest.exports?.['./invariant'] + if (typeof invariantExport !== 'object' + || invariantExport.default !== './lib/invariant.js' + || !manifest.files?.includes('lib/invariant.js')) { + failures.push(`${packageName}: manifest does not publish ./lib/invariant.js as ./invariant`) continue } - let files - try { - const result = JSON.parse(pack.stdout) - files = result[0]?.files - if (!Array.isArray(files)) throw new Error('npm pack returned no file inventory') - } catch (error) { - failures.push(`${packageName}: cannot parse npm pack inventory: ${String(error)}`) - continue - } - - // Keep the packed view below its owning package so Node reaches the real + // Keep the staged view below its owning package so Node reaches the real // pnpm dependency links. Junctioning node_modules elsewhere breaks pnpm's - // relative workspace links on Windows. - const stagedPackageDir = mkdtempSync(resolve(packageDir, '.dsh-packed-invariant-')) + // relative workspace links on Windows. Copy only the statically required + // runtime entry so a companion that imports an undeclared chunk fails here. + const stagedPackageDir = mkdtempSync(resolve(packageDir, '.dsh-built-invariant-')) try { - for (const file of files) { - if (typeof file.path !== 'string' - || (file.path !== 'package.json' && !file.path.startsWith('lib/'))) continue - const target = resolve(stagedPackageDir, file.path) - mkdirSync(dirname(target), { recursive: true }) - copyFileSync(resolve(packageDir, file.path), target) - } - - const probe = ` - const companion = await import(${JSON.stringify(`${packageName}/invariant`)}); - const { default: Loader } = await import(${JSON.stringify(loaderUrl)}); - if ('default' in companion) throw new Error('companion has a default export'); - const loader = Object.create(Loader.prototype); - const unwrapped = loader.unwrapExports(companion); - if (unwrapped !== companion) throw new Error('Loader collapsed the companion namespace'); - if (typeof unwrapped.name !== 'string') throw new Error('companion name is missing'); - if (!Array.isArray(unwrapped.inject) || !unwrapped.inject.includes('invariants')) { - throw new Error('companion does not inject invariants'); - } - if (typeof unwrapped.apply !== 'function') throw new Error('companion apply is missing'); - ` - const result = spawnSync(process.execPath, ['--input-type=module', '--eval', probe], { - cwd: stagedPackageDir, - encoding: 'utf8', - }) - if (result.status !== 0) { - const detail = result.error?.message - ?? (result.stderr.trim() || result.stdout.trim() || `node exited ${result.status}`) - failures.push(`${packageName}: ${detail}`) + copyFileSync(resolve(packageDir, 'package.json'), resolve(stagedPackageDir, 'package.json')) + copyDeclaredLibFiles(packageDir, stagedPackageDir, manifest.files) + const probePath = resolve(stagedPackageDir, 'probe.mjs') + writeFileSync( + probePath, + `import * as companion from ${JSON.stringify(`${packageName}/invariant`)}\nexport default companion\n`, + ) + const { default: companion } = await import(pathToFileURL(probePath).href) + if ('default' in companion) throw new Error('companion has a default export') + const unwrapped = loader.unwrapExports(companion) + if (unwrapped !== companion) throw new Error('Loader collapsed the companion namespace') + if (typeof unwrapped.name !== 'string') throw new Error('companion name is missing') + if (!Array.isArray(unwrapped.inject) || !unwrapped.inject.includes('invariants')) { + throw new Error('companion does not inject invariants') } + if (typeof unwrapped.apply !== 'function') throw new Error('companion apply is missing') + } catch (error) { + failures.push(`${packageName}: ${error instanceof Error ? error.message : String(error)}`) } finally { rmSync(stagedPackageDir, { recursive: true, force: true }) } } if (failures.length > 0) { - console.error('verify-built-package-invariants: packed companion failures:') + console.error('verify-built-package-invariants: compiled companion failures:') for (const failure of failures) console.error(` ${failure}`) process.exit(1) } -console.log(`verify-built-package-invariants: ${manifests.length} packed companion(s) passed plain-Node Loader checks.`) +console.log(`verify-built-package-invariants: ${manifests.length} compiled companion(s) passed plain-Node Loader checks.`) + +function parseOptions(args) { + const allowed = new Set(['--packages-root', '--loader-url']) + const parsed = new Map() + for (let index = 0; index < args.length; index += 2) { + const name = args[index] + const value = args[index + 1] + if (!allowed.has(name) || value === undefined || value.startsWith('--')) { + throw new Error(`verify-built-package-invariants: expected [--packages-root PATH] [--loader-url URL], got ${JSON.stringify(args)}.`) + } + if (parsed.has(name)) throw new Error(`verify-built-package-invariants: duplicate option ${name}.`) + parsed.set(name, value) + } + return parsed +} + +function copyDeclaredLibFiles(packageDir, stagedPackageDir, files) { + for (const pattern of files) { + if (!pattern.startsWith('lib/')) continue + for (const relativePath of globSync(pattern, { cwd: packageDir })) { + const source = resolve(packageDir, relativePath) + if (!existsSync(source)) continue + const target = resolve(stagedPackageDir, relativePath) + mkdirSync(dirname(target), { recursive: true }) + cpSync(source, target, { recursive: true }) + } + } +} diff --git a/scripts/verify-built-package-invariants.spec.ts b/scripts/verify-built-package-invariants.spec.ts new file mode 100644 index 0000000000..4863e30c54 --- /dev/null +++ b/scripts/verify-built-package-invariants.spec.ts @@ -0,0 +1,88 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { fileURLToPath, pathToFileURL } from 'node:url' +import { spawnSync } from 'node:child_process' +import { afterEach, describe, expect, it } from 'vitest' + +const verifier = fileURLToPath(new URL('./verify-built-package-invariants.mjs', import.meta.url)) +const roots: string[] = [] + +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }) +}) + +function fixture(options: { + invariantSource?: string + invariantExport?: string + runtimeChunk?: string +} = {}): { root: string; loaderUrl: string } { + const root = mkdtempSync(join(tmpdir(), 'dsh-built-package-invariants-')) + roots.push(root) + const packageDir = join(root, 'packages/core/probe') + mkdirSync(join(packageDir, 'lib'), { recursive: true }) + writeFileSync(join(packageDir, 'package.json'), `${JSON.stringify({ + name: '@deepseek-ai/dsh-probe', + type: 'module', + files: ['lib/invariant.js'], + exports: { + './invariant': { + default: options.invariantExport ?? './lib/invariant.js', + }, + }, + }, null, 2)}\n`) + writeFileSync( + join(packageDir, 'lib/invariant.js'), + options.invariantSource ?? "export const name = 'probe-invariant'\nexport const inject = ['invariants']\nexport const apply = () => {}\n", + ) + if (options.runtimeChunk !== undefined) { + writeFileSync(join(packageDir, 'lib/chunk.js'), options.runtimeChunk) + } + const loaderPath = join(root, 'loader.mjs') + writeFileSync(loaderPath, 'export default class Loader { unwrapExports(value) { return value } }\n') + return { root, loaderUrl: pathToFileURL(loaderPath).href } +} + +function verify(root: string, loaderUrl: string) { + return spawnSync(process.execPath, [ + verifier, + '--packages-root', root, + '--loader-url', loaderUrl, + ], { + encoding: 'utf8', + timeout: 5_000, + }) +} + +describe('built package invariant verifier', () => { + it('loads the staged compiled self-reference through plain Node and Loader normalization', () => { + const { root, loaderUrl } = fixture() + const result = verify(root, loaderUrl) + expect(result.status, result.stderr).toBe(0) + expect(result.stdout).toContain('1 compiled companion(s) passed plain-Node Loader checks') + }) + + it('rejects a default export and a broken invariant export map', () => { + const withDefault = fixture({ + invariantSource: "export default {}\nexport const name = 'probe-invariant'\nexport const inject = ['invariants']\nexport const apply = () => {}\n", + }) + const defaultResult = verify(withDefault.root, withDefault.loaderUrl) + expect(defaultResult.status).toBe(1) + expect(defaultResult.stderr).toContain('companion has a default export') + + const brokenExport = fixture({ invariantExport: './lib/missing.js' }) + const exportResult = verify(brokenExport.root, brokenExport.loaderUrl) + expect(exportResult.status).toBe(1) + expect(exportResult.stderr).toContain('@deepseek-ai/dsh-probe') + }) + + it('rejects an invariant bundle that needs an unstaged runtime chunk', () => { + const { root, loaderUrl } = fixture({ + invariantSource: "export * from './chunk.js'\n", + runtimeChunk: "export const name = 'probe-invariant'\nexport const inject = ['invariants']\nexport const apply = () => {}\n", + }) + const result = verify(root, loaderUrl) + expect(result.status).toBe(1) + expect(result.stderr).toContain('chunk.js') + }) +}) From 91bd51c23a6bdea7109c2847659189c89d056da6 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Tue, 21 Jul 2026 20:32:57 +0800 Subject: [PATCH 02/49] ci: add coverage lane headroom --- .github/workflows/ci.yml | 42 ++++++++++++++++++--- scripts/coverage-shards.ts | 30 ++++++++++++--- scripts/verify-built-package-invariants.mjs | 4 +- 3 files changed, 62 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9be2932c11..a8dd6f9ec0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,11 +58,16 @@ jobs: command: pnpm run check:ci:lint gate_concurrency: '1' eslint_cache: '1' - - lane: coverage-spine + - lane: coverage-core command: pnpm run check:ci:coverage gate_concurrency: '1' coverage_max_workers: '4' - coverage_shard: spine + coverage_shard: core + - lane: coverage-models + command: pnpm run check:ci:coverage + gate_concurrency: '1' + coverage_max_workers: '4' + coverage_shard: models - lane: coverage-sdk command: pnpm run check:ci:coverage gate_concurrency: '1' @@ -78,16 +83,36 @@ jobs: gate_concurrency: '1' coverage_max_workers: '4' coverage_shard: execution - - lane: coverage-orchestration + - lane: coverage-workflow command: pnpm run check:ci:coverage gate_concurrency: '1' coverage_max_workers: '4' - coverage_shard: orchestration - - lane: coverage-infrastructure + coverage_shard: workflow + - lane: coverage-workflow-worker command: pnpm run check:ci:coverage gate_concurrency: '1' coverage_max_workers: '4' - coverage_shard: infrastructure + coverage_shard: workflow-worker + - lane: coverage-delegation + command: pnpm run check:ci:coverage + gate_concurrency: '1' + coverage_max_workers: '4' + coverage_shard: delegation + - lane: coverage-repository + command: pnpm run check:ci:coverage + gate_concurrency: '1' + coverage_max_workers: '4' + coverage_shard: repository + - lane: coverage-scripts + command: pnpm run check:ci:coverage + gate_concurrency: '1' + coverage_max_workers: '4' + coverage_shard: scripts + - lane: coverage-integrations + command: pnpm run check:ci:coverage + gate_concurrency: '1' + coverage_max_workers: '4' + coverage_shard: integrations - lane: coverage-session-state command: pnpm run check:ci:coverage gate_concurrency: '1' @@ -98,6 +123,11 @@ jobs: gate_concurrency: '1' coverage_max_workers: '4' coverage_shard: hooks-claude + - lane: coverage-hook-protocol + command: pnpm run check:ci:coverage + gate_concurrency: '1' + coverage_max_workers: '4' + coverage_shard: hook-protocol - lane: coverage-hooks-codex command: pnpm run check:ci:coverage gate_concurrency: '1' diff --git a/scripts/coverage-shards.ts b/scripts/coverage-shards.ts index 18c360401c..5ac8f999f3 100644 --- a/scripts/coverage-shards.ts +++ b/scripts/coverage-shards.ts @@ -13,10 +13,11 @@ export interface CoverageShard { /** Exhaustive, non-overlapping ownership of workspace packages in coverage CI. */ export const coverageShards = [ { - name: 'spine', - packageRoots: ['core', 'llm', 'compact', 'context'], + name: 'core', + packageRoots: ['core', 'context'], extraTestRoots: ['packages/examples/cli-demo/tests'], }, + { name: 'models', packageRoots: ['llm', 'compact'] }, { name: 'sdk', packageRoots: ['sdk'] }, { name: 'interfaces', @@ -24,17 +25,34 @@ export const coverageShards = [ extraTestRoots: ['examples'], }, { name: 'execution', packageRoots: ['fs', 'bash', 'sandbox', 'code-runtime'] }, - { name: 'orchestration', packageRoots: ['workflow', 'subagent', 'tasks'] }, { - name: 'infrastructure', - packageRoots: ['cordis', 'support', 'lsp', 'mcp'], + name: 'workflow', + packageRoots: ['workflow/workflow', 'workflow/tool-workflow', 'workflow/tool-ralph'], + }, + { name: 'workflow-worker', packageRoots: ['workflow/workflow-workerthread'] }, + { name: 'delegation', packageRoots: ['subagent', 'tasks'] }, + { + name: 'repository', + packageRoots: [ + 'cordis', + 'support/acp-snapshot', + 'support/agent-loop-testkit', + 'support/llm-replay', + 'support/loader-smoke', + ], + }, + { + name: 'scripts', + packageRoots: ['support/invariants'], extraTestRoots: ['scripts'], }, + { name: 'integrations', packageRoots: ['lsp', 'mcp'] }, { name: 'session-state', packageRoots: ['session-persistence', 'session-query'], }, - { name: 'hooks-claude', packageRoots: ['hooks/hook-protocol', 'hooks/hooks-claude'] }, + { name: 'hook-protocol', packageRoots: ['hooks/hook-protocol'] }, + { name: 'hooks-claude', packageRoots: ['hooks/hooks-claude'] }, { name: 'hooks-codex', packageRoots: ['hooks/hooks-codex'] }, { name: 'capabilities', diff --git a/scripts/verify-built-package-invariants.mjs b/scripts/verify-built-package-invariants.mjs index 2d5a496354..9c672e05f0 100644 --- a/scripts/verify-built-package-invariants.mjs +++ b/scripts/verify-built-package-invariants.mjs @@ -42,8 +42,8 @@ for (const manifestPath of manifests) { // Keep the staged view below its owning package so Node reaches the real // pnpm dependency links. Junctioning node_modules elsewhere breaks pnpm's - // relative workspace links on Windows. Copy only the statically required - // runtime entry so a companion that imports an undeclared chunk fails here. + // relative workspace links on Windows. Copy the manifest-declared lib view + // so a companion that imports an undeclared runtime chunk fails here. const stagedPackageDir = mkdtempSync(resolve(packageDir, '.dsh-built-invariant-')) try { copyFileSync(resolve(packageDir, 'package.json'), resolve(stagedPackageDir, 'package.json')) From 3d965082443d7216e8755b122652255226095aa3 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Tue, 21 Jul 2026 20:51:28 +0800 Subject: [PATCH 03/49] ci: restore hosted-run headroom --- .../2026-07-06-parallel-github-ci-gates.md | 4 +- .github/workflows/ci.yml | 24 +++++++++-- scripts/lint-shards.spec.ts | 34 ++++++++++++++++ scripts/lint-shards.ts | 40 +++++++++++++++++++ scripts/run-gates.ts | 13 +++--- scripts/static-shards.ts | 2 +- 6 files changed, 105 insertions(+), 12 deletions(-) create mode 100644 scripts/lint-shards.spec.ts create mode 100644 scripts/lint-shards.ts diff --git a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md index 0058a30660..a76c425da7 100644 --- a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md +++ b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md @@ -14,9 +14,9 @@ The artifact boundary remains load-bearing. `publint`, `verify-node-next-types`, [CI](../../../../.github/workflows/ci.yml) bounds every non-Windows job to one minute and every Windows job to three minutes. The timeout is an executable regression ceiling; the lane design leaves headroom below it rather than treating a timeout as normal control flow. -[scripts/run-gates.ts](../../../../scripts/run-gates.ts) remains the common bounded scheduler, but GitHub supplies explicit shard names for the expensive gate families. [scripts/static-shards.ts](../../../../scripts/static-shards.ts) partitions static gates into foundation, API-contract, catalog, prose, and documentation-site lanes and rejects a missing or duplicate gate assignment. [scripts/coverage-shards.ts](../../../../scripts/coverage-shards.ts) assigns every workspace package to exactly one source-coverage lane; its test expands the live package tree, so a new package makes CI red until it has an owner. Each coverage lane includes only its owned source files, repeats the exhaustive companion topology test, and runs without a preceding build because the complete coverage suite passes from a tree with every generated `lib/` removed. +[scripts/run-gates.ts](../../../../scripts/run-gates.ts) remains the common bounded scheduler, but GitHub supplies explicit shard names for the expensive gate families. [scripts/static-shards.ts](../../../../scripts/static-shards.ts) partitions static gates into foundation, API-contract, catalog, prose, and documentation-site lanes and rejects a missing or duplicate gate assignment. Lint uses disjoint package-source, package-test, and repository-complement lanes; the complement still starts from `.` so a new top-level lint target cannot disappear between shards, and it owns the single cross-file duplication run. [scripts/coverage-shards.ts](../../../../scripts/coverage-shards.ts) assigns every workspace package to exactly one source-coverage lane; its test expands the live package tree, so a new package makes CI red until it has an owner. Each coverage lane includes only its owned source files, repeats the exhaustive companion topology test, and runs without a preceding build because the complete coverage suite passes from a tree with every generated `lib/` removed. -Snapshot replay is four Vitest file shards. Each snapshot job builds the shipped runtime while its Linux runner installs bubblewrap, then runs only its assigned replay files. Static, coverage, and snapshot sharding changes only GitHub scheduling: the ordinary local package scripts still run their complete suites. +Snapshot replay is four Vitest file shards. Each snapshot job builds the shipped runtime while its Linux runner installs bubblewrap from the hosted image's existing package index, then runs only its assigned replay files. CI explicitly retains the suite's bounded concurrency of five subprocesses instead of clamping it to the runner's two logical CPUs, because replay spends most of its time waiting on child protocol I/O. Static, lint, coverage, and snapshot sharding changes only GitHub scheduling: the ordinary local package scripts still run their complete suites. Artifacts use three lanes: one metadata lane for `publint`, NodeNext declarations, and compiled invariant loading, plus two Vitest shards for built-bin smoke. Each lane produces its own build before its consumers. Repeating the short build costs runner minutes but avoids an upload/download dependency and keeps each job's critical path bounded. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a8dd6f9ec0..1ae8eb35de 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,9 @@ jobs: DSH_ARTIFACT_SHARD: ${{ matrix.artifact_shard }} DSH_COVERAGE_MAX_WORKERS: ${{ matrix.coverage_max_workers }} DSH_COVERAGE_SHARD: ${{ matrix.coverage_shard }} + DSH_LINT_SHARD: ${{ matrix.lint_shard }} DSH_STATIC_SHARD: ${{ matrix.static_shard }} + DSH_SNAPSHOT_MAX_CONCURRENCY: ${{ matrix.snapshot_max_concurrency }} DSH_SNAPSHOT_PREBUILT: ${{ matrix.snapshot_prebuilt }} DSH_SNAPSHOT_SHARD: ${{ matrix.snapshot_shard }} DSH_ESLINT_CACHE: ${{ matrix.eslint_cache }} @@ -54,10 +56,21 @@ jobs: command: pnpm run check:ci:static gate_concurrency: '1' static_shard: site - - lane: lint + - lane: lint-package-sources command: pnpm run check:ci:lint gate_concurrency: '1' eslint_cache: '1' + lint_shard: package-sources + - lane: lint-package-tests + command: pnpm run check:ci:lint + gate_concurrency: '1' + eslint_cache: '1' + lint_shard: package-tests + - lane: lint-repository + command: pnpm run check:ci:lint + gate_concurrency: '1' + eslint_cache: '1' + lint_shard: repository - lane: coverage-core command: pnpm run check:ci:coverage gate_concurrency: '1' @@ -141,21 +154,25 @@ jobs: - lane: snapshot-1 command: pnpm run check:ci:snapshot gate_concurrency: '1' + snapshot_max_concurrency: '5' snapshot_prebuilt: '1' snapshot_shard: '1/4' - lane: snapshot-2 command: pnpm run check:ci:snapshot gate_concurrency: '1' + snapshot_max_concurrency: '5' snapshot_prebuilt: '1' snapshot_shard: '2/4' - lane: snapshot-3 command: pnpm run check:ci:snapshot gate_concurrency: '1' + snapshot_max_concurrency: '5' snapshot_prebuilt: '1' snapshot_shard: '3/4' - lane: snapshot-4 command: pnpm run check:ci:snapshot gate_concurrency: '1' + snapshot_max_concurrency: '5' snapshot_prebuilt: '1' snapshot_shard: '4/4' - lane: artifacts-metadata @@ -207,8 +224,7 @@ jobs: pnpm run build & build_pid=$! ( - sudo apt-get update -q - sudo apt-get install -yq bubblewrap + sudo apt-get install -yq --no-install-recommends bubblewrap sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 \ || echo "apparmor userns knob absent — the functional probe decides" ) & @@ -221,7 +237,7 @@ jobs: exit "$sandbox_status" - uses: actions/cache@v4 - if: matrix.lane == 'lint' + if: startsWith(matrix.lane, 'lint-') with: path: .cache/eslint key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-${{ hashFiles('pnpm-lock.yaml', 'eslint.config.mjs', 'tsconfig.json', 'packages/*/*/tsconfig.json', 'examples/*/tsconfig.json') }} diff --git a/scripts/lint-shards.spec.ts b/scripts/lint-shards.spec.ts new file mode 100644 index 0000000000..df50790260 --- /dev/null +++ b/scripts/lint-shards.spec.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from 'vitest' +import { selectLintShard } from './lint-shards.ts' + +describe('lint gate shards', () => { + it('keeps the unsharded local command complete', () => { + expect(selectLintShard()).toEqual({ eslintTargets: ['.'], includeDuplication: true }) + expect(selectLintShard('')).toEqual({ eslintTargets: ['.'], includeDuplication: true }) + }) + + it('partitions package sources, package tests, and their repository complement', () => { + expect(selectLintShard('package-sources')).toEqual({ + eslintTargets: ['packages/*/*/src/**/*.ts'], + includeDuplication: false, + }) + expect(selectLintShard('package-tests')).toEqual({ + eslintTargets: ['packages/*/*/tests/**/*.ts'], + includeDuplication: false, + }) + expect(selectLintShard('repository')).toEqual({ + eslintTargets: [ + '.', + '--ignore-pattern', + 'packages/*/*/src/**', + '--ignore-pattern', + 'packages/*/*/tests/**', + ], + includeDuplication: true, + }) + }) + + it('rejects an unknown lane', () => { + expect(() => selectLintShard('missing')).toThrow('unknown DSH_LINT_SHARD') + }) +}) diff --git a/scripts/lint-shards.ts b/scripts/lint-shards.ts new file mode 100644 index 0000000000..ec09a7aca2 --- /dev/null +++ b/scripts/lint-shards.ts @@ -0,0 +1,40 @@ +/** Lint-lane selection for GitHub Actions. */ + +/** One ESLint target set and whether it owns the cross-file duplication gate. */ +export interface LintSelection { + /** Shell-free arguments passed to ESLint before its cache options. */ + eslintTargets: readonly string[] + /** Whether this lane also runs the repository-wide duplication check. */ + includeDuplication: boolean +} + +/** + * Select an exhaustive lint partition without changing the ordinary local lint command. + * + * @param name Optional stable shard name from `DSH_LINT_SHARD`. + * @returns ESLint targets and ownership of the duplication gate. + */ +export function selectLintShard(name?: string): LintSelection { + switch (name) { + case undefined: + case '': + return { eslintTargets: ['.'], includeDuplication: true } + case 'package-sources': + return { eslintTargets: ['packages/*/*/src/**/*.ts'], includeDuplication: false } + case 'package-tests': + return { eslintTargets: ['packages/*/*/tests/**/*.ts'], includeDuplication: false } + case 'repository': + return { + eslintTargets: [ + '.', + '--ignore-pattern', + 'packages/*/*/src/**', + '--ignore-pattern', + 'packages/*/*/tests/**', + ], + includeDuplication: true, + } + default: + throw new Error(`run-gates: unknown DSH_LINT_SHARD ${JSON.stringify(name)}.`) + } +} diff --git a/scripts/run-gates.ts b/scripts/run-gates.ts index 8eade29028..262fac886f 100644 --- a/scripts/run-gates.ts +++ b/scripts/run-gates.ts @@ -9,6 +9,7 @@ import { availableParallelism } from 'node:os' import { resolve } from 'node:path' import { performance } from 'node:perf_hooks' import { coverageArgs } from './coverage-shards.ts' +import { selectLintShard } from './lint-shards.ts' import { selectStaticGates } from './static-shards.ts' type Mode = @@ -162,11 +163,13 @@ function gatesForMode(selected: Mode): Gate[] { return ciPrimaryGates() case 'ci-static': return ciStaticGates() - case 'ci-lint': + case 'ci-lint': { + const selection = selectLintShard(process.env.DSH_LINT_SHARD) return [ - lintGate(), - pnpmScript('duplication', 'duplication'), + lintGate(selection.eslintTargets), + ...selection.includeDuplication ? [pnpmScript('duplication', 'duplication')] : [], ] + } case 'ci-coverage': return [coverageGate()] case 'ci-snapshot': @@ -267,11 +270,11 @@ function ciArtifactGates(): Gate[] { return [...metadataGates, builtBinSmokeGate()] } -function lintGate(): Gate { +function lintGate(eslintTargets: readonly string[] = ['.']): Gate { if (process.env.DSH_ESLINT_CACHE === '1') { return pnpmExec('lint', [ 'eslint', - '.', + ...eslintTargets, '--cache', '--cache-location', '.cache/eslint/', diff --git a/scripts/static-shards.ts b/scripts/static-shards.ts index 4d7a63b8c9..7376aa44d7 100644 --- a/scripts/static-shards.ts +++ b/scripts/static-shards.ts @@ -23,7 +23,7 @@ export const staticShards = [ }, { name: 'api-contracts', - gateIds: ['doc-typecheck', 'export-jsdoc', 'scoped-events', 'type-equivalence'], + gateIds: ['doc-typecheck', 'cordis-api', 'export-jsdoc', 'scoped-events', 'type-equivalence'], }, { name: 'catalogs', From 25f9035ecdbd470f5c4ec85db62e9258e5e0f854 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Tue, 21 Jul 2026 21:05:39 +0800 Subject: [PATCH 04/49] ci: split remaining one-minute lanes --- .../2026-07-06-parallel-github-ci-gates.md | 6 +- .github/workflows/ci.yml | 56 ++++++++++++++----- examples/acp-agent/tests/acp.snapshot.ts | 4 ++ .../tests/snapshot-scenario-shard.spec.ts | 20 +++++++ .../tests/snapshot-scenario-shard.ts | 18 ++++++ packages/support/acp-snapshot/README.md | 2 +- packages/support/acp-snapshot/src/index.ts | 1 + .../acp-snapshot/src/scenario-shard.ts | 36 ++++++++++++ packages/support/acp-snapshot/src/suite.ts | 14 ++++- .../acp-snapshot/tests/scenario-shard.spec.ts | 25 +++++++++ .../support/acp-snapshot/tests/suite.spec.ts | 22 ++++++++ scripts/run-gates.ts | 20 ++++--- scripts/snapshot-shards.spec.ts | 45 +++++++++++++++ scripts/snapshot-shards.ts | 54 ++++++++++++++++++ scripts/static-shards.ts | 3 +- 15 files changed, 297 insertions(+), 29 deletions(-) create mode 100644 examples/acp-agent/tests/snapshot-scenario-shard.spec.ts create mode 100644 examples/acp-agent/tests/snapshot-scenario-shard.ts create mode 100644 packages/support/acp-snapshot/src/scenario-shard.ts create mode 100644 packages/support/acp-snapshot/tests/scenario-shard.spec.ts create mode 100644 scripts/snapshot-shards.spec.ts create mode 100644 scripts/snapshot-shards.ts diff --git a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md index a76c425da7..a77493d8a9 100644 --- a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md +++ b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md @@ -14,15 +14,15 @@ The artifact boundary remains load-bearing. `publint`, `verify-node-next-types`, [CI](../../../../.github/workflows/ci.yml) bounds every non-Windows job to one minute and every Windows job to three minutes. The timeout is an executable regression ceiling; the lane design leaves headroom below it rather than treating a timeout as normal control flow. -[scripts/run-gates.ts](../../../../scripts/run-gates.ts) remains the common bounded scheduler, but GitHub supplies explicit shard names for the expensive gate families. [scripts/static-shards.ts](../../../../scripts/static-shards.ts) partitions static gates into foundation, API-contract, catalog, prose, and documentation-site lanes and rejects a missing or duplicate gate assignment. Lint uses disjoint package-source, package-test, and repository-complement lanes; the complement still starts from `.` so a new top-level lint target cannot disappear between shards, and it owns the single cross-file duplication run. [scripts/coverage-shards.ts](../../../../scripts/coverage-shards.ts) assigns every workspace package to exactly one source-coverage lane; its test expands the live package tree, so a new package makes CI red until it has an owner. Each coverage lane includes only its owned source files, repeats the exhaustive companion topology test, and runs without a preceding build because the complete coverage suite passes from a tree with every generated `lib/` removed. +[scripts/run-gates.ts](../../../../scripts/run-gates.ts) remains the common bounded scheduler, but GitHub supplies explicit shard names for the expensive gate families. [scripts/static-shards.ts](../../../../scripts/static-shards.ts) partitions static gates into foundation, API-contract, catalog, prose, documentation-projection, and documentation-build lanes and rejects a missing or duplicate gate assignment. Lint uses disjoint package-source, package-test, and repository-complement lanes; the complement still starts from `.` so a new top-level lint target cannot disappear between shards, and it owns the single cross-file duplication run. [scripts/coverage-shards.ts](../../../../scripts/coverage-shards.ts) assigns every workspace package to exactly one source-coverage lane; its test expands the live package tree, so a new package makes CI red until it has an owner. Each coverage lane includes only its owned source files, repeats the exhaustive companion topology test, and runs without a preceding build because the complete coverage suite passes from a tree with every generated `lib/` removed. -Snapshot replay is four Vitest file shards. Each snapshot job builds the shipped runtime while its Linux runner installs bubblewrap from the hosted image's existing package index, then runs only its assigned replay files. CI explicitly retains the suite's bounded concurrency of five subprocesses instead of clamping it to the runner's two logical CPUs, because replay spends most of its time waiting on child protocol I/O. Static, lint, coverage, and snapshot sharding changes only GitHub scheduling: the ordinary local package scripts still run their complete suites. +Snapshot replay uses three explicit multi-file lanes and four scenario partitions of the large ACP file. [scripts/snapshot-shards.ts](../../../../scripts/snapshot-shards.ts) owns that inventory, and its test discovers every file admitted by the snapshot config so a new file cannot land outside CI. Each snapshot job builds the shipped runtime while its Linux runner installs bubblewrap from the hosted image's existing package index, then runs only its assigned replay surface. CI explicitly retains the suite's bounded concurrency of five subprocesses instead of clamping it to the runner's two logical CPUs, because replay spends most of its time waiting on child protocol I/O. Fixture guards still inspect the complete ACP scenario table in every partition. Static, lint, coverage, and snapshot sharding changes only GitHub scheduling: the ordinary local package scripts still run their complete suites. Artifacts use three lanes: one metadata lane for `publint`, NodeNext declarations, and compiled invariant loading, plus two Vitest shards for built-bin smoke. Each lane produces its own build before its consumers. Repeating the short build costs runner minutes but avoids an upload/download dependency and keeps each job's critical path bounded. [scripts/publint-all.ts](../../../../scripts/publint-all.ts) calls publint's supported API in-process against an in-memory publication view made from each manifest's declared files and npm's mandatory metadata files. This preserves the distinction between workspace files and published files without spawning a package-manager pack command 103 times. [scripts/verify-built-package-invariants.mjs](../../../../scripts/verify-built-package-invariants.mjs) stages those structurally validated manifest-declared `lib/` files below the real package, then imports the compiled self-reference through plain Node and Cordis Loader normalization. A companion that reaches an undeclared runtime chunk still fails. -Compatibility lanes run the source worker and Zstandard runtime smokes on every advertised Node line. TypeScript checks the source graph once on the primary Node 24 lane; repeating the same compiler analysis on Node 22 and 26 added time without runtime-specific signal. +Compatibility lanes run the source worker and Zstandard runtime smokes on every advertised Node line. TypeScript checks the source graph once in a dedicated primary Node 24 lane; repeating the same compiler analysis in runtime compatibility jobs added time without runtime-specific signal. The workflow caches the pnpm store, preserves native PowerShell for Windows measurements, and retains one aggregate `all checks passed` status for branch protection. Windows build remains blocking; the wider Windows static, lint, and artifact matrix remains observational while carrying the same three-minute ceiling. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1ae8eb35de..d78cb52af4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,10 +27,10 @@ jobs: DSH_COVERAGE_MAX_WORKERS: ${{ matrix.coverage_max_workers }} DSH_COVERAGE_SHARD: ${{ matrix.coverage_shard }} DSH_LINT_SHARD: ${{ matrix.lint_shard }} + DSH_SNAPSHOT_LANE: ${{ matrix.snapshot_lane }} DSH_STATIC_SHARD: ${{ matrix.static_shard }} DSH_SNAPSHOT_MAX_CONCURRENCY: ${{ matrix.snapshot_max_concurrency }} DSH_SNAPSHOT_PREBUILT: ${{ matrix.snapshot_prebuilt }} - DSH_SNAPSHOT_SHARD: ${{ matrix.snapshot_shard }} DSH_ESLINT_CACHE: ${{ matrix.eslint_cache }} strategy: fail-fast: false @@ -52,10 +52,16 @@ jobs: command: pnpm run check:ci:static gate_concurrency: '4' static_shard: prose - - lane: static-site + - lane: static-site-projection command: pnpm run check:ci:static gate_concurrency: '1' - static_shard: site + static_shard: site-projection + - lane: static-site-build + command: pnpm run check:ci:static + gate_concurrency: '1' + static_shard: site-build + - lane: typecheck + command: pnpm run typecheck - lane: lint-package-sources command: pnpm run check:ci:lint gate_concurrency: '1' @@ -151,30 +157,48 @@ jobs: gate_concurrency: '1' coverage_max_workers: '4' coverage_shard: capabilities - - lane: snapshot-1 + - lane: snapshot-support command: pnpm run check:ci:snapshot gate_concurrency: '1' + snapshot_lane: support snapshot_max_concurrency: '5' snapshot_prebuilt: '1' - snapshot_shard: '1/4' - - lane: snapshot-2 + - lane: snapshot-demos command: pnpm run check:ci:snapshot gate_concurrency: '1' + snapshot_lane: demos snapshot_max_concurrency: '5' snapshot_prebuilt: '1' - snapshot_shard: '2/4' - - lane: snapshot-3 + - lane: snapshot-agents command: pnpm run check:ci:snapshot gate_concurrency: '1' + snapshot_lane: agents snapshot_max_concurrency: '5' snapshot_prebuilt: '1' - snapshot_shard: '3/4' - - lane: snapshot-4 + - lane: snapshot-acp-1 command: pnpm run check:ci:snapshot gate_concurrency: '1' + snapshot_lane: acp-1 + snapshot_max_concurrency: '5' + snapshot_prebuilt: '1' + - lane: snapshot-acp-2 + command: pnpm run check:ci:snapshot + gate_concurrency: '1' + snapshot_lane: acp-2 + snapshot_max_concurrency: '5' + snapshot_prebuilt: '1' + - lane: snapshot-acp-3 + command: pnpm run check:ci:snapshot + gate_concurrency: '1' + snapshot_lane: acp-3 + snapshot_max_concurrency: '5' + snapshot_prebuilt: '1' + - lane: snapshot-acp-4 + command: pnpm run check:ci:snapshot + gate_concurrency: '1' + snapshot_lane: acp-4 snapshot_max_concurrency: '5' snapshot_prebuilt: '1' - snapshot_shard: '4/4' - lane: artifacts-metadata command: pnpm run check:ci:artifacts gate_concurrency: '3' @@ -261,7 +285,7 @@ jobs: - node: '22.19' skip_typecheck: '1' - node: 24 - skip_typecheck: '' + skip_typecheck: '1' - node: 26 skip_typecheck: '1' steps: @@ -368,10 +392,14 @@ jobs: command: pnpm run check:ci:static gate_concurrency: '4' static_shard: prose - - lane: static-site + - lane: static-site-projection command: pnpm run check:ci:static gate_concurrency: '1' - static_shard: site + static_shard: site-projection + - lane: static-site-build + command: pnpm run check:ci:static + gate_concurrency: '1' + static_shard: site-build - lane: lint command: pnpm run check:ci:lint gate_concurrency: '1' diff --git a/examples/acp-agent/tests/acp.snapshot.ts b/examples/acp-agent/tests/acp.snapshot.ts index a3d939c874..485811c42f 100644 --- a/examples/acp-agent/tests/acp.snapshot.ts +++ b/examples/acp-agent/tests/acp.snapshot.ts @@ -1,6 +1,7 @@ import { fileURLToPath } from 'node:url' import { dirname, join } from 'node:path' import { defineAcpSnapshotSuite, type Scenario, type SnapshotSuiteOptions } from '@deepseek-ai/dsh-acp-snapshot' +import { snapshotScenarioShardFromEnv } from './snapshot-scenario-shard.ts' /** * The acp-agent example's snapshot suite: the scenario table for @@ -191,9 +192,12 @@ const SCENARIOS: Scenario[] = [ { name: 'fs-escalation-approved', hasModelTurn: true, recorded: true, headerClass: 'sandbox' }, ] +const scenarioShard = snapshotScenarioShardFromEnv(process.env.DSH_SNAPSHOT_SCENARIO_SHARD) + defineAcpSnapshotSuite({ agent: AGENT, snapshotsDir: join(dirname(fileURLToPath(import.meta.url)), 'snapshots'), scenarios: SCENARIOS, mode: snapshotModeFromEnv(process.env.DSH_SNAPSHOT), + ...scenarioShard === undefined ? {} : { scenarioShard }, }) diff --git a/examples/acp-agent/tests/snapshot-scenario-shard.spec.ts b/examples/acp-agent/tests/snapshot-scenario-shard.spec.ts new file mode 100644 index 0000000000..5a70cfe3b1 --- /dev/null +++ b/examples/acp-agent/tests/snapshot-scenario-shard.spec.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from 'vitest' +import { snapshotScenarioShardFromEnv } from './snapshot-scenario-shard.ts' + +describe('ACP snapshot scenario shard environment', () => { + it('keeps ordinary snapshot runs complete', () => { + expect(snapshotScenarioShardFromEnv()).toBeUndefined() + expect(snapshotScenarioShardFromEnv('')).toBeUndefined() + }) + + it('parses a valid one-based shard', () => { + expect(snapshotScenarioShardFromEnv('2/4')).toEqual({ index: 2, total: 4 }) + }) + + it.each(['0/1', '1/0', '2/1', '1.5/2', 'missing', '999999999999999999999/999999999999999999999'])( + 'rejects %s', + (value) => { + expect(() => snapshotScenarioShardFromEnv(value)).toThrow('DSH_SNAPSHOT_SCENARIO_SHARD') + }, + ) +}) diff --git a/examples/acp-agent/tests/snapshot-scenario-shard.ts b/examples/acp-agent/tests/snapshot-scenario-shard.ts new file mode 100644 index 0000000000..4e2763bdb6 --- /dev/null +++ b/examples/acp-agent/tests/snapshot-scenario-shard.ts @@ -0,0 +1,18 @@ +import type { SnapshotScenarioShard } from '@deepseek-ai/dsh-acp-snapshot' + +/** + * Parse the optional CI scenario shard passed to this snapshot suite. + * + * @param value An `INDEX/TOTAL` string or an unset value. + * @returns A validated one-based shard, or undefined for the complete suite. + */ +export function snapshotScenarioShardFromEnv(value?: string): SnapshotScenarioShard | undefined { + if (value === undefined || value === '') return undefined + const match = /^([1-9]\d*)\/([1-9]\d*)$/.exec(value) + if (match === null) throw new Error(`DSH_SNAPSHOT_SCENARIO_SHARD must be INDEX/TOTAL, got ${JSON.stringify(value)}`) + const shard = { index: Number(match[1]), total: Number(match[2]) } + if (!Number.isSafeInteger(shard.index) || !Number.isSafeInteger(shard.total) || shard.index > shard.total) { + throw new Error(`DSH_SNAPSHOT_SCENARIO_SHARD is out of range: ${JSON.stringify(value)}`) + } + return shard +} diff --git a/packages/support/acp-snapshot/README.md b/packages/support/acp-snapshot/README.md index b109ca3afa..2766240135 100644 --- a/packages/support/acp-snapshot/README.md +++ b/packages/support/acp-snapshot/README.md @@ -7,7 +7,7 @@ Four layers, importable separately: - **`launchAcpTestAgent` (launcher)** — boots an unbuilt ACP agent from a temp cwd, pins tsx to the repo tsconfig, connects the SDK client over a raw-byte stdout tee, collects session updates and stderr, surfaces asynchronous spawn failures through its startup lifecycle, fails closed on unhandled permission requests, and owns graceful or signalled shutdown. Shutdown waits for process exit, inherited stdio closure, and ACP parser exhaustion before resolving or propagating a child error, so captures are complete and callers can remove owned paths after either outcome. Snapshot and ordinary e2e suites share this process boundary; a test supplies only agent paths, cwd, environment overrides, and any permission policy. - **`runScenario` (harness)** — boots the real agent bin as a subprocess via tsx (unbuilt, Loader path), drives it over ACP JSON-RPC stdio from a deterministic `input.json` script, tees raw stdout for the expected-output and purity checks, and harvests every persisted session JSONL (parent + subagent children, primary-first) after a graceful stdin-EOF shutdown. Parameterized by `AgentUnderTest` (`binScript`, `configPath`, `tsconfigPath` — absolute paths; the subprocess cwd is a temp dir outside the repo). Startup failures preserve captured agent stderr in the rejected diagnostic. - **Normalizers** — pure functions turning the two captured surfaces into stable text: `normalizeStdout` (JSON-RPC ids → first-seen sequence; UUIDs/cwd → tokens; doubles as the stdout-purity check), `normalizeSessionLog` (times zeroed, `seq` kept), `scrubSystemPrompts` (prompt text → `{{system}}`), `scrubToolSchemas` (schema bulk → `{{tools}}`), and `scrubRequestHeaders` (all header bulk → `{{system}}`/`{{tools}}`/`{{messagePrefix}}` outside each pin, structure kept — [pinned-header Agent Note](../../../.agents/notes/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md)). -- **`defineAcpSnapshotSuite` (factory)** — registers the whole describe/it tree for a scenario table: per-scenario expected-output and re-persisted-log comparisons, record/refresh fixture write-back, rejection of structured `UNKNOWN_TOOL` results, the per-header-class pin (`system-prompt.expected.md` plus `tool-schemas.expected.json`) with its live uniformity guard, and the fixture guard block (no orphan scenario dirs, required files present, exactly one pin per class, every JSONL prompt/schema-scrubbed, non-pinning fixtures fully header-scrubbed). Each scenario directory's `session.jsonl` plus contiguous `session..jsonl` siblings are the ordered primary/child inventory; the scenario table does not duplicate their count. Must be called at vitest collection time. +- **`defineAcpSnapshotSuite` (factory)** — registers the whole describe/it tree for a scenario table: per-scenario expected-output and re-persisted-log comparisons, record/refresh fixture write-back, rejection of structured `UNKNOWN_TOOL` results, the per-header-class pin (`system-prompt.expected.md` plus `tool-schemas.expected.json`) with its live uniformity guard, and the fixture guard block (no orphan scenario dirs, required files present, exactly one pin per class, every JSONL prompt/schema-scrubbed, non-pinning fixtures fully header-scrubbed). Replay may partition subprocess-backed scenarios with `scenarioShard`; every lane still runs fixture guards against the complete table, while record and refresh reject sharding because they write fixtures. Each scenario directory's `session.jsonl` plus contiguous `session..jsonl` siblings are the ordered primary/child inventory; the scenario table does not duplicate their count. Must be called at vitest collection time. A consuming `*.snapshot.ts` is the scenario table plus one factory call: diff --git a/packages/support/acp-snapshot/src/index.ts b/packages/support/acp-snapshot/src/index.ts index 4d99cc96a2..6dd87b7cab 100644 --- a/packages/support/acp-snapshot/src/index.ts +++ b/packages/support/acp-snapshot/src/index.ts @@ -44,3 +44,4 @@ export { type Scenario, type SnapshotSuiteOptions, } from './suite.ts' +export type { SnapshotScenarioShard } from './scenario-shard.ts' diff --git a/packages/support/acp-snapshot/src/scenario-shard.ts b/packages/support/acp-snapshot/src/scenario-shard.ts new file mode 100644 index 0000000000..67b2492b80 --- /dev/null +++ b/packages/support/acp-snapshot/src/scenario-shard.ts @@ -0,0 +1,36 @@ +/** Scenario-level sharding for one ACP snapshot suite. */ + +/** A one-based, exhaustive partition of a scenario table. */ +export interface SnapshotScenarioShard { + /** One-based lane index. */ + index: number + /** Total number of lanes. */ + total: number +} + +/** + * Select one stable modulo partition while rejecting empty or malformed lanes. + * + * @param scenarios Complete ordered scenario table. + * @param shard Optional one-based shard description. + * @returns The complete table or the selected non-empty partition. + */ +export function selectSnapshotScenarios( + scenarios: readonly T[], + shard?: SnapshotScenarioShard, +): T[] { + if (shard === undefined) return [...scenarios] + if (!Number.isSafeInteger(shard.index) || shard.index < 1) { + throw new Error(`acp-snapshot: shard index must be a positive integer, got ${shard.index}`) + } + if (!Number.isSafeInteger(shard.total) || shard.total < 1) { + throw new Error(`acp-snapshot: shard total must be a positive integer, got ${shard.total}`) + } + if (shard.index > shard.total) { + throw new Error(`acp-snapshot: shard index ${shard.index} exceeds total ${shard.total}`) + } + if (shard.total > scenarios.length) { + throw new Error(`acp-snapshot: ${shard.total} shards exceed ${scenarios.length} scenarios`) + } + return scenarios.filter((_, offset) => offset % shard.total === shard.index - 1) +} diff --git a/packages/support/acp-snapshot/src/suite.ts b/packages/support/acp-snapshot/src/suite.ts index 60f5cfeadb..1c2d16b630 100644 --- a/packages/support/acp-snapshot/src/suite.ts +++ b/packages/support/acp-snapshot/src/suite.ts @@ -28,6 +28,7 @@ import { scrubSystemPrompts, scrubToolSchemas, } from './normalize.ts' +import { selectSnapshotScenarios, type SnapshotScenarioShard } from './scenario-shard.ts' /** The readable system-prompt snapshot beside each header-pinning fixture. */ const SYSTEM_PROMPT_SNAPSHOT = 'system-prompt.expected.md' @@ -110,6 +111,11 @@ export interface SnapshotSuiteOptions { snapshotsDir: string /** The scenario table; exactly one entry per header class must set `pinsHeader`. */ scenarios: Scenario[] + /** + * Optional replay-only scenario partition. Fixture guards still validate the + * complete table in every lane; only subprocess-backed scenario tests split. + */ + scenarioShard?: SnapshotScenarioShard /** * `replay` (keyless, the default tier), `record` (live API; re-records the * `recorded` scenarios' fixtures and refreshes the Vitest expected outputs under @@ -439,7 +445,11 @@ export function stabilizeRefreshLog(fresh: string, existing: string, replacement * @param options The agent, snapshots directory, scenario table, and mode. */ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void { - const { agent, snapshotsDir, scenarios, mode } = options + const { agent, snapshotsDir, scenarios, mode, scenarioShard } = options + if (scenarioShard !== undefined && mode !== 'replay') { + throw new Error('acp-snapshot: scenario sharding is supported only in replay mode') + } + const selectedScenarios = selectSnapshotScenarios(scenarios, scenarioShard) const RECORDING = mode === 'record' const REFRESHING = mode === 'refresh' const childMode: 'replay' | 'record' = RECORDING ? 'record' : 'replay' @@ -464,7 +474,7 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void { } scenarioSuite('snapshot scenarios', () => { - for (const scenario of scenarios) { + for (const scenario of selectedScenarios) { // In RECORD mode, only re-run the `recorded` (live-API) scenarios; the `authored` ones // (sidecar-driven errors/cancel) are never re-recorded. it.skipIf(RECORDING && !scenario.recorded)(`snapshot: ${scenario.name} matches the expected outputs`, async ({ expect }) => { diff --git a/packages/support/acp-snapshot/tests/scenario-shard.spec.ts b/packages/support/acp-snapshot/tests/scenario-shard.spec.ts new file mode 100644 index 0000000000..eb5c117153 --- /dev/null +++ b/packages/support/acp-snapshot/tests/scenario-shard.spec.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from 'vitest' +import { selectSnapshotScenarios } from '../src/scenario-shard.ts' + +describe('ACP snapshot scenario shards', () => { + it('keeps the ordinary suite complete', () => { + expect(selectSnapshotScenarios(['a', 'b', 'c'])).toEqual(['a', 'b', 'c']) + }) + + it('partitions the ordered table without gaps or overlap', () => { + const scenarios = ['a', 'b', 'c', 'd', 'e'] + expect(selectSnapshotScenarios(scenarios, { index: 1, total: 2 })).toEqual(['a', 'c', 'e']) + expect(selectSnapshotScenarios(scenarios, { index: 2, total: 2 })).toEqual(['b', 'd']) + }) + + it.each([ + [{ index: 0, total: 1 }, 'index must be a positive integer'], + [{ index: 1.5, total: 2 }, 'index must be a positive integer'], + [{ index: 1, total: 0 }, 'total must be a positive integer'], + [{ index: 1, total: Number.NaN }, 'total must be a positive integer'], + [{ index: 3, total: 2 }, 'exceeds total'], + [{ index: 1, total: 4 }, 'exceed 3 scenarios'], + ] as const)('rejects malformed shard %#', (shard, message) => { + expect(() => selectSnapshotScenarios(['a', 'b', 'c'], shard)).toThrow(message) + }) +}) diff --git a/packages/support/acp-snapshot/tests/suite.spec.ts b/packages/support/acp-snapshot/tests/suite.spec.ts index 6fbb06bb9d..33fff5d2bc 100644 --- a/packages/support/acp-snapshot/tests/suite.spec.ts +++ b/packages/support/acp-snapshot/tests/suite.spec.ts @@ -105,6 +105,16 @@ describe('defineAcpSnapshotSuite: replay mode', () => { defineAcpSnapshotSuite({ agent: AGENT, snapshotsDir: REPLAY_DIR, scenarios: REPLAY_SCENARIOS, mode: 'replay' }) }) +describe('defineAcpSnapshotSuite: sharded replay mode', () => { + defineAcpSnapshotSuite({ + agent: AGENT, + snapshotsDir: REPLAY_DIR, + scenarios: REPLAY_SCENARIOS, + mode: 'replay', + scenarioShard: { index: 2, total: 2 }, + }) +}) + // The record suite's tests run in registration order: rec-pin re-records the // pinned fixture FIRST, so rec-child's uniformity guard reads the fresh pin. describe('defineAcpSnapshotSuite: record mode', () => { @@ -154,6 +164,18 @@ describe('defineAcpSnapshotSuite: record inventory write-back', () => { }) describe('defineAcpSnapshotSuite: registration contract', () => { + it('rejects scenario sharding in a fixture-writing mode', () => { + expect(() => { + defineAcpSnapshotSuite({ + agent: AGENT, + snapshotsDir: REPLAY_DIR, + scenarios: REPLAY_SCENARIOS, + mode: 'refresh', + scenarioShard: { index: 1, total: 2 }, + }) + }).toThrow('supported only in replay mode') + }) + it("throws when a scenario's header class has no pinning scenario", () => { expect(() => { defineAcpSnapshotSuite({ diff --git a/scripts/run-gates.ts b/scripts/run-gates.ts index 262fac886f..e45a254905 100644 --- a/scripts/run-gates.ts +++ b/scripts/run-gates.ts @@ -10,6 +10,7 @@ import { resolve } from 'node:path' import { performance } from 'node:perf_hooks' import { coverageArgs } from './coverage-shards.ts' import { selectLintShard } from './lint-shards.ts' +import { selectSnapshotLane } from './snapshot-shards.ts' import { selectStaticGates } from './static-shards.ts' type Mode = @@ -307,19 +308,19 @@ function coverageGate(): Gate { // plugins via real exports) — CI and pre-push already build, so they exercise what ships rather // than the tsx/source path dev uses. It therefore waits on `build`. function snapshotGate(needs: string[] = ['build']): Gate { - const shard = process.env.DSH_SNAPSHOT_SHARD - if (shard !== undefined && shard !== '' && !/^\d+\/\d+$/.test(shard)) { - throw new Error(`run-gates: DSH_SNAPSHOT_SHARD must be INDEX/TOTAL, got ${JSON.stringify(shard)}.`) - } + const lane = selectSnapshotLane(process.env.DSH_SNAPSHOT_LANE) return pnpmExec('snapshot', [ 'vitest', 'run', '--config', 'vitest.snapshot.config.ts', - ...(shard === undefined || shard === '' ? [] : [`--shard=${shard}`]), + ...lane.files, ], { label: 'test:snapshot', - env: { DSH_EXAMPLE_MODE: 'lib' }, + env: { + DSH_EXAMPLE_MODE: 'lib', + ...lane.scenarioShard === undefined ? {} : { DSH_SNAPSHOT_SCENARIO_SHARD: lane.scenarioShard }, + }, ...needs.length === 0 ? {} : { needs }, }) } @@ -392,8 +393,11 @@ function docSyncLeafGates(options: { pnpmScript('translation-prompt', 'verify-translation-prompt', { label: 'translation prompt' }), pnpmScript('translation-pairing', 'verify-translation-pairing', { label: 'translation pairing' }), pnpmScript('doc-budgets', 'verify-doc-budgets', { label: 'doc budgets' }), - // Keep the VitePress build in this single gate because projection rewrites website/.generated. - pnpmScript('docs-site', 'docs:check', { label: 'documentation site' }), + pnpmExec('docs-site-projection', ['vitest', 'run', 'scripts/project-doc-site.spec.ts'], { + label: 'documentation projection', + }), + // Keep the VitePress build itself in one gate because projection rewrites website/.generated. + pnpmScript('docs-site-build', 'docs:build', { label: 'documentation build' }), pnpmScript('package-readme-limitations', 'verify-package-readme-limitations', { label: 'package README limitations' }), ] } diff --git a/scripts/snapshot-shards.spec.ts b/scripts/snapshot-shards.spec.ts new file mode 100644 index 0000000000..dc4bef62ad --- /dev/null +++ b/scripts/snapshot-shards.spec.ts @@ -0,0 +1,45 @@ +import { existsSync, readdirSync } from 'node:fs' +import { join, relative, sep } from 'node:path' +import { describe, expect, it } from 'vitest' +import { selectSnapshotLane, snapshotLanes } from './snapshot-shards.ts' + +const root = join(import.meta.dirname, '..') + +function snapshotFiles(dir: string): string[] { + if (!existsSync(dir)) return [] + return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { + const path = join(dir, entry.name) + if (entry.isDirectory()) return snapshotFiles(path) + return entry.name.endsWith('.snapshot.ts') ? [relative(root, path).split(sep).join('/')] : [] + }) +} + +describe('snapshot lanes', () => { + it('assigns every configured snapshot file and every ACP scenario shard', () => { + const discovered = [ + ...snapshotFiles(join(root, 'examples')), + ...snapshotFiles(join(root, 'packages/sdk')), + ...snapshotFiles(join(root, 'packages/ui/tui')), + ].filter(path => !path.includes('/node_modules/') && !path.includes('/lib/')).sort() + const ordinary = snapshotLanes.filter(lane => lane.scenarioShard === undefined).flatMap(lane => lane.files) + const acp = snapshotLanes.filter(lane => lane.scenarioShard !== undefined) + + expect(new Set(ordinary).size).toBe(ordinary.length) + expect(acp.map(lane => lane.files)).toEqual(Array.from( + { length: 4 }, + () => ['examples/acp-agent/tests/acp.snapshot.ts'], + )) + expect(acp.map(lane => lane.scenarioShard)).toEqual(['1/4', '2/4', '3/4', '4/4']) + expect([...ordinary, 'examples/acp-agent/tests/acp.snapshot.ts'].sort()).toEqual(discovered) + }) + + it('keeps ordinary runs complete and selects known lanes', () => { + expect(selectSnapshotLane()).toEqual({ name: 'complete', files: [] }) + expect(selectSnapshotLane('')).toEqual({ name: 'complete', files: [] }) + for (const lane of snapshotLanes) expect(selectSnapshotLane(lane.name)).toBe(lane) + }) + + it('rejects an unknown lane', () => { + expect(() => selectSnapshotLane('missing')).toThrow('unknown DSH_SNAPSHOT_LANE') + }) +}) diff --git a/scripts/snapshot-shards.ts b/scripts/snapshot-shards.ts new file mode 100644 index 0000000000..a6126f23c2 --- /dev/null +++ b/scripts/snapshot-shards.ts @@ -0,0 +1,54 @@ +/** Snapshot-lane definitions for GitHub Actions. */ + +/** One explicit snapshot file lane, optionally split again by ACP scenarios. */ +export interface SnapshotLane { + /** Stable lane name passed through `DSH_SNAPSHOT_LANE`. */ + name: string + /** Snapshot test files owned by the lane. */ + files: readonly string[] + /** Optional one-based ACP scenario partition. */ + scenarioShard?: string +} + +/** Exhaustive file ownership plus scenario partitions for the large ACP suite. */ +export const snapshotLanes: readonly SnapshotLane[] = [ + { + name: 'support', + files: [ + 'packages/sdk/scripts/tests/config.snapshot.ts', + 'packages/ui/tui/tests/tui.snapshot.ts', + ], + }, + { + name: 'demos', + files: [ + 'examples/tui-agent/tests/tui.snapshot.ts', + 'packages/sdk/create-sdk/tests/create.snapshot.ts', + ], + }, + { + name: 'agents', + files: [ + 'examples/acp-agent/tests/goal.snapshot.ts', + 'examples/headless-agent/tests/headless.snapshot.ts', + ], + }, + ...Array.from({ length: 4 }, (_, offset) => ({ + name: `acp-${offset + 1}`, + files: ['examples/acp-agent/tests/acp.snapshot.ts'], + scenarioShard: `${offset + 1}/4`, + })), +] + +/** + * Resolve one CI lane while preserving a complete ordinary snapshot run. + * + * @param name Optional stable lane name. + * @returns An empty file list for the full suite, or one explicit CI lane. + */ +export function selectSnapshotLane(name?: string): SnapshotLane { + if (name === undefined || name === '') return { name: 'complete', files: [] } + const lane = snapshotLanes.find(candidate => candidate.name === name) + if (lane === undefined) throw new Error(`run-gates: unknown DSH_SNAPSHOT_LANE ${JSON.stringify(name)}.`) + return lane +} diff --git a/scripts/static-shards.ts b/scripts/static-shards.ts index 7376aa44d7..4133470d87 100644 --- a/scripts/static-shards.ts +++ b/scripts/static-shards.ts @@ -46,7 +46,8 @@ export const staticShards = [ 'package-readme-limitations', ], }, - { name: 'site', gateIds: ['docs-site'] }, + { name: 'site-projection', gateIds: ['docs-site-projection'] }, + { name: 'site-build', gateIds: ['docs-site-build'] }, ] as const satisfies readonly StaticShard[] /** From 7813dbcf0f227c7a7c1807e02a35c9b52a796bb8 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Tue, 21 Jul 2026 21:14:55 +0800 Subject: [PATCH 05/49] ci: bound hosted fanout --- .../2026-07-06-parallel-github-ci-gates.md | 4 +- .github/workflows/ci.yml | 70 ++++--------------- scripts/coverage-shards.ts | 17 ++--- scripts/run-gates.ts | 3 +- scripts/snapshot-shards.spec.ts | 4 +- scripts/snapshot-shards.ts | 13 ++-- 6 files changed, 26 insertions(+), 85 deletions(-) diff --git a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md index a77493d8a9..5321db9bd7 100644 --- a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md +++ b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md @@ -16,9 +16,9 @@ The artifact boundary remains load-bearing. `publint`, `verify-node-next-types`, [scripts/run-gates.ts](../../../../scripts/run-gates.ts) remains the common bounded scheduler, but GitHub supplies explicit shard names for the expensive gate families. [scripts/static-shards.ts](../../../../scripts/static-shards.ts) partitions static gates into foundation, API-contract, catalog, prose, documentation-projection, and documentation-build lanes and rejects a missing or duplicate gate assignment. Lint uses disjoint package-source, package-test, and repository-complement lanes; the complement still starts from `.` so a new top-level lint target cannot disappear between shards, and it owns the single cross-file duplication run. [scripts/coverage-shards.ts](../../../../scripts/coverage-shards.ts) assigns every workspace package to exactly one source-coverage lane; its test expands the live package tree, so a new package makes CI red until it has an owner. Each coverage lane includes only its owned source files, repeats the exhaustive companion topology test, and runs without a preceding build because the complete coverage suite passes from a tree with every generated `lib/` removed. -Snapshot replay uses three explicit multi-file lanes and four scenario partitions of the large ACP file. [scripts/snapshot-shards.ts](../../../../scripts/snapshot-shards.ts) owns that inventory, and its test discovers every file admitted by the snapshot config so a new file cannot land outside CI. Each snapshot job builds the shipped runtime while its Linux runner installs bubblewrap from the hosted image's existing package index, then runs only its assigned replay surface. CI explicitly retains the suite's bounded concurrency of five subprocesses instead of clamping it to the runner's two logical CPUs, because replay spends most of its time waiting on child protocol I/O. Fixture guards still inspect the complete ACP scenario table in every partition. Static, lint, coverage, and snapshot sharding changes only GitHub scheduling: the ordinary local package scripts still run their complete suites. +Snapshot replay uses two explicit multi-file lanes and two scenario partitions of the large ACP file. [scripts/snapshot-shards.ts](../../../../scripts/snapshot-shards.ts) owns that inventory, and its test discovers every file admitted by the snapshot config so a new file cannot land outside CI. Each snapshot job builds the shipped runtime while its Linux runner installs bubblewrap from the hosted image's existing package index, then runs only its assigned replay surface. CI explicitly retains the suite's bounded concurrency of five subprocesses instead of clamping it to the runner's two logical CPUs, because replay spends most of its time waiting on child protocol I/O. Fixture guards still inspect the complete ACP scenario table in every partition. Static, lint, coverage, and snapshot sharding changes only GitHub scheduling: the ordinary local package scripts still run their complete suites. -Artifacts use three lanes: one metadata lane for `publint`, NodeNext declarations, and compiled invariant loading, plus two Vitest shards for built-bin smoke. Each lane produces its own build before its consumers. Repeating the short build costs runner minutes but avoids an upload/download dependency and keeps each job's critical path bounded. +Artifacts use two lanes: one metadata lane for `publint`, NodeNext declarations, and compiled invariant loading, plus one built-bin smoke lane. Each lane produces its own build before its consumers. Repeating the short build costs runner minutes but avoids an upload/download dependency and keeps each job's critical path bounded. [scripts/publint-all.ts](../../../../scripts/publint-all.ts) calls publint's supported API in-process against an in-memory publication view made from each manifest's declared files and npm's mandatory metadata files. This preserves the distinction between workspace files and published files without spawning a package-manager pack command 103 times. [scripts/verify-built-package-invariants.mjs](../../../../scripts/verify-built-package-invariants.mjs) stages those structurally validated manifest-declared `lib/` files below the real package, then imports the compiled self-reference through plain Node and Cordis Loader normalization. A companion that reaches an undeclared runtime chunk still fails. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d78cb52af4..cb08cefae7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,16 +82,16 @@ jobs: gate_concurrency: '1' coverage_max_workers: '4' coverage_shard: core - - lane: coverage-models + - lane: coverage-models-protocol command: pnpm run check:ci:coverage gate_concurrency: '1' coverage_max_workers: '4' - coverage_shard: models - - lane: coverage-sdk + coverage_shard: models-protocol + - lane: coverage-sdk-codex command: pnpm run check:ci:coverage gate_concurrency: '1' coverage_max_workers: '4' - coverage_shard: sdk + coverage_shard: sdk-codex - lane: coverage-interfaces command: pnpm run check:ci:coverage gate_concurrency: '1' @@ -122,36 +122,16 @@ jobs: gate_concurrency: '1' coverage_max_workers: '4' coverage_shard: repository - - lane: coverage-scripts + - lane: coverage-integrations-claude command: pnpm run check:ci:coverage gate_concurrency: '1' coverage_max_workers: '4' - coverage_shard: scripts - - lane: coverage-integrations + coverage_shard: integrations-claude + - lane: coverage-session-scripts command: pnpm run check:ci:coverage gate_concurrency: '1' coverage_max_workers: '4' - coverage_shard: integrations - - lane: coverage-session-state - command: pnpm run check:ci:coverage - gate_concurrency: '1' - coverage_max_workers: '4' - coverage_shard: session-state - - lane: coverage-hooks-claude - command: pnpm run check:ci:coverage - gate_concurrency: '1' - coverage_max_workers: '4' - coverage_shard: hooks-claude - - lane: coverage-hook-protocol - command: pnpm run check:ci:coverage - gate_concurrency: '1' - coverage_max_workers: '4' - coverage_shard: hook-protocol - - lane: coverage-hooks-codex - command: pnpm run check:ci:coverage - gate_concurrency: '1' - coverage_max_workers: '4' - coverage_shard: hooks-codex + coverage_shard: session-scripts - lane: coverage-capabilities command: pnpm run check:ci:coverage gate_concurrency: '1' @@ -163,12 +143,6 @@ jobs: snapshot_lane: support snapshot_max_concurrency: '5' snapshot_prebuilt: '1' - - lane: snapshot-demos - command: pnpm run check:ci:snapshot - gate_concurrency: '1' - snapshot_lane: demos - snapshot_max_concurrency: '5' - snapshot_prebuilt: '1' - lane: snapshot-agents command: pnpm run check:ci:snapshot gate_concurrency: '1' @@ -187,31 +161,15 @@ jobs: snapshot_lane: acp-2 snapshot_max_concurrency: '5' snapshot_prebuilt: '1' - - lane: snapshot-acp-3 - command: pnpm run check:ci:snapshot - gate_concurrency: '1' - snapshot_lane: acp-3 - snapshot_max_concurrency: '5' - snapshot_prebuilt: '1' - - lane: snapshot-acp-4 - command: pnpm run check:ci:snapshot - gate_concurrency: '1' - snapshot_lane: acp-4 - snapshot_max_concurrency: '5' - snapshot_prebuilt: '1' - lane: artifacts-metadata command: pnpm run check:ci:artifacts gate_concurrency: '3' publint_concurrency: '8' artifact_shard: metadata - - lane: artifacts-smoke-1 + - lane: artifacts-smoke command: pnpm run check:ci:artifacts gate_concurrency: '1' - artifact_shard: smoke-1 - - lane: artifacts-smoke-2 - command: pnpm run check:ci:artifacts - gate_concurrency: '1' - artifact_shard: smoke-2 + artifact_shard: smoke steps: - uses: actions/checkout@v6 @@ -409,14 +367,10 @@ jobs: gate_concurrency: '3' publint_concurrency: '8' artifact_shard: metadata - - lane: artifacts-smoke-1 + - lane: artifacts-smoke command: pnpm run check:ci:artifacts gate_concurrency: '1' - artifact_shard: smoke-1 - - lane: artifacts-smoke-2 - command: pnpm run check:ci:artifacts - gate_concurrency: '1' - artifact_shard: smoke-2 + artifact_shard: smoke steps: - uses: actions/checkout@v6 diff --git a/scripts/coverage-shards.ts b/scripts/coverage-shards.ts index 5ac8f999f3..f866bbb04d 100644 --- a/scripts/coverage-shards.ts +++ b/scripts/coverage-shards.ts @@ -17,8 +17,8 @@ export const coverageShards = [ packageRoots: ['core', 'context'], extraTestRoots: ['packages/examples/cli-demo/tests'], }, - { name: 'models', packageRoots: ['llm', 'compact'] }, - { name: 'sdk', packageRoots: ['sdk'] }, + { name: 'models-protocol', packageRoots: ['llm', 'compact', 'hooks/hook-protocol'] }, + { name: 'sdk-codex', packageRoots: ['sdk', 'hooks/hooks-codex'] }, { name: 'interfaces', packageRoots: ['ui', 'examples', 'goal'], @@ -42,18 +42,11 @@ export const coverageShards = [ ], }, { - name: 'scripts', - packageRoots: ['support/invariants'], + name: 'session-scripts', + packageRoots: ['session-persistence', 'session-query', 'support/invariants'], extraTestRoots: ['scripts'], }, - { name: 'integrations', packageRoots: ['lsp', 'mcp'] }, - { - name: 'session-state', - packageRoots: ['session-persistence', 'session-query'], - }, - { name: 'hook-protocol', packageRoots: ['hooks/hook-protocol'] }, - { name: 'hooks-claude', packageRoots: ['hooks/hooks-claude'] }, - { name: 'hooks-codex', packageRoots: ['hooks/hooks-codex'] }, + { name: 'integrations-claude', packageRoots: ['lsp', 'mcp', 'hooks/hooks-claude'] }, { name: 'capabilities', packageRoots: ['web', 'skill', 'spill', 'util', 'guard', 'todo', 'timeout'], diff --git a/scripts/run-gates.ts b/scripts/run-gates.ts index e45a254905..efd94c92a0 100644 --- a/scripts/run-gates.ts +++ b/scripts/run-gates.ts @@ -263,8 +263,7 @@ function ciArtifactGates(): Gate[] { builtPackageInvariantsGate(['build']), ] if (shard === 'metadata') return metadataGates - if (shard === 'smoke-1') return [pnpmScript('build', 'build'), builtBinSmokeGate('1/2')] - if (shard === 'smoke-2') return [pnpmScript('build', 'build'), builtBinSmokeGate('2/2')] + if (shard === 'smoke') return [pnpmScript('build', 'build'), builtBinSmokeGate()] if (shard !== undefined && shard !== '') { throw new Error(`run-gates: unknown DSH_ARTIFACT_SHARD ${JSON.stringify(shard)}.`) } diff --git a/scripts/snapshot-shards.spec.ts b/scripts/snapshot-shards.spec.ts index dc4bef62ad..9f3c4e7cbd 100644 --- a/scripts/snapshot-shards.spec.ts +++ b/scripts/snapshot-shards.spec.ts @@ -26,10 +26,10 @@ describe('snapshot lanes', () => { expect(new Set(ordinary).size).toBe(ordinary.length) expect(acp.map(lane => lane.files)).toEqual(Array.from( - { length: 4 }, + { length: 2 }, () => ['examples/acp-agent/tests/acp.snapshot.ts'], )) - expect(acp.map(lane => lane.scenarioShard)).toEqual(['1/4', '2/4', '3/4', '4/4']) + expect(acp.map(lane => lane.scenarioShard)).toEqual(['1/2', '2/2']) expect([...ordinary, 'examples/acp-agent/tests/acp.snapshot.ts'].sort()).toEqual(discovered) }) diff --git a/scripts/snapshot-shards.ts b/scripts/snapshot-shards.ts index a6126f23c2..3106297a2e 100644 --- a/scripts/snapshot-shards.ts +++ b/scripts/snapshot-shards.ts @@ -16,27 +16,22 @@ export const snapshotLanes: readonly SnapshotLane[] = [ name: 'support', files: [ 'packages/sdk/scripts/tests/config.snapshot.ts', - 'packages/ui/tui/tests/tui.snapshot.ts', - ], - }, - { - name: 'demos', - files: [ - 'examples/tui-agent/tests/tui.snapshot.ts', 'packages/sdk/create-sdk/tests/create.snapshot.ts', + 'packages/ui/tui/tests/tui.snapshot.ts', ], }, { name: 'agents', files: [ + 'examples/tui-agent/tests/tui.snapshot.ts', 'examples/acp-agent/tests/goal.snapshot.ts', 'examples/headless-agent/tests/headless.snapshot.ts', ], }, - ...Array.from({ length: 4 }, (_, offset) => ({ + ...Array.from({ length: 2 }, (_, offset) => ({ name: `acp-${offset + 1}`, files: ['examples/acp-agent/tests/acp.snapshot.ts'], - scenarioShard: `${offset + 1}/4`, + scenarioShard: `${offset + 1}/2`, })), ] From ca0c9a7c48ec7f97ecc228e20e49ebfca35cf5b4 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Tue, 21 Jul 2026 21:28:49 +0800 Subject: [PATCH 06/49] ci: restore hosted lane headroom --- .../2026-07-06-parallel-github-ci-gates.md | 4 ++- .github/workflows/ci.yml | 31 +++++++++++-------- package.json | 1 + scripts/run-gates.ts | 10 ++++-- scripts/snapshot-shards.spec.ts | 4 +-- scripts/snapshot-shards.ts | 11 ++----- scripts/static-shards.ts | 2 +- 7 files changed, 36 insertions(+), 27 deletions(-) diff --git a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md index 5321db9bd7..22d3b1ad5c 100644 --- a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md +++ b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md @@ -16,7 +16,9 @@ The artifact boundary remains load-bearing. `publint`, `verify-node-next-types`, [scripts/run-gates.ts](../../../../scripts/run-gates.ts) remains the common bounded scheduler, but GitHub supplies explicit shard names for the expensive gate families. [scripts/static-shards.ts](../../../../scripts/static-shards.ts) partitions static gates into foundation, API-contract, catalog, prose, documentation-projection, and documentation-build lanes and rejects a missing or duplicate gate assignment. Lint uses disjoint package-source, package-test, and repository-complement lanes; the complement still starts from `.` so a new top-level lint target cannot disappear between shards, and it owns the single cross-file duplication run. [scripts/coverage-shards.ts](../../../../scripts/coverage-shards.ts) assigns every workspace package to exactly one source-coverage lane; its test expands the live package tree, so a new package makes CI red until it has an owner. Each coverage lane includes only its owned source files, repeats the exhaustive companion topology test, and runs without a preceding build because the complete coverage suite passes from a tree with every generated `lib/` removed. -Snapshot replay uses two explicit multi-file lanes and two scenario partitions of the large ACP file. [scripts/snapshot-shards.ts](../../../../scripts/snapshot-shards.ts) owns that inventory, and its test discovers every file admitted by the snapshot config so a new file cannot land outside CI. Each snapshot job builds the shipped runtime while its Linux runner installs bubblewrap from the hosted image's existing package index, then runs only its assigned replay surface. CI explicitly retains the suite's bounded concurrency of five subprocesses instead of clamping it to the runner's two logical CPUs, because replay spends most of its time waiting on child protocol I/O. Fixture guards still inspect the complete ACP scenario table in every partition. Static, lint, coverage, and snapshot sharding changes only GitHub scheduling: the ordinary local package scripts still run their complete suites. +Snapshot replay uses one explicit multi-file lane and four scenario partitions of the large ACP file. [scripts/snapshot-shards.ts](../../../../scripts/snapshot-shards.ts) owns that inventory, and its test discovers every file admitted by the snapshot config so a new file cannot land outside CI. Each snapshot job builds the shipped runtime while its Linux runner installs bubblewrap from the hosted image's existing package index, then runs only its assigned replay surface. CI explicitly retains the suite's bounded concurrency of five subprocesses instead of clamping it to the runner's two logical CPUs, because replay spends most of its time waiting on child protocol I/O. Fixture guards still inspect the complete ACP scenario table in every partition. Static, lint, coverage, and snapshot sharding changes only GitHub scheduling: the ordinary local package scripts still run their complete suites. + +Cold standalone documentation typechecking rebuilds the complete project-reference graph, so the API-contract lane now builds once and checks Markdown blocks against those declarations. The Linux documentation lane uses VitePress's MPA build to retain page rendering and dead-link validation inside the one-minute budget; the blocking Windows build also produces the normal production SPA, preserving the shipped-site check under its three-minute budget. Artifacts use two lanes: one metadata lane for `publint`, NodeNext declarations, and compiled invariant loading, plus one built-bin smoke lane. Each lane produces its own build before its consumers. Repeating the short build costs runner minutes but avoids an upload/download dependency and keeps each job's critical path bounded. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cb08cefae7..9c98781cc6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,7 +56,7 @@ jobs: command: pnpm run check:ci:static gate_concurrency: '1' static_shard: site-projection - - lane: static-site-build + - lane: static-site-mpa command: pnpm run check:ci:static gate_concurrency: '1' static_shard: site-build @@ -137,16 +137,10 @@ jobs: gate_concurrency: '1' coverage_max_workers: '4' coverage_shard: capabilities - - lane: snapshot-support + - lane: snapshot-ordinary command: pnpm run check:ci:snapshot gate_concurrency: '1' - snapshot_lane: support - snapshot_max_concurrency: '5' - snapshot_prebuilt: '1' - - lane: snapshot-agents - command: pnpm run check:ci:snapshot - gate_concurrency: '1' - snapshot_lane: agents + snapshot_lane: ordinary snapshot_max_concurrency: '5' snapshot_prebuilt: '1' - lane: snapshot-acp-1 @@ -161,6 +155,18 @@ jobs: snapshot_lane: acp-2 snapshot_max_concurrency: '5' snapshot_prebuilt: '1' + - lane: snapshot-acp-3 + command: pnpm run check:ci:snapshot + gate_concurrency: '1' + snapshot_lane: acp-3 + snapshot_max_concurrency: '5' + snapshot_prebuilt: '1' + - lane: snapshot-acp-4 + command: pnpm run check:ci:snapshot + gate_concurrency: '1' + snapshot_lane: acp-4 + snapshot_max_concurrency: '5' + snapshot_prebuilt: '1' - lane: artifacts-metadata command: pnpm run check:ci:artifacts gate_concurrency: '3' @@ -314,6 +320,9 @@ jobs: - name: Build (tsc -b + tsdown) run: pnpm run build + - name: Build documentation site (production SPA) + run: pnpm run docs:build + # Observational, non-blocking Windows static, lint, and artifact lanes. Coverage # and snapshot stay Linux-only until their platform-specific runtime failures # have dedicated support. Run the gates from native PowerShell: an MSYS parent @@ -354,10 +363,6 @@ jobs: command: pnpm run check:ci:static gate_concurrency: '1' static_shard: site-projection - - lane: static-site-build - command: pnpm run check:ci:static - gate_concurrency: '1' - static_shard: site-build - lane: lint command: pnpm run check:ci:lint gate_concurrency: '1' diff --git a/package.json b/package.json index c3998bad0e..9dca83dc6a 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,7 @@ "verify-doc-budgets": "tsx scripts/verify-doc-budgets.ts", "docs:dev": "pnpm --filter @deepseek-ai/website run dev", "docs:build": "pnpm --filter @deepseek-ai/website run build", + "docs:build:mpa": "pnpm --filter @deepseek-ai/website exec vitepress build . --mpa", "docs:preview": "pnpm --filter @deepseek-ai/website run preview", "docs:check": "pnpm exec vitest run scripts/project-doc-site.spec.ts && pnpm run docs:build", "website:dev": "pnpm run docs:dev", diff --git a/scripts/run-gates.ts b/scripts/run-gates.ts index efd94c92a0..504dc26ba7 100644 --- a/scripts/run-gates.ts +++ b/scripts/run-gates.ts @@ -244,7 +244,12 @@ function ciStaticGates(): Gate[] { pnpmScript('constraints', 'constraints'), pnpmScript('package-invariants', 'verify-package-invariants', { label: 'package invariants' }), pnpmScript('cordis-config', 'verify-cordis-config', { label: 'Cordis config' }), - ...docSyncLeafGates(), + pnpmScript('build', 'build'), + ...docSyncLeafGates({ + docTypecheckNeeds: ['build'], + docTypecheckEnv: { DSH_DOC_TYPECHECK_USE_BUILD_OUTPUT: '1' }, + docsBuildScript: 'docs:build:mpa', + }), pnpmScript('module-graph', 'verify-module-graph', { label: 'module graph' }), pnpmScript('knip', 'knip'), ] @@ -366,6 +371,7 @@ function hygieneLeafGates(options: { artifactNeeds?: string[] } = {}): Gate[] { function docSyncLeafGates(options: { docTypecheckNeeds?: string[] docTypecheckEnv?: Record + docsBuildScript?: 'docs:build' | 'docs:build:mpa' } = {}): Gate[] { const docTypecheckOptions: Partial = {} if (options.docTypecheckNeeds !== undefined) docTypecheckOptions.needs = options.docTypecheckNeeds @@ -396,7 +402,7 @@ function docSyncLeafGates(options: { label: 'documentation projection', }), // Keep the VitePress build itself in one gate because projection rewrites website/.generated. - pnpmScript('docs-site-build', 'docs:build', { label: 'documentation build' }), + pnpmScript('docs-site-build', options.docsBuildScript ?? 'docs:build', { label: 'documentation build' }), pnpmScript('package-readme-limitations', 'verify-package-readme-limitations', { label: 'package README limitations' }), ] } diff --git a/scripts/snapshot-shards.spec.ts b/scripts/snapshot-shards.spec.ts index 9f3c4e7cbd..dc4bef62ad 100644 --- a/scripts/snapshot-shards.spec.ts +++ b/scripts/snapshot-shards.spec.ts @@ -26,10 +26,10 @@ describe('snapshot lanes', () => { expect(new Set(ordinary).size).toBe(ordinary.length) expect(acp.map(lane => lane.files)).toEqual(Array.from( - { length: 2 }, + { length: 4 }, () => ['examples/acp-agent/tests/acp.snapshot.ts'], )) - expect(acp.map(lane => lane.scenarioShard)).toEqual(['1/2', '2/2']) + expect(acp.map(lane => lane.scenarioShard)).toEqual(['1/4', '2/4', '3/4', '4/4']) expect([...ordinary, 'examples/acp-agent/tests/acp.snapshot.ts'].sort()).toEqual(discovered) }) diff --git a/scripts/snapshot-shards.ts b/scripts/snapshot-shards.ts index 3106297a2e..a72a4e0e77 100644 --- a/scripts/snapshot-shards.ts +++ b/scripts/snapshot-shards.ts @@ -13,25 +13,20 @@ export interface SnapshotLane { /** Exhaustive file ownership plus scenario partitions for the large ACP suite. */ export const snapshotLanes: readonly SnapshotLane[] = [ { - name: 'support', + name: 'ordinary', files: [ 'packages/sdk/scripts/tests/config.snapshot.ts', 'packages/sdk/create-sdk/tests/create.snapshot.ts', 'packages/ui/tui/tests/tui.snapshot.ts', - ], - }, - { - name: 'agents', - files: [ 'examples/tui-agent/tests/tui.snapshot.ts', 'examples/acp-agent/tests/goal.snapshot.ts', 'examples/headless-agent/tests/headless.snapshot.ts', ], }, - ...Array.from({ length: 2 }, (_, offset) => ({ + ...Array.from({ length: 4 }, (_, offset) => ({ name: `acp-${offset + 1}`, files: ['examples/acp-agent/tests/acp.snapshot.ts'], - scenarioShard: `${offset + 1}/2`, + scenarioShard: `${offset + 1}/4`, })), ] diff --git a/scripts/static-shards.ts b/scripts/static-shards.ts index 4133470d87..4f495b5fda 100644 --- a/scripts/static-shards.ts +++ b/scripts/static-shards.ts @@ -23,7 +23,7 @@ export const staticShards = [ }, { name: 'api-contracts', - gateIds: ['doc-typecheck', 'cordis-api', 'export-jsdoc', 'scoped-events', 'type-equivalence'], + gateIds: ['build', 'doc-typecheck', 'cordis-api', 'export-jsdoc', 'scoped-events', 'type-equivalence'], }, { name: 'catalogs', From 89add445fe7571728f7b19450963a9502c4480c5 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Tue, 21 Jul 2026 21:39:51 +0800 Subject: [PATCH 07/49] ci: split boundary lanes --- .../2026-07-06-parallel-github-ci-gates.md | 6 +-- .github/workflows/ci.yml | 40 +++++++++++-------- scripts/static-shards.spec.ts | 10 +++++ scripts/static-shards.ts | 20 +++++++--- 4 files changed, 51 insertions(+), 25 deletions(-) diff --git a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md index 22d3b1ad5c..f80568510e 100644 --- a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md +++ b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md @@ -14,11 +14,11 @@ The artifact boundary remains load-bearing. `publint`, `verify-node-next-types`, [CI](../../../../.github/workflows/ci.yml) bounds every non-Windows job to one minute and every Windows job to three minutes. The timeout is an executable regression ceiling; the lane design leaves headroom below it rather than treating a timeout as normal control flow. -[scripts/run-gates.ts](../../../../scripts/run-gates.ts) remains the common bounded scheduler, but GitHub supplies explicit shard names for the expensive gate families. [scripts/static-shards.ts](../../../../scripts/static-shards.ts) partitions static gates into foundation, API-contract, catalog, prose, documentation-projection, and documentation-build lanes and rejects a missing or duplicate gate assignment. Lint uses disjoint package-source, package-test, and repository-complement lanes; the complement still starts from `.` so a new top-level lint target cannot disappear between shards, and it owns the single cross-file duplication run. [scripts/coverage-shards.ts](../../../../scripts/coverage-shards.ts) assigns every workspace package to exactly one source-coverage lane; its test expands the live package tree, so a new package makes CI red until it has an owner. Each coverage lane includes only its owned source files, repeats the exhaustive companion topology test, and runs without a preceding build because the complete coverage suite passes from a tree with every generated `lib/` removed. +[scripts/run-gates.ts](../../../../scripts/run-gates.ts) remains the common bounded scheduler, but GitHub supplies explicit shard names for the expensive gate families. [scripts/static-shards.ts](../../../../scripts/static-shards.ts) partitions static gates into foundation, documentation-type, API-contract, catalog, prose, documentation-projection, and documentation-build lanes and rejects a missing or duplicate gate assignment. Lint uses disjoint package-source, package-test, and repository-complement lanes; the complement still starts from `.` so a new top-level lint target cannot disappear between shards, and it owns the single cross-file duplication run. [scripts/coverage-shards.ts](../../../../scripts/coverage-shards.ts) assigns every workspace package to exactly one source-coverage lane; its test expands the live package tree, so a new package makes CI red until it has an owner. Each coverage lane includes only its owned source files, repeats the exhaustive companion topology test, and runs without a preceding build because the complete coverage suite passes from a tree with every generated `lib/` removed. Snapshot replay uses one explicit multi-file lane and four scenario partitions of the large ACP file. [scripts/snapshot-shards.ts](../../../../scripts/snapshot-shards.ts) owns that inventory, and its test discovers every file admitted by the snapshot config so a new file cannot land outside CI. Each snapshot job builds the shipped runtime while its Linux runner installs bubblewrap from the hosted image's existing package index, then runs only its assigned replay surface. CI explicitly retains the suite's bounded concurrency of five subprocesses instead of clamping it to the runner's two logical CPUs, because replay spends most of its time waiting on child protocol I/O. Fixture guards still inspect the complete ACP scenario table in every partition. Static, lint, coverage, and snapshot sharding changes only GitHub scheduling: the ordinary local package scripts still run their complete suites. -Cold standalone documentation typechecking rebuilds the complete project-reference graph, so the API-contract lane now builds once and checks Markdown blocks against those declarations. The Linux documentation lane uses VitePress's MPA build to retain page rendering and dead-link validation inside the one-minute budget; the blocking Windows build also produces the normal production SPA, preserving the shipped-site check under its three-minute budget. +Cold standalone documentation typechecking rebuilds the complete project-reference graph, so a dedicated documentation-type lane builds once and checks Markdown blocks against those declarations. The Linux documentation lane uses VitePress's MPA build to retain page rendering and dead-link validation inside the one-minute budget; the blocking Windows build also produces the normal production SPA, preserving the shipped-site check under its three-minute budget. Artifacts use two lanes: one metadata lane for `publint`, NodeNext declarations, and compiled invariant loading, plus one built-bin smoke lane. Each lane produces its own build before its consumers. Repeating the short build costs runner minutes but avoids an upload/download dependency and keeps each job's critical path bounded. @@ -26,7 +26,7 @@ Artifacts use two lanes: one metadata lane for `publint`, NodeNext declarations, Compatibility lanes run the source worker and Zstandard runtime smokes on every advertised Node line. TypeScript checks the source graph once in a dedicated primary Node 24 lane; repeating the same compiler analysis in runtime compatibility jobs added time without runtime-specific signal. -The workflow caches the pnpm store, preserves native PowerShell for Windows measurements, and retains one aggregate `all checks passed` status for branch protection. Windows build remains blocking; the wider Windows static, lint, and artifact matrix remains observational while carrying the same three-minute ceiling. +The workflow caches the pnpm store, preserves native PowerShell for Windows measurements, and retains one aggregate `all checks passed` status for branch protection. Windows reuses the three exhaustive lint partitions and groups foundation, catalog, and prose gates behind one shared runner setup; only scheduling differs from the Linux partitions. Windows build remains blocking; the wider Windows static, lint, and artifact matrix remains observational while carrying the same three-minute ceiling. ## Alternatives considered diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c98781cc6..5abe8a7371 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,6 +40,10 @@ jobs: command: pnpm run check:ci:static gate_concurrency: '4' static_shard: foundation + - lane: static-doc-types + command: pnpm run check:ci:static + gate_concurrency: '1' + static_shard: doc-types - lane: static-api-contracts command: pnpm run check:ci:static gate_concurrency: '4' @@ -337,36 +341,40 @@ jobs: DSH_GATE_CONCURRENCY: ${{ matrix.gate_concurrency }} DSH_PUBLINT_CONCURRENCY: ${{ matrix.publint_concurrency }} DSH_ARTIFACT_SHARD: ${{ matrix.artifact_shard }} + DSH_LINT_SHARD: ${{ matrix.lint_shard }} DSH_STATIC_SHARD: ${{ matrix.static_shard }} DSH_ESLINT_CACHE: ${{ matrix.eslint_cache }} strategy: fail-fast: false matrix: include: - - lane: static-foundation + - lane: static-general command: pnpm run check:ci:static gate_concurrency: '4' - static_shard: foundation + static_shard: 'foundation,catalogs,prose' + - lane: static-doc-types + command: pnpm run check:ci:static + gate_concurrency: '1' + static_shard: doc-types - lane: static-api-contracts command: pnpm run check:ci:static gate_concurrency: '4' static_shard: api-contracts - - lane: static-catalogs - command: pnpm run check:ci:static - gate_concurrency: '4' - static_shard: catalogs - - lane: static-prose - command: pnpm run check:ci:static - gate_concurrency: '4' - static_shard: prose - - lane: static-site-projection - command: pnpm run check:ci:static - gate_concurrency: '1' - static_shard: site-projection - - lane: lint + - lane: lint-package-sources command: pnpm run check:ci:lint gate_concurrency: '1' eslint_cache: '1' + lint_shard: package-sources + - lane: lint-package-tests + command: pnpm run check:ci:lint + gate_concurrency: '1' + eslint_cache: '1' + lint_shard: package-tests + - lane: lint-repository + command: pnpm run check:ci:lint + gate_concurrency: '1' + eslint_cache: '1' + lint_shard: repository - lane: artifacts-metadata command: pnpm run check:ci:artifacts gate_concurrency: '3' @@ -410,7 +418,7 @@ jobs: run: pnpm install --frozen-lockfile - uses: actions/cache@v4 - if: matrix.lane == 'lint' + if: startsWith(matrix.lane, 'lint-') with: path: .cache/eslint key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-${{ hashFiles('pnpm-lock.yaml', 'eslint.config.mjs', 'tsconfig.json', 'packages/*/*/tsconfig.json', 'examples/*/tsconfig.json') }} diff --git a/scripts/static-shards.spec.ts b/scripts/static-shards.spec.ts index 2c57be6dd5..df9e2922f1 100644 --- a/scripts/static-shards.spec.ts +++ b/scripts/static-shards.spec.ts @@ -8,9 +8,19 @@ describe('static gate shards', () => { expect(selectStaticGates(completeInventory, shard.name).map(gate => gate.id)).toEqual(shard.gateIds) }) + it('selects multiple lanes in gate inventory order', () => { + const selectedNames = new Set(['foundation', 'catalogs', 'prose']) + const expected = staticShards + .filter(shard => selectedNames.has(shard.name)) + .flatMap(shard => shard.gateIds) + expect(selectStaticGates(completeInventory, 'foundation,catalogs,prose').map(gate => gate.id)).toEqual(expected) + }) + it('rejects missing, duplicate, and unknown assignments', () => { expect(() => selectStaticGates(completeInventory.slice(1))).toThrow('assign every static gate exactly once') expect(() => selectStaticGates([...completeInventory, completeInventory[0]!])).toThrow('static gate IDs must be unique') expect(() => selectStaticGates(completeInventory, 'missing')).toThrow('unknown DSH_STATIC_SHARD') + expect(() => selectStaticGates(completeInventory, 'foundation,foundation')).toThrow('nonempty and unique') + expect(() => selectStaticGates(completeInventory, 'foundation,,prose')).toThrow('nonempty and unique') }) }) diff --git a/scripts/static-shards.ts b/scripts/static-shards.ts index 4f495b5fda..3476099993 100644 --- a/scripts/static-shards.ts +++ b/scripts/static-shards.ts @@ -21,9 +21,10 @@ export const staticShards = [ 'knip', ], }, + { name: 'doc-types', gateIds: ['build', 'doc-typecheck'] }, { name: 'api-contracts', - gateIds: ['build', 'doc-typecheck', 'cordis-api', 'export-jsdoc', 'scoped-events', 'type-equivalence'], + gateIds: ['cordis-api', 'export-jsdoc', 'scoped-events', 'type-equivalence'], }, { name: 'catalogs', @@ -54,8 +55,8 @@ export const staticShards = [ * Validate the complete gate partition and optionally select one lane. * * @param gates Complete static gate inventory. - * @param name Optional stable shard name. - * @returns All gates when no shard is requested, otherwise the selected lane. + * @param name Optional comma-separated stable shard names. + * @returns All gates when no shard is requested, otherwise the selected lanes in inventory order. */ export function selectStaticGates(gates: readonly T[], name?: string): T[] { const gateIds = gates.map(gate => gate.id) @@ -71,8 +72,15 @@ export function selectStaticGates(gates: readonly T[], } if (name === undefined || name === '') return [...gates] - const shard = staticShards.find(candidate => candidate.name === name) - if (shard === undefined) throw new Error(`run-gates: unknown DSH_STATIC_SHARD ${JSON.stringify(name)}.`) - const selectedIds = new Set(shard.gateIds) + const shardNames = name.split(',') + if (shardNames.some(shardName => shardName === '') || new Set(shardNames).size !== shardNames.length) { + throw new Error(`run-gates: DSH_STATIC_SHARD names must be nonempty and unique, got ${JSON.stringify(name)}.`) + } + const selectedShards = shardNames.map((shardName) => { + const shard = staticShards.find(candidate => candidate.name === shardName) + if (shard === undefined) throw new Error(`run-gates: unknown DSH_STATIC_SHARD ${JSON.stringify(shardName)}.`) + return shard + }) + const selectedIds = new Set(selectedShards.flatMap(shard => shard.gateIds)) return gates.filter(gate => selectedIds.has(gate.id)) } From ba1e34f8fbeb02da2d5e87133000e027f2086ed8 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Tue, 21 Jul 2026 21:47:38 +0800 Subject: [PATCH 08/49] test(lsp): synchronize closed-stdin fixture --- packages/lsp/lsp-local/tests/connection.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/lsp/lsp-local/tests/connection.spec.ts b/packages/lsp/lsp-local/tests/connection.spec.ts index 6d9ca6d6a3..f12fd04e84 100644 --- a/packages/lsp/lsp-local/tests/connection.spec.ts +++ b/packages/lsp/lsp-local/tests/connection.spec.ts @@ -210,8 +210,8 @@ describe('LspConnection edge behavior', () => { }) it('rejects a pending request when child stdin closes but the process stays alive', async () => { - const conn = connectScript('require("node:fs").closeSync(0); setInterval(()=>{}, 1000)') - await new Promise(resolve => setTimeout(resolve, 100)) + const conn = connectScript('require("node:fs").closeSync(0); process.stderr.write("stdin closed"); setInterval(()=>{}, 1000)') + await waitFor(() => conn.stderrTail === 'stdin closed') const timeout = new Promise((_resolve, reject) => { setTimeout(() => { reject(new Error('request timed out')) }, 1000) }) From 311cbe1e4c36b98ee6f353059315b35ff5841fbd Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Tue, 21 Jul 2026 21:49:35 +0800 Subject: [PATCH 09/49] ci: rebalance final boundary lanes --- .../2026-07-06-parallel-github-ci-gates.md | 4 ++-- .github/workflows/ci.yml | 18 ++++++++++-------- scripts/snapshot-shards.ts | 7 ++++++- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md index f80568510e..70a8a4d2a1 100644 --- a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md +++ b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md @@ -16,7 +16,7 @@ The artifact boundary remains load-bearing. `publint`, `verify-node-next-types`, [scripts/run-gates.ts](../../../../scripts/run-gates.ts) remains the common bounded scheduler, but GitHub supplies explicit shard names for the expensive gate families. [scripts/static-shards.ts](../../../../scripts/static-shards.ts) partitions static gates into foundation, documentation-type, API-contract, catalog, prose, documentation-projection, and documentation-build lanes and rejects a missing or duplicate gate assignment. Lint uses disjoint package-source, package-test, and repository-complement lanes; the complement still starts from `.` so a new top-level lint target cannot disappear between shards, and it owns the single cross-file duplication run. [scripts/coverage-shards.ts](../../../../scripts/coverage-shards.ts) assigns every workspace package to exactly one source-coverage lane; its test expands the live package tree, so a new package makes CI red until it has an owner. Each coverage lane includes only its owned source files, repeats the exhaustive companion topology test, and runs without a preceding build because the complete coverage suite passes from a tree with every generated `lib/` removed. -Snapshot replay uses one explicit multi-file lane and four scenario partitions of the large ACP file. [scripts/snapshot-shards.ts](../../../../scripts/snapshot-shards.ts) owns that inventory, and its test discovers every file admitted by the snapshot config so a new file cannot land outside CI. Each snapshot job builds the shipped runtime while its Linux runner installs bubblewrap from the hosted image's existing package index, then runs only its assigned replay surface. CI explicitly retains the suite's bounded concurrency of five subprocesses instead of clamping it to the runner's two logical CPUs, because replay spends most of its time waiting on child protocol I/O. Fixture guards still inspect the complete ACP scenario table in every partition. Static, lint, coverage, and snapshot sharding changes only GitHub scheduling: the ordinary local package scripts still run their complete suites. +Snapshot replay uses two explicit multi-file lanes and four scenario partitions of the large ACP file. [scripts/snapshot-shards.ts](../../../../scripts/snapshot-shards.ts) owns that inventory, and its test discovers every file admitted by the snapshot config so a new file cannot land outside CI. Each snapshot job builds the shipped runtime while its Linux runner installs bubblewrap from the hosted image's existing package index, then runs only its assigned replay surface. CI explicitly retains the suite's bounded concurrency of five subprocesses instead of clamping it to the runner's two logical CPUs, because replay spends most of its time waiting on child protocol I/O. Fixture guards still inspect the complete ACP scenario table in every partition. Static, lint, coverage, and snapshot sharding changes only GitHub scheduling: the ordinary local package scripts still run their complete suites. Cold standalone documentation typechecking rebuilds the complete project-reference graph, so a dedicated documentation-type lane builds once and checks Markdown blocks against those declarations. The Linux documentation lane uses VitePress's MPA build to retain page rendering and dead-link validation inside the one-minute budget; the blocking Windows build also produces the normal production SPA, preserving the shipped-site check under its three-minute budget. @@ -26,7 +26,7 @@ Artifacts use two lanes: one metadata lane for `publint`, NodeNext declarations, Compatibility lanes run the source worker and Zstandard runtime smokes on every advertised Node line. TypeScript checks the source graph once in a dedicated primary Node 24 lane; repeating the same compiler analysis in runtime compatibility jobs added time without runtime-specific signal. -The workflow caches the pnpm store, preserves native PowerShell for Windows measurements, and retains one aggregate `all checks passed` status for branch protection. Windows reuses the three exhaustive lint partitions and groups foundation, catalog, and prose gates behind one shared runner setup; only scheduling differs from the Linux partitions. Windows build remains blocking; the wider Windows static, lint, and artifact matrix remains observational while carrying the same three-minute ceiling. +The workflow caches the pnpm store, preserves native PowerShell for Windows measurements, and retains one aggregate `all checks passed` status for branch protection. Windows reuses the three exhaustive lint partitions and groups foundation/catalog/prose plus documentation-type/API-contract gates behind shared runner setups; only scheduling differs from the Linux partitions. Windows build remains blocking; the wider Windows static, lint, and artifact matrix remains observational while carrying the same three-minute ceiling. ## Alternatives considered diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5abe8a7371..82df288a23 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,10 +141,16 @@ jobs: gate_concurrency: '1' coverage_max_workers: '4' coverage_shard: capabilities - - lane: snapshot-ordinary + - lane: snapshot-support command: pnpm run check:ci:snapshot gate_concurrency: '1' - snapshot_lane: ordinary + snapshot_lane: support + snapshot_max_concurrency: '5' + snapshot_prebuilt: '1' + - lane: snapshot-agents + command: pnpm run check:ci:snapshot + gate_concurrency: '1' + snapshot_lane: agents snapshot_max_concurrency: '5' snapshot_prebuilt: '1' - lane: snapshot-acp-1 @@ -352,14 +358,10 @@ jobs: command: pnpm run check:ci:static gate_concurrency: '4' static_shard: 'foundation,catalogs,prose' - - lane: static-doc-types - command: pnpm run check:ci:static - gate_concurrency: '1' - static_shard: doc-types - - lane: static-api-contracts + - lane: static-contracts command: pnpm run check:ci:static gate_concurrency: '4' - static_shard: api-contracts + static_shard: 'doc-types,api-contracts' - lane: lint-package-sources command: pnpm run check:ci:lint gate_concurrency: '1' diff --git a/scripts/snapshot-shards.ts b/scripts/snapshot-shards.ts index a72a4e0e77..179c2e1517 100644 --- a/scripts/snapshot-shards.ts +++ b/scripts/snapshot-shards.ts @@ -13,11 +13,16 @@ export interface SnapshotLane { /** Exhaustive file ownership plus scenario partitions for the large ACP suite. */ export const snapshotLanes: readonly SnapshotLane[] = [ { - name: 'ordinary', + name: 'support', files: [ 'packages/sdk/scripts/tests/config.snapshot.ts', 'packages/sdk/create-sdk/tests/create.snapshot.ts', 'packages/ui/tui/tests/tui.snapshot.ts', + ], + }, + { + name: 'agents', + files: [ 'examples/tui-agent/tests/tui.snapshot.ts', 'examples/acp-agent/tests/goal.snapshot.ts', 'examples/headless-agent/tests/headless.snapshot.ts', From e4c56b35e805504a9f8f948fcd82317723223bf2 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Tue, 21 Jul 2026 22:00:34 +0800 Subject: [PATCH 10/49] ci: add tail-lane headroom --- .../2026-07-06-parallel-github-ci-gates.md | 2 +- .github/workflows/ci.yml | 44 +++++++------- scripts/coverage-shards.ts | 57 +++++++++++++------ scripts/snapshot-shards.spec.ts | 4 +- scripts/snapshot-shards.ts | 4 +- 5 files changed, 69 insertions(+), 42 deletions(-) diff --git a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md index 70a8a4d2a1..fbbd8c46d3 100644 --- a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md +++ b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md @@ -16,7 +16,7 @@ The artifact boundary remains load-bearing. `publint`, `verify-node-next-types`, [scripts/run-gates.ts](../../../../scripts/run-gates.ts) remains the common bounded scheduler, but GitHub supplies explicit shard names for the expensive gate families. [scripts/static-shards.ts](../../../../scripts/static-shards.ts) partitions static gates into foundation, documentation-type, API-contract, catalog, prose, documentation-projection, and documentation-build lanes and rejects a missing or duplicate gate assignment. Lint uses disjoint package-source, package-test, and repository-complement lanes; the complement still starts from `.` so a new top-level lint target cannot disappear between shards, and it owns the single cross-file duplication run. [scripts/coverage-shards.ts](../../../../scripts/coverage-shards.ts) assigns every workspace package to exactly one source-coverage lane; its test expands the live package tree, so a new package makes CI red until it has an owner. Each coverage lane includes only its owned source files, repeats the exhaustive companion topology test, and runs without a preceding build because the complete coverage suite passes from a tree with every generated `lib/` removed. -Snapshot replay uses two explicit multi-file lanes and four scenario partitions of the large ACP file. [scripts/snapshot-shards.ts](../../../../scripts/snapshot-shards.ts) owns that inventory, and its test discovers every file admitted by the snapshot config so a new file cannot land outside CI. Each snapshot job builds the shipped runtime while its Linux runner installs bubblewrap from the hosted image's existing package index, then runs only its assigned replay surface. CI explicitly retains the suite's bounded concurrency of five subprocesses instead of clamping it to the runner's two logical CPUs, because replay spends most of its time waiting on child protocol I/O. Fixture guards still inspect the complete ACP scenario table in every partition. Static, lint, coverage, and snapshot sharding changes only GitHub scheduling: the ordinary local package scripts still run their complete suites. +Snapshot replay uses two explicit multi-file lanes and six scenario partitions of the large ACP file. [scripts/snapshot-shards.ts](../../../../scripts/snapshot-shards.ts) owns that inventory, and its test discovers every file admitted by the snapshot config so a new file cannot land outside CI. Each snapshot job builds the shipped runtime while its Linux runner installs bubblewrap from the hosted image's existing package index, then runs only its assigned replay surface. CI explicitly retains the suite's bounded concurrency of five subprocesses instead of clamping it to the runner's two logical CPUs, because replay spends most of its time waiting on child protocol I/O. Fixture guards still inspect the complete ACP scenario table in every partition. Static, lint, coverage, and snapshot sharding changes only GitHub scheduling: the ordinary local package scripts still run their complete suites. Cold standalone documentation typechecking rebuilds the complete project-reference graph, so a dedicated documentation-type lane builds once and checks Markdown blocks against those declarations. The Linux documentation lane uses VitePress's MPA build to retain page rendering and dead-link validation inside the one-minute budget; the blocking Windows build also produces the normal production SPA, preserving the shipped-site check under its three-minute budget. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 82df288a23..45b8cd5407 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,21 +81,26 @@ jobs: gate_concurrency: '1' eslint_cache: '1' lint_shard: repository - - lane: coverage-core + - lane: coverage-core-loop command: pnpm run check:ci:coverage gate_concurrency: '1' coverage_max_workers: '4' - coverage_shard: core - - lane: coverage-models-protocol + coverage_shard: core-loop + - lane: coverage-state-session command: pnpm run check:ci:coverage gate_concurrency: '1' coverage_max_workers: '4' - coverage_shard: models-protocol - - lane: coverage-sdk-codex + coverage_shard: state-session + - lane: coverage-models-integrations command: pnpm run check:ci:coverage gate_concurrency: '1' coverage_max_workers: '4' - coverage_shard: sdk-codex + coverage_shard: models-integrations + - lane: coverage-sdk-capabilities + command: pnpm run check:ci:coverage + gate_concurrency: '1' + coverage_max_workers: '4' + coverage_shard: sdk-capabilities - lane: coverage-interfaces command: pnpm run check:ci:coverage gate_concurrency: '1' @@ -126,21 +131,6 @@ jobs: gate_concurrency: '1' coverage_max_workers: '4' coverage_shard: repository - - lane: coverage-integrations-claude - command: pnpm run check:ci:coverage - gate_concurrency: '1' - coverage_max_workers: '4' - coverage_shard: integrations-claude - - lane: coverage-session-scripts - command: pnpm run check:ci:coverage - gate_concurrency: '1' - coverage_max_workers: '4' - coverage_shard: session-scripts - - lane: coverage-capabilities - command: pnpm run check:ci:coverage - gate_concurrency: '1' - coverage_max_workers: '4' - coverage_shard: capabilities - lane: snapshot-support command: pnpm run check:ci:snapshot gate_concurrency: '1' @@ -177,6 +167,18 @@ jobs: snapshot_lane: acp-4 snapshot_max_concurrency: '5' snapshot_prebuilt: '1' + - lane: snapshot-acp-5 + command: pnpm run check:ci:snapshot + gate_concurrency: '1' + snapshot_lane: acp-5 + snapshot_max_concurrency: '5' + snapshot_prebuilt: '1' + - lane: snapshot-acp-6 + command: pnpm run check:ci:snapshot + gate_concurrency: '1' + snapshot_lane: acp-6 + snapshot_max_concurrency: '5' + snapshot_prebuilt: '1' - lane: artifacts-metadata command: pnpm run check:ci:artifacts gate_concurrency: '3' diff --git a/scripts/coverage-shards.ts b/scripts/coverage-shards.ts index f866bbb04d..4bcc0277c1 100644 --- a/scripts/coverage-shards.ts +++ b/scripts/coverage-shards.ts @@ -13,12 +13,44 @@ export interface CoverageShard { /** Exhaustive, non-overlapping ownership of workspace packages in coverage CI. */ export const coverageShards = [ { - name: 'core', - packageRoots: ['core', 'context'], - extraTestRoots: ['packages/examples/cli-demo/tests'], + name: 'core-loop', + packageRoots: ['core/agent', 'core/agent-loop', 'core/tools'], + }, + { + name: 'state-session', + packageRoots: [ + 'core/session', + 'core/scope', + 'core/system-prompt', + 'context', + 'session-persistence', + 'session-query', + 'support/invariants', + ], + extraTestRoots: [ + 'packages/examples/cli-demo/tests', + 'packages/llm/token-meter/tests', + 'scripts', + ], + }, + { + name: 'models-integrations', + packageRoots: ['llm', 'compact', 'hooks/hook-protocol', 'lsp', 'mcp', 'hooks/hooks-claude'], + }, + { + name: 'sdk-capabilities', + packageRoots: [ + 'sdk', + 'hooks/hooks-codex', + 'web', + 'skill', + 'spill', + 'util', + 'guard', + 'todo', + 'timeout', + ], }, - { name: 'models-protocol', packageRoots: ['llm', 'compact', 'hooks/hook-protocol'] }, - { name: 'sdk-codex', packageRoots: ['sdk', 'hooks/hooks-codex'] }, { name: 'interfaces', packageRoots: ['ui', 'examples', 'goal'], @@ -29,7 +61,10 @@ export const coverageShards = [ name: 'workflow', packageRoots: ['workflow/workflow', 'workflow/tool-workflow', 'workflow/tool-ralph'], }, - { name: 'workflow-worker', packageRoots: ['workflow/workflow-workerthread'] }, + { + name: 'workflow-worker', + packageRoots: ['workflow/workflow-workerthread'], + }, { name: 'delegation', packageRoots: ['subagent', 'tasks'] }, { name: 'repository', @@ -41,16 +76,6 @@ export const coverageShards = [ 'support/loader-smoke', ], }, - { - name: 'session-scripts', - packageRoots: ['session-persistence', 'session-query', 'support/invariants'], - extraTestRoots: ['scripts'], - }, - { name: 'integrations-claude', packageRoots: ['lsp', 'mcp', 'hooks/hooks-claude'] }, - { - name: 'capabilities', - packageRoots: ['web', 'skill', 'spill', 'util', 'guard', 'todo', 'timeout'], - }, ] as const satisfies readonly CoverageShard[] /** diff --git a/scripts/snapshot-shards.spec.ts b/scripts/snapshot-shards.spec.ts index dc4bef62ad..7cc37fb7bd 100644 --- a/scripts/snapshot-shards.spec.ts +++ b/scripts/snapshot-shards.spec.ts @@ -26,10 +26,10 @@ describe('snapshot lanes', () => { expect(new Set(ordinary).size).toBe(ordinary.length) expect(acp.map(lane => lane.files)).toEqual(Array.from( - { length: 4 }, + { length: 6 }, () => ['examples/acp-agent/tests/acp.snapshot.ts'], )) - expect(acp.map(lane => lane.scenarioShard)).toEqual(['1/4', '2/4', '3/4', '4/4']) + expect(acp.map(lane => lane.scenarioShard)).toEqual(['1/6', '2/6', '3/6', '4/6', '5/6', '6/6']) expect([...ordinary, 'examples/acp-agent/tests/acp.snapshot.ts'].sort()).toEqual(discovered) }) diff --git a/scripts/snapshot-shards.ts b/scripts/snapshot-shards.ts index 179c2e1517..fae8010261 100644 --- a/scripts/snapshot-shards.ts +++ b/scripts/snapshot-shards.ts @@ -28,10 +28,10 @@ export const snapshotLanes: readonly SnapshotLane[] = [ 'examples/headless-agent/tests/headless.snapshot.ts', ], }, - ...Array.from({ length: 4 }, (_, offset) => ({ + ...Array.from({ length: 6 }, (_, offset) => ({ name: `acp-${offset + 1}`, files: ['examples/acp-agent/tests/acp.snapshot.ts'], - scenarioShard: `${offset + 1}/4`, + scenarioShard: `${offset + 1}/6`, })), ] From 3fa5795bf47494bff879859ac99f014c2d85e9b4 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Tue, 21 Jul 2026 22:09:50 +0800 Subject: [PATCH 11/49] test(lsp): synchronize idle server eviction --- .../lsp/lsp-local/tests/lifecycle.spec.ts | 45 +++++++++++++++---- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/packages/lsp/lsp-local/tests/lifecycle.spec.ts b/packages/lsp/lsp-local/tests/lifecycle.spec.ts index 826905ed26..dd235e800f 100644 --- a/packages/lsp/lsp-local/tests/lifecycle.spec.ts +++ b/packages/lsp/lsp-local/tests/lifecycle.spec.ts @@ -1,11 +1,11 @@ -import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises' import { realpath } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { pathToFileURL, fileURLToPath } from 'node:url' import { Context } from 'cordis' -import Lsp, { type LspQueryRequest, type LspQueryResult } from '@deepseek-ai/dsh-lsp' +import Lsp, { type LspProvider, type LspQueryRequest, type LspQueryResult } from '@deepseek-ai/dsh-lsp' import { deadline } from '@deepseek-ai/dsh-timeout' import * as LspLocal from '@deepseek-ai/dsh-lsp-local' import type { LspLocalServerConfig } from '@deepseek-ai/dsh-lsp-local' @@ -38,12 +38,27 @@ function fakeServer(fakeEnv: Record = {}, overrides: Partial = {}, overrides: Partial = {}): Promise { +async function mount( + fakeEnv: Record = {}, + overrides: Partial = {}, + captureProvider?: (provider: LspProvider) => void, +): Promise { const ctx = new Context() await ctx.plugin(Lsp) - await ctx.plugin(LspLocal, { - servers: { fake: fakeServer(fakeEnv, overrides) }, - }) + const register = ctx.lsp.registerProvider.bind(ctx.lsp) + const registrationSpy = captureProvider === undefined + ? undefined + : vi.spyOn(ctx.lsp, 'registerProvider').mockImplementation((provider) => { + captureProvider(provider) + return register(provider) + }) + try { + await ctx.plugin(LspLocal, { + servers: { fake: fakeServer(fakeEnv, overrides) }, + }) + } finally { + registrationSpy?.mockRestore() + } return ctx } @@ -241,10 +256,22 @@ describe('lsp-local end to end over a fake server', () => { // The first query succeeds, then the server exits before the second arrives, leaving a dead // instance in the pool. The next query must evict-and-replace it and still succeed, rather than // failing once on the closed connection first. - const ctx = await mount({ LSP_FAKE_EXIT_AFTER_REPLY: '1', LSP_FAKE_DEF: JSON.stringify(locationJson(0)) }) + let provider: LspProvider | undefined + const ctx = await mount( + { LSP_FAKE_EXIT_AFTER_REPLY: '1', LSP_FAKE_DEF: JSON.stringify(locationJson(0)) }, + {}, + (registered) => { provider = registered }, + ) expect(await ctx.lsp.query(query('goToDefinition'))).toMatchObject({ kind: 'locations' }) - // Wait past the fixture's post-reply exit so the pooled instance is observably dead. - await new Promise(resolve => setTimeout(resolve, 60)) + if (provider === undefined) throw new Error('expected lsp-local to register a provider') + // This implementation-local test reaches the private pool only to synchronize with its actual + // close state. A fixed wall-clock sleep can expire before a CPU-starved child runs its exit timer. + const instances = (provider as unknown as { + readonly instances: ReadonlyMap + }).instances + const instance = [...instances.values()][0] + if (instance === undefined) throw new Error('expected one pooled LSP instance') + await waitFor(async () => instance.dead) expect(await ctx.lsp.query(query('goToDefinition'))).toMatchObject({ kind: 'locations' }) await ctx.fiber.dispose() }) From 06f3b246b51fb81ef57d39804f3f94d8fe85859e Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Tue, 21 Jul 2026 23:03:52 +0800 Subject: [PATCH 12/49] ci: add serial cross-platform reference runs --- .../2026-07-06-parallel-github-ci-gates.md | 8 +- ...rial-cross-platform-ci-reference.i18n.yaml | 6 + ...7-21-serial-cross-platform-ci-reference.md | 39 ++++++ ...1-serial-cross-platform-ci-reference.zh.md | 39 ++++++ .github/workflows/ci.yml | 112 ++++++++++++++++-- 5 files changed, 193 insertions(+), 11 deletions(-) create mode 100644 .agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.i18n.yaml create mode 100644 .agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.md create mode 100644 .agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.zh.md diff --git a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md index fbbd8c46d3..b657fd1e93 100644 --- a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md +++ b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md @@ -12,13 +12,13 @@ The artifact boundary remains load-bearing. `publint`, `verify-node-next-types`, ## Decision -[CI](../../../../.github/workflows/ci.yml) bounds every non-Windows job to one minute and every Windows job to three minutes. The timeout is an executable regression ceiling; the lane design leaves headroom below it rather than treating a timeout as normal control flow. +[CI](../../../../.github/workflows/ci.yml) treats one minute for non-Windows jobs and three minutes for Windows jobs as observed performance targets, not cancellation deadlines. Hosted-runner variance should leave complete timing evidence and useful failure logs instead of cancelling an otherwise-correct gate. The [serial cross-platform CI reference](2026-07-21-serial-cross-platform-ci-reference.md) independently runs the complete unsharded primary Node aggregate on Linux, macOS, and Windows so the optimized lane inventory is not its own completeness oracle. [scripts/run-gates.ts](../../../../scripts/run-gates.ts) remains the common bounded scheduler, but GitHub supplies explicit shard names for the expensive gate families. [scripts/static-shards.ts](../../../../scripts/static-shards.ts) partitions static gates into foundation, documentation-type, API-contract, catalog, prose, documentation-projection, and documentation-build lanes and rejects a missing or duplicate gate assignment. Lint uses disjoint package-source, package-test, and repository-complement lanes; the complement still starts from `.` so a new top-level lint target cannot disappear between shards, and it owns the single cross-file duplication run. [scripts/coverage-shards.ts](../../../../scripts/coverage-shards.ts) assigns every workspace package to exactly one source-coverage lane; its test expands the live package tree, so a new package makes CI red until it has an owner. Each coverage lane includes only its owned source files, repeats the exhaustive companion topology test, and runs without a preceding build because the complete coverage suite passes from a tree with every generated `lib/` removed. Snapshot replay uses two explicit multi-file lanes and six scenario partitions of the large ACP file. [scripts/snapshot-shards.ts](../../../../scripts/snapshot-shards.ts) owns that inventory, and its test discovers every file admitted by the snapshot config so a new file cannot land outside CI. Each snapshot job builds the shipped runtime while its Linux runner installs bubblewrap from the hosted image's existing package index, then runs only its assigned replay surface. CI explicitly retains the suite's bounded concurrency of five subprocesses instead of clamping it to the runner's two logical CPUs, because replay spends most of its time waiting on child protocol I/O. Fixture guards still inspect the complete ACP scenario table in every partition. Static, lint, coverage, and snapshot sharding changes only GitHub scheduling: the ordinary local package scripts still run their complete suites. -Cold standalone documentation typechecking rebuilds the complete project-reference graph, so a dedicated documentation-type lane builds once and checks Markdown blocks against those declarations. The Linux documentation lane uses VitePress's MPA build to retain page rendering and dead-link validation inside the one-minute budget; the blocking Windows build also produces the normal production SPA, preserving the shipped-site check under its three-minute budget. +Cold standalone documentation typechecking rebuilds the complete project-reference graph, so a dedicated documentation-type lane builds once and checks Markdown blocks against those declarations. The Linux documentation lane uses VitePress's MPA build to retain page rendering and dead-link validation within the observed non-Windows target; the blocking Windows build also produces the normal production SPA, preserving the shipped-site check within the observed Windows target. Artifacts use two lanes: one metadata lane for `publint`, NodeNext declarations, and compiled invariant loading, plus one built-bin smoke lane. Each lane produces its own build before its consumers. Repeating the short build costs runner minutes but avoids an upload/download dependency and keeps each job's critical path bounded. @@ -26,11 +26,11 @@ Artifacts use two lanes: one metadata lane for `publint`, NodeNext declarations, Compatibility lanes run the source worker and Zstandard runtime smokes on every advertised Node line. TypeScript checks the source graph once in a dedicated primary Node 24 lane; repeating the same compiler analysis in runtime compatibility jobs added time without runtime-specific signal. -The workflow caches the pnpm store, preserves native PowerShell for Windows measurements, and retains one aggregate `all checks passed` status for branch protection. Windows reuses the three exhaustive lint partitions and groups foundation/catalog/prose plus documentation-type/API-contract gates behind shared runner setups; only scheduling differs from the Linux partitions. Windows build remains blocking; the wider Windows static, lint, and artifact matrix remains observational while carrying the same three-minute ceiling. +The workflow caches the pnpm store, preserves native PowerShell for Windows measurements, and retains one aggregate `all checks passed` status for branch protection. Windows reuses the three exhaustive lint partitions and groups foundation/catalog/prose plus documentation-type/API-contract gates behind shared runner setups; only scheduling differs from the Linux partitions. Windows build remains blocking, while the wider Windows static, lint, and artifact matrix remains observational. ## Alternatives considered -- **Keep the broad lanes and raise timeouts** - minimizes workflow YAML, but it preserves the measured multi-minute feedback loop and offers no regression budget. +- **Keep the broad lanes** - minimizes workflow YAML, but it preserves the measured multi-minute feedback loop. - **Run every leaf gate as a separate GitHub job** - maximizes fan-out, but short generators and prose checks would spend more time preparing a runner than checking the repository. - **Upload one build to artifact consumers** - avoids repeated compilation, but upload/download and dependency scheduling lengthen wall time; the clean build is short enough to repeat inside bounded lanes. - **Keep package-manager packing in both publication gates** - delegates inventory selection to pnpm, but repeats more than 200 package-manager processes. The manifest structural gate plus publication-view fixtures make the optimized inventory contract explicit and fail on an on-disk but unpublished dependency. diff --git a/.agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.i18n.yaml b/.agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.i18n.yaml new file mode 100644 index 0000000000..5871fa1b07 --- /dev/null +++ b/.agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.i18n.yaml @@ -0,0 +1,6 @@ +# Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each +# side as of the last confirmed-consistent state. Both languages carry equal authority; +# after editing either side, bring the other along and re-record with: +# pnpm run verify-translation-pairing --write +2026-07-21-serial-cross-platform-ci-reference.md: e2cb9dbc5f8778b5a28236cb135f6dcbb979d38b +2026-07-21-serial-cross-platform-ci-reference.zh.md: 85c3403f1acb78e60e3a811503b5b264f84242c7 diff --git a/.agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.md b/.agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.md new file mode 100644 index 0000000000..e2cb9dbc5f --- /dev/null +++ b/.agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.md @@ -0,0 +1,39 @@ +# Agent Note: Serial cross-platform CI reference + +Status: implemented + +English | [中文](2026-07-21-serial-cross-platform-ci-reference.zh.md) + +## Problem + +The pull-request workflow reaches its latency targets by partitioning static checks, lint, coverage, snapshot replay, and artifact validation across explicit GitHub jobs. Those partitions are exhaustively checked in code, but the optimized workflow still should not be its own only completeness oracle: a defect shared by shard selection and its inventory test could omit work while every optimized lane stays green. + +Encoding the one-minute non-Windows target and three-minute Windows target as job timeouts creates a separate failure mode. Hosted-runner startup and performance vary, so a correct gate can be cancelled at the target boundary before it emits useful diagnostics. The performance objective needs measurement against GitHub timestamps, while correctness needs enough time to finish. + +Reviewers also need a direct answer to a simpler question: what happens when the repository's complete primary Node CI aggregate runs without matrix selection, shard variables, or concurrent gates on each selected hosted operating system? + +## Decision + +[CI](../../../../.github/workflows/ci.yml) accepts `workflow_dispatch` in addition to its normal push and pull-request events. A manual dispatch skips the optimized and compatibility jobs and exposes three explicit jobs named `serial / linux`, `serial / macos`, and `serial / windows`. They intentionally duplicate their short checkout, runtime setup, and immutable install sequences instead of hiding the operating systems behind a matrix or reusable workflow. + +Each reference job runs `pnpm run check:ci` without any shard selector. `DSH_GATE_CONCURRENCY=1` makes the top-level aggregate execute one ready gate at a time; coverage, snapshot replay, built-bin smoke, and publication validation also receive worker counts of one. The three operating-system jobs may run beside one another, but each host's repository gates are serial and complete. Linux installs bubblewrap before replaying snapshots, and Windows enables Developer Mode before installing the symlinked workspace. + +Manual reference jobs are diagnostic and do not participate in the required `all checks passed` result. Pull-request and push events continue to run only the optimized lanes. The one-minute non-Windows and three-minute Windows objectives are evaluated from completed hosted-job timestamps and reported as measurements; they are not `timeout-minutes` values. + +The portable reference uses GitHub's standard `ubuntu-latest`, `macos-latest`, and `windows-2025` labels. A higher-core hosted runner remains a possible future benchmark, but it is not the default: larger runners require organization-owned labels and provisioning, while a reference oracle should remain runnable without repository-external runner configuration. Provisioning one later can change the performance experiment without changing this correctness baseline. + +## Alternatives considered + +- **Set each timeout equal to its latency target** - rejected because scheduling variance would cancel correct work and suppress the evidence needed to diagnose a regression. +- **Trust only the optimized shard inventory** - rejected because selection and validation share implementation assumptions; an unsharded aggregate is an independent completeness check. +- **Run the serial references on every pull request** - rejected because they deliberately trade wall time and runner consumption for simplicity and are not needed in the fast feedback loop. +- **Use one operating-system matrix** - rejected because three named jobs make the reference surface visible without another selection mechanism. +- **Move the fast workflow to larger runners now** - rejected as the portable default because it would couple ordinary CI to organization-specific runner capacity. It remains an opt-in experiment after such capacity has an owned label and budget. + +## Consequences + +The workflow contains duplicated setup steps and a manual reference run can take much longer than the optimized pull-request path. That duplication is deliberate: reviewers can inspect each operating system's complete command without resolving a matrix or shard inventory. + +The reference may expose platform failures that the optimized blocking set does not yet claim to support, especially on Windows. Such a failure is evidence about current cross-platform behavior rather than a reason to weaken or silently skip the aggregate. + +Removing strict duration timeouts means a latency regression is observed rather than automatically cancelled. Hosted measurements must therefore accompany performance changes, while the completed logs retain the information needed to optimize the slow lane. diff --git a/.agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.zh.md b/.agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.zh.md new file mode 100644 index 0000000000..85c3403f1a --- /dev/null +++ b/.agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.zh.md @@ -0,0 +1,39 @@ +# Agent Note: 跨平台串行 CI 参考流程 + +Status: implemented + +[English](2026-07-21-serial-cross-platform-ci-reference.md) | 中文 + +## 问题 + +拉取请求工作流通过把静态检查、lint、覆盖率、快照回放和产物验证拆分到显式的 GitHub 作业中来达到延迟目标。这些分区由代码穷举校验，但优化工作流仍不应成为自身唯一的完整性判定基准：如果分片选择逻辑及其清单测试存在同一缺陷，即使所有优化通道都保持绿灯，也可能漏掉部分工作。 + +将非 Windows 作业的 1 分钟目标和 Windows 作业的 3 分钟目标写成作业超时，会引入另一种失败模式。托管运行器的启动时间和性能会波动，因此即使门禁本身正确，也可能在到达目标时间边界时被取消，来不及输出有用的诊断信息。性能目标需要根据 GitHub 时间戳衡量，而正确性验证需要给门禁留足完成时间。 + +评审人还需要直接回答一个更简单的问题：在每个选定的托管操作系统上，如果仓库完整的主 Node CI 聚合流程不使用矩阵选择、分片变量或并发门禁，运行结果会怎样？ + +## 决策 + +[CI](../../../../.github/workflows/ci.yml) 除正常的 push 和拉取请求事件外，也接受 `workflow_dispatch`。手动触发时会跳过优化作业和兼容性作业，只提供三个名为 `serial / linux`、`serial / macos` 和 `serial / windows` 的显式作业。这些作业有意分别重复简短的代码检出、运行时设置和依赖锁定的安装步骤，不用矩阵或可复用工作流把操作系统差异隐藏起来。 + +每个参考作业均在不设置任何分片选择器的情况下运行 `pnpm run check:ci`。`DSH_GATE_CONCURRENCY=1` 使顶层聚合每次只执行一个已经就绪的门禁；覆盖率、快照回放、built-bin 冒烟测试和发布验证的并发数也设为 1。三种操作系统的作业可以彼此并行，但每台主机上的仓库门禁都串行运行且完整执行。Linux 在回放快照前安装 bubblewrap，Windows 则在安装采用符号链接的工作区前启用开发人员模式。 + +手动参考作业仅用于诊断，不参与必需的 `all checks passed` 结果。拉取请求和 push 事件仍只运行优化通道。系统根据已完成托管作业的时间戳评估非 Windows 作业的 1 分钟目标和 Windows 作业的 3 分钟目标，并将其报告为测量结果，而不是写成 `timeout-minutes` 值。 + +可移植的参考流程使用 GitHub 标准的 `ubuntu-latest`、`macos-latest` 和 `windows-2025` 标签。仍可将更高核心数的托管运行器作为未来的基准测试，但不将其设为默认选择：更大型运行器需要组织自有的标签和预配，而参考判定基准应无需仓库外部的运行器配置即可运行。日后完成这类预配，可以改变性能实验而无需改变该正确性基线。 + +## 曾考虑的替代方案 + +- **将每个超时值设为相应延迟目标**：不予采纳，因为调度波动会中止原本正确的执行，并使诊断回归所需的证据无法产生。 +- **仅信任优化分片清单**：不予采纳，因为选择逻辑与校验逻辑共享实现假设；无分片的聚合流程是一项独立的完整性检查。 +- **在每个拉取请求上运行串行参考作业**：不予采纳，因为这些作业有意以更长的总耗时和更多运行器用量换取简单性，快速反馈循环不需要它们。 +- **使用一个操作系统矩阵**：不予采纳，因为三个具名作业无需另一套选择机制，就能让参考流程的构成清晰可见。 +- **立即把快速工作流迁移到更大型运行器**：不作为可移植的默认方案，因为这会让常规 CI 与特定组织的运行器容量耦合。等到这类容量拥有明确归属的标签和预算后，仍可将其作为一项可选实验。 + +## 后果 + +工作流包含重复的设置步骤，手动参考运行也可能比优化后的拉取请求路径耗时长得多。这些重复是有意保留的：评审人无需解析矩阵或分片清单，就能直接检查每种操作系统执行的完整命令。 + +参考流程可能暴露某些平台上的故障，而优化后的阻塞门禁集合尚未声明支持这些平台，Windows 尤其如此。这类失败反映了当前的跨平台行为，不应成为削弱或静默跳过该聚合流程的理由。 + +移除严格的时长超时后，系统会观测到延迟回归，而不是在发生回归时自动取消运行。因此，性能改动必须附带托管环境测量结果，已完成的日志则保留优化最慢通道所需的信息。 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 45b8cd5407..05427373e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,7 @@ on: push: branches: [main, master] pull_request: + workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -17,9 +18,9 @@ env: jobs: node-24: + if: github.event_name != 'workflow_dispatch' runs-on: ubuntu-latest name: node 24 / ${{ matrix.lane }} - timeout-minutes: 1 env: DSH_GATE_CONCURRENCY: ${{ matrix.gate_concurrency }} DSH_PUBLINT_CONCURRENCY: ${{ matrix.publint_concurrency }} @@ -248,9 +249,9 @@ jobs: run: ${{ matrix.command }} node-compat: + if: github.event_name != 'workflow_dispatch' runs-on: ubuntu-latest name: node ${{ matrix.node }} - timeout-minutes: 1 env: DSH_GATE_CONCURRENCY: '2' DSH_NODE_COMPAT_SKIP_TYPECHECK: ${{ matrix.skip_typecheck }} @@ -292,9 +293,9 @@ jobs: run: pnpm run check:node-compat python-sdk: + if: github.event_name != 'workflow_dispatch' runs-on: ubuntu-latest name: python 3.10 / keyless SDK - timeout-minutes: 1 steps: - uses: actions/checkout@v6 @@ -313,9 +314,9 @@ jobs: # while the broader observational gate matrix below exposes the remaining # portability work without blocking mainline merges. windows-build: + if: github.event_name != 'workflow_dispatch' runs-on: windows-2025 name: windows / build - timeout-minutes: 3 steps: - uses: actions/checkout@v6 @@ -341,10 +342,10 @@ jobs: # would change the environment being measured. This job intentionally stays # out of all-checks-passed.needs. windows-gates: + if: github.event_name != 'workflow_dispatch' continue-on-error: true runs-on: windows-2025 name: windows node 24 / ${{ matrix.lane }} - timeout-minutes: 3 env: DSH_GATE_CONCURRENCY: ${{ matrix.gate_concurrency }} DSH_PUBLINT_CONCURRENCY: ${{ matrix.publint_concurrency }} @@ -433,6 +434,104 @@ jobs: shell: pwsh run: ${{ matrix.command }} + # Manual reference runs deliberately avoid the optimized matrices above. + # Each host executes the complete, unsharded primary Node aggregate with one + # gate worker, giving reviewers a simple cross-platform oracle for completeness + # and timing. + serial-linux: + if: github.event_name == 'workflow_dispatch' + name: serial / linux + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-node@v6 + with: + node-version: ${{ env.PRIMARY_NODE_VERSION }} + + - name: Enable corepack (pnpm) + run: corepack enable + + - name: Install (immutable) + run: pnpm install --frozen-lockfile + + - name: Install bubblewrap (unrestrict userns) + run: | + sudo apt-get update -q + sudo apt-get install -yq --no-install-recommends bubblewrap + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 \ + || echo "apparmor userns knob absent — the functional probe decides" + + - name: Run complete unsharded primary Node CI serially + env: + DSH_COVERAGE_MAX_WORKERS: '1' + DSH_E2E_MAX_WORKERS: '1' + DSH_GATE_CONCURRENCY: '1' + DSH_PUBLINT_CONCURRENCY: '1' + DSH_SNAPSHOT_MAX_CONCURRENCY: '1' + run: pnpm run check:ci + + serial-macos: + if: github.event_name == 'workflow_dispatch' + name: serial / macos + runs-on: macos-latest + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-node@v6 + with: + node-version: ${{ env.PRIMARY_NODE_VERSION }} + + - name: Enable corepack (pnpm) + run: corepack enable + + - name: Install (immutable) + run: pnpm install --frozen-lockfile + + - name: Run complete unsharded primary Node CI serially + env: + DSH_COVERAGE_MAX_WORKERS: '1' + DSH_E2E_MAX_WORKERS: '1' + DSH_GATE_CONCURRENCY: '1' + DSH_PUBLINT_CONCURRENCY: '1' + DSH_SNAPSHOT_MAX_CONCURRENCY: '1' + run: pnpm run check:ci + + serial-windows: + if: github.event_name == 'workflow_dispatch' + name: serial / windows + runs-on: windows-2025 + steps: + - uses: actions/checkout@v6 + + - name: Enable Developer Mode (symlink support) + shell: pwsh + run: >- + reg add "HKLM\SOFTWARE\Microsoft\Windows\CurrentVersion\AppModelUnlock" + /t REG_DWORD /f /v "AllowDevelopmentWithoutDevLicense" /d "1" + + - uses: actions/setup-node@v6 + with: + node-version: ${{ env.PRIMARY_NODE_VERSION }} + + - name: Enable corepack (pnpm) + shell: pwsh + run: corepack enable + + - name: Install (immutable) + shell: pwsh + run: pnpm install --frozen-lockfile + + - name: Run complete unsharded primary Node CI serially + shell: pwsh + env: + DSH_COVERAGE_MAX_WORKERS: '1' + DSH_E2E_MAX_WORKERS: '1' + DSH_GATE_CONCURRENCY: '1' + DSH_PUBLINT_CONCURRENCY: '1' + DSH_SNAPSHOT_MAX_CONCURRENCY: '1' + run: pnpm run check:ci + # Single stable required check for branch protection: require "all checks # passed" instead of enumerating matrix legs whose names change as lanes and # node versions evolve. Every blocking job in THIS workflow must be listed in @@ -445,9 +544,8 @@ jobs: all-checks-passed: name: all checks passed runs-on: ubuntu-latest - timeout-minutes: 1 needs: [node-24, node-compat, python-sdk, windows-build] - if: always() + if: always() && github.event_name != 'workflow_dispatch' steps: - name: Fail if any needed job did not succeed if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'skipped') From 9568f9ee272e32905d9593e3ee1e7dfc254eaf31 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Tue, 21 Jul 2026 23:18:49 +0800 Subject: [PATCH 13/49] ci: split Windows build and site validation --- .../2026-07-06-parallel-github-ci-gates.md | 4 +- .github/workflows/ci.yml | 58 +++++++++++-------- 2 files changed, 37 insertions(+), 25 deletions(-) diff --git a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md index b657fd1e93..0e2fbf76ac 100644 --- a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md +++ b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md @@ -18,7 +18,7 @@ The artifact boundary remains load-bearing. `publint`, `verify-node-next-types`, Snapshot replay uses two explicit multi-file lanes and six scenario partitions of the large ACP file. [scripts/snapshot-shards.ts](../../../../scripts/snapshot-shards.ts) owns that inventory, and its test discovers every file admitted by the snapshot config so a new file cannot land outside CI. Each snapshot job builds the shipped runtime while its Linux runner installs bubblewrap from the hosted image's existing package index, then runs only its assigned replay surface. CI explicitly retains the suite's bounded concurrency of five subprocesses instead of clamping it to the runner's two logical CPUs, because replay spends most of its time waiting on child protocol I/O. Fixture guards still inspect the complete ACP scenario table in every partition. Static, lint, coverage, and snapshot sharding changes only GitHub scheduling: the ordinary local package scripts still run their complete suites. -Cold standalone documentation typechecking rebuilds the complete project-reference graph, so a dedicated documentation-type lane builds once and checks Markdown blocks against those declarations. The Linux documentation lane uses VitePress's MPA build to retain page rendering and dead-link validation within the observed non-Windows target; the blocking Windows build also produces the normal production SPA, preserving the shipped-site check within the observed Windows target. +Cold standalone documentation typechecking rebuilds the complete project-reference graph, so a dedicated documentation-type lane builds once and checks Markdown blocks against those declarations. The Linux documentation lane uses VitePress's MPA build to retain page rendering and dead-link validation within the observed non-Windows target; separate blocking Windows build and production-site lanes preserve the emitted-package and shipped-site checks without putting both critical paths in one job. Artifacts use two lanes: one metadata lane for `publint`, NodeNext declarations, and compiled invariant loading, plus one built-bin smoke lane. Each lane produces its own build before its consumers. Repeating the short build costs runner minutes but avoids an upload/download dependency and keeps each job's critical path bounded. @@ -26,7 +26,7 @@ Artifacts use two lanes: one metadata lane for `publint`, NodeNext declarations, Compatibility lanes run the source worker and Zstandard runtime smokes on every advertised Node line. TypeScript checks the source graph once in a dedicated primary Node 24 lane; repeating the same compiler analysis in runtime compatibility jobs added time without runtime-specific signal. -The workflow caches the pnpm store, preserves native PowerShell for Windows measurements, and retains one aggregate `all checks passed` status for branch protection. Windows reuses the three exhaustive lint partitions and groups foundation/catalog/prose plus documentation-type/API-contract gates behind shared runner setups; only scheduling differs from the Linux partitions. Windows build remains blocking, while the wider Windows static, lint, and artifact matrix remains observational. +The workflow caches the pnpm store, preserves native PowerShell for Windows measurements, and retains one aggregate `all checks passed` status for branch protection. Windows reuses the three exhaustive lint partitions and groups foundation/catalog/prose plus documentation-type/API-contract gates behind shared runner setups; only scheduling differs from the Linux partitions. Windows build and production-site validation remain blocking, while the wider Windows static, lint, and artifact matrix remains observational. ## Alternatives considered diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05427373e9..9c5e5b0b3c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -196,12 +196,11 @@ jobs: with: node-version: ${{ env.PRIMARY_NODE_VERSION }} - - name: Enable corepack (pnpm) - run: corepack enable - - - name: Resolve pnpm store path + - name: Enable corepack and resolve pnpm store path id: pnpm-store - run: echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" + run: | + corepack enable + echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" - uses: actions/cache@v4 with: @@ -272,12 +271,11 @@ jobs: with: node-version: ${{ matrix.node }} - - name: Enable corepack (pnpm) - run: corepack enable - - - name: Resolve pnpm store path + - name: Enable corepack and resolve pnpm store path id: pnpm-store - run: echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" + run: | + corepack enable + echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" - uses: actions/cache@v4 with: @@ -310,8 +308,9 @@ jobs: - name: Run complete keyless Python suite run: uv run --python 3.10 --group test --project python/sdk pytest - # Blocking Windows build lane: keep the already-green native build protected - # while the broader observational gate matrix below exposes the remaining + # Blocking Windows build and production-site lanes keep the already-green + # native outputs protected without putting both critical paths in one job. + # The broader observational gate matrix below exposes the remaining # portability work without blocking mainline merges. windows-build: if: github.event_name != 'workflow_dispatch' @@ -324,15 +323,30 @@ jobs: with: node-version: ${{ env.PRIMARY_NODE_VERSION }} - - name: Enable corepack (pnpm) - run: corepack enable - - name: Install (immutable) - run: pnpm install --frozen-lockfile + run: | + corepack enable + pnpm install --frozen-lockfile - name: Build (tsc -b + tsdown) run: pnpm run build + windows-site: + if: github.event_name != 'workflow_dispatch' + runs-on: windows-2025 + name: windows / production site + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-node@v6 + with: + node-version: ${{ env.PRIMARY_NODE_VERSION }} + + - name: Install (immutable) + run: | + corepack enable + pnpm install --frozen-lockfile + - name: Build documentation site (production SPA) run: pnpm run docs:build @@ -402,14 +416,12 @@ jobs: with: node-version: ${{ env.PRIMARY_NODE_VERSION }} - - name: Enable corepack (pnpm) - shell: pwsh - run: corepack enable - - - name: Resolve pnpm store path + - name: Enable corepack and resolve pnpm store path id: pnpm-store shell: pwsh - run: '"path=$(pnpm store path --silent)" >> $env:GITHUB_OUTPUT' + run: | + corepack enable + "path=$(pnpm store path --silent)" >> $env:GITHUB_OUTPUT - uses: actions/cache@v4 with: @@ -544,7 +556,7 @@ jobs: all-checks-passed: name: all checks passed runs-on: ubuntu-latest - needs: [node-24, node-compat, python-sdk, windows-build] + needs: [node-24, node-compat, python-sdk, windows-build, windows-site] if: always() && github.event_name != 'workflow_dispatch' steps: - name: Fail if any needed job did not succeed From efba4c259ba46077f71016fd13eb9f9eae84e053 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Tue, 21 Jul 2026 23:25:52 +0800 Subject: [PATCH 14/49] ci: split model and integration coverage --- .github/workflows/ci.yml | 9 +++++++-- scripts/coverage-shards.ts | 8 ++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c5e5b0b3c..a296769bb4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,11 +92,16 @@ jobs: gate_concurrency: '1' coverage_max_workers: '4' coverage_shard: state-session - - lane: coverage-models-integrations + - lane: coverage-models command: pnpm run check:ci:coverage gate_concurrency: '1' coverage_max_workers: '4' - coverage_shard: models-integrations + coverage_shard: models + - lane: coverage-integrations + command: pnpm run check:ci:coverage + gate_concurrency: '1' + coverage_max_workers: '4' + coverage_shard: integrations - lane: coverage-sdk-capabilities command: pnpm run check:ci:coverage gate_concurrency: '1' diff --git a/scripts/coverage-shards.ts b/scripts/coverage-shards.ts index 4bcc0277c1..dc471e33b4 100644 --- a/scripts/coverage-shards.ts +++ b/scripts/coverage-shards.ts @@ -34,8 +34,12 @@ export const coverageShards = [ ], }, { - name: 'models-integrations', - packageRoots: ['llm', 'compact', 'hooks/hook-protocol', 'lsp', 'mcp', 'hooks/hooks-claude'], + name: 'models', + packageRoots: ['llm', 'compact'], + }, + { + name: 'integrations', + packageRoots: ['hooks/hook-protocol', 'lsp', 'mcp', 'hooks/hooks-claude'], }, { name: 'sdk-capabilities', From e887c0f7c03bd5113f384d0c23cc994ddfa95199 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Tue, 21 Jul 2026 23:36:35 +0800 Subject: [PATCH 15/49] ci: assign session title coverage --- .github/workflows/ci.yml | 5 +++++ scripts/coverage-shards.ts | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a296769bb4..d5b9d6e76f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,6 +92,11 @@ jobs: gate_concurrency: '1' coverage_max_workers: '4' coverage_shard: state-session + - lane: coverage-session-title + command: pnpm run check:ci:coverage + gate_concurrency: '1' + coverage_max_workers: '4' + coverage_shard: session-title - lane: coverage-models command: pnpm run check:ci:coverage gate_concurrency: '1' diff --git a/scripts/coverage-shards.ts b/scripts/coverage-shards.ts index dc471e33b4..c58665f02a 100644 --- a/scripts/coverage-shards.ts +++ b/scripts/coverage-shards.ts @@ -37,6 +37,10 @@ export const coverageShards = [ name: 'models', packageRoots: ['llm', 'compact'], }, + { + name: 'session-title', + packageRoots: ['session-title'], + }, { name: 'integrations', packageRoots: ['hooks/hook-protocol', 'lsp', 'mcp', 'hooks/hooks-claude'], From 53a9459609a7466dcf2645906804f9e6c3df09e2 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Tue, 21 Jul 2026 23:45:35 +0800 Subject: [PATCH 16/49] ci: make coverage filters path-exact --- scripts/coverage-shards.spec.ts | 5 ++++- scripts/coverage-shards.ts | 6 ++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/coverage-shards.spec.ts b/scripts/coverage-shards.spec.ts index 8db6ac1aba..678011618e 100644 --- a/scripts/coverage-shards.spec.ts +++ b/scripts/coverage-shards.spec.ts @@ -28,11 +28,14 @@ describe('coverage shards', () => { it.each(coverageShards)('selects tests and source includes for $name', (shard) => { const args = coverageArgs(shard.name) for (const packageRoot of shard.packageRoots) { - expect(args).toContain(`packages/${packageRoot}`) + expect(args).toContain(`packages/${packageRoot}/`) expect(args).toContain(packageRoot.includes('/') ? `--coverage.include=packages/${packageRoot}/src/**/*.ts` : `--coverage.include=packages/${packageRoot}/*/src/**/*.ts`) } + if ('extraTestRoots' in shard) { + for (const testRoot of shard.extraTestRoots) expect(args).toContain(`${testRoot}/`) + } expect(args).toContain('scripts/test-invariants.spec.ts') expect(new Set(args).size).toBe(args.length) }) diff --git a/scripts/coverage-shards.ts b/scripts/coverage-shards.ts index c58665f02a..84f4d1a63e 100644 --- a/scripts/coverage-shards.ts +++ b/scripts/coverage-shards.ts @@ -98,9 +98,11 @@ export function coverageArgs(name: string): string[] { throw new Error(`run-gates: unknown DSH_COVERAGE_SHARD ${JSON.stringify(name)}.`) } + // Vitest positional filters are substrings; the trailing separator keeps + // prefix-named sibling packages out of each lane. const testRoots = new Set([ - ...shard.packageRoots.map(packageRoot => `packages/${packageRoot}`), - ...('extraTestRoots' in shard ? shard.extraTestRoots : []), + ...shard.packageRoots.map(packageRoot => `packages/${packageRoot}/`), + ...('extraTestRoots' in shard ? shard.extraTestRoots.map(testRoot => `${testRoot}/`) : []), 'scripts/test-invariants.spec.ts', ]) return [ From 710f062586ab12b7c077d4d0be7df2abb2f646e9 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Tue, 21 Jul 2026 23:56:53 +0800 Subject: [PATCH 17/49] ci: add snapshot lane headroom --- .../2026-07-06-parallel-github-ci-gates.md | 4 +-- .github/workflows/ci.yml | 29 +++++++++++++++---- scripts/snapshot-shards.spec.ts | 13 +++++++-- scripts/snapshot-shards.ts | 4 +-- 4 files changed, 39 insertions(+), 11 deletions(-) diff --git a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md index 0e2fbf76ac..35de6cca40 100644 --- a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md +++ b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md @@ -14,9 +14,9 @@ The artifact boundary remains load-bearing. `publint`, `verify-node-next-types`, [CI](../../../../.github/workflows/ci.yml) treats one minute for non-Windows jobs and three minutes for Windows jobs as observed performance targets, not cancellation deadlines. Hosted-runner variance should leave complete timing evidence and useful failure logs instead of cancelling an otherwise-correct gate. The [serial cross-platform CI reference](2026-07-21-serial-cross-platform-ci-reference.md) independently runs the complete unsharded primary Node aggregate on Linux, macOS, and Windows so the optimized lane inventory is not its own completeness oracle. -[scripts/run-gates.ts](../../../../scripts/run-gates.ts) remains the common bounded scheduler, but GitHub supplies explicit shard names for the expensive gate families. [scripts/static-shards.ts](../../../../scripts/static-shards.ts) partitions static gates into foundation, documentation-type, API-contract, catalog, prose, documentation-projection, and documentation-build lanes and rejects a missing or duplicate gate assignment. Lint uses disjoint package-source, package-test, and repository-complement lanes; the complement still starts from `.` so a new top-level lint target cannot disappear between shards, and it owns the single cross-file duplication run. [scripts/coverage-shards.ts](../../../../scripts/coverage-shards.ts) assigns every workspace package to exactly one source-coverage lane; its test expands the live package tree, so a new package makes CI red until it has an owner. Each coverage lane includes only its owned source files, repeats the exhaustive companion topology test, and runs without a preceding build because the complete coverage suite passes from a tree with every generated `lib/` removed. +[scripts/run-gates.ts](../../../../scripts/run-gates.ts) remains the common bounded scheduler, but GitHub supplies explicit shard names for the expensive gate families. [scripts/static-shards.ts](../../../../scripts/static-shards.ts) partitions static gates into foundation, documentation-type, API-contract, catalog, prose, documentation-projection, and documentation-build lanes and rejects a missing or duplicate gate assignment. Lint uses disjoint package-source, package-test, and repository-complement lanes; the complement still starts from `.` so a new top-level lint target cannot disappear between shards, and it owns the single cross-file duplication run. [scripts/coverage-shards.ts](../../../../scripts/coverage-shards.ts) assigns every workspace package to exactly one source-coverage lane; its test expands the live package tree, so a new package makes CI red until it has an owner. Directory filters retain a trailing separator because Vitest positional filters match substrings and would otherwise admit prefix-named sibling packages. Each coverage lane includes only its owned source files, repeats the exhaustive companion topology test, and runs without a preceding build because the complete coverage suite passes from a tree with every generated `lib/` removed. -Snapshot replay uses two explicit multi-file lanes and six scenario partitions of the large ACP file. [scripts/snapshot-shards.ts](../../../../scripts/snapshot-shards.ts) owns that inventory, and its test discovers every file admitted by the snapshot config so a new file cannot land outside CI. Each snapshot job builds the shipped runtime while its Linux runner installs bubblewrap from the hosted image's existing package index, then runs only its assigned replay surface. CI explicitly retains the suite's bounded concurrency of five subprocesses instead of clamping it to the runner's two logical CPUs, because replay spends most of its time waiting on child protocol I/O. Fixture guards still inspect the complete ACP scenario table in every partition. Static, lint, coverage, and snapshot sharding changes only GitHub scheduling: the ordinary local package scripts still run their complete suites. +Snapshot replay uses two explicit multi-file lanes and eight scenario partitions of the large ACP file. [scripts/snapshot-shards.ts](../../../../scripts/snapshot-shards.ts) owns that inventory, and its test discovers every file admitted by the snapshot config so a new file cannot land outside CI. Each snapshot job installs dependencies while its Linux runner installs bubblewrap, then builds the shipped runtime and runs only its assigned replay surface. CI explicitly retains the suite's bounded concurrency of five subprocesses instead of clamping it to the runner's two logical CPUs, because replay spends most of its time waiting on child protocol I/O. Fixture guards still inspect the complete ACP scenario table in every partition. Static, lint, coverage, and snapshot sharding changes only GitHub scheduling: the ordinary local package scripts still run their complete suites. Cold standalone documentation typechecking rebuilds the complete project-reference graph, so a dedicated documentation-type lane builds once and checks Markdown blocks against those declarations. The Linux documentation lane uses VitePress's MPA build to retain page rendering and dead-link validation within the observed non-Windows target; separate blocking Windows build and production-site lanes preserve the emitted-package and shipped-site checks without putting both critical paths in one job. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d5b9d6e76f..eb946fe117 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -190,6 +190,18 @@ jobs: snapshot_lane: acp-6 snapshot_max_concurrency: '5' snapshot_prebuilt: '1' + - lane: snapshot-acp-7 + command: pnpm run check:ci:snapshot + gate_concurrency: '1' + snapshot_lane: acp-7 + snapshot_max_concurrency: '5' + snapshot_prebuilt: '1' + - lane: snapshot-acp-8 + command: pnpm run check:ci:snapshot + gate_concurrency: '1' + snapshot_lane: acp-8 + snapshot_max_concurrency: '5' + snapshot_prebuilt: '1' - lane: artifacts-metadata command: pnpm run check:ci:artifacts gate_concurrency: '3' @@ -220,6 +232,7 @@ jobs: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm- - name: Install (immutable) + if: ${{ ! startsWith(matrix.lane, 'snapshot-') }} run: pnpm install --frozen-lockfile # The snapshot lanes REPLAY the sandbox example's recorded scenarios, @@ -227,22 +240,28 @@ jobs: # no bubblewrap preinstalled and no built Landlock launcher, so without # this the confined executions fail closed (SANDBOX_UNAVAILABLE). Same # install as sandbox.yml's bwrap leg (incl. the Ubuntu 24.04 AppArmor - # userns knob). Building does not depend on bubblewrap, so overlap them. - - name: Prepare built snapshot runtime and bubblewrap + # userns knob). Bubblewrap preparation is independent of dependency + # installation and the build, so it runs beside both. + - name: Install and prepare built snapshot runtime and bubblewrap if: startsWith(matrix.lane, 'snapshot-') run: | - pnpm run build & - build_pid=$! + pnpm install --frozen-lockfile & + install_pid=$! ( sudo apt-get install -yq --no-install-recommends bubblewrap sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 \ || echo "apparmor userns knob absent — the functional probe decides" ) & sandbox_pid=$! + install_status=0 + wait "$install_pid" || install_status=$? build_status=0 + if (( install_status == 0 )); then + pnpm run build || build_status=$? + fi sandbox_status=0 - wait "$build_pid" || build_status=$? wait "$sandbox_pid" || sandbox_status=$? + if (( install_status != 0 )); then exit "$install_status"; fi if (( build_status != 0 )); then exit "$build_status"; fi exit "$sandbox_status" diff --git a/scripts/snapshot-shards.spec.ts b/scripts/snapshot-shards.spec.ts index 7cc37fb7bd..1b8db7dc39 100644 --- a/scripts/snapshot-shards.spec.ts +++ b/scripts/snapshot-shards.spec.ts @@ -26,10 +26,19 @@ describe('snapshot lanes', () => { expect(new Set(ordinary).size).toBe(ordinary.length) expect(acp.map(lane => lane.files)).toEqual(Array.from( - { length: 6 }, + { length: 8 }, () => ['examples/acp-agent/tests/acp.snapshot.ts'], )) - expect(acp.map(lane => lane.scenarioShard)).toEqual(['1/6', '2/6', '3/6', '4/6', '5/6', '6/6']) + expect(acp.map(lane => lane.scenarioShard)).toEqual([ + '1/8', + '2/8', + '3/8', + '4/8', + '5/8', + '6/8', + '7/8', + '8/8', + ]) expect([...ordinary, 'examples/acp-agent/tests/acp.snapshot.ts'].sort()).toEqual(discovered) }) diff --git a/scripts/snapshot-shards.ts b/scripts/snapshot-shards.ts index fae8010261..5439be94fb 100644 --- a/scripts/snapshot-shards.ts +++ b/scripts/snapshot-shards.ts @@ -28,10 +28,10 @@ export const snapshotLanes: readonly SnapshotLane[] = [ 'examples/headless-agent/tests/headless.snapshot.ts', ], }, - ...Array.from({ length: 6 }, (_, offset) => ({ + ...Array.from({ length: 8 }, (_, offset) => ({ name: `acp-${offset + 1}`, files: ['examples/acp-agent/tests/acp.snapshot.ts'], - scenarioShard: `${offset + 1}/6`, + scenarioShard: `${offset + 1}/8`, })), ] From 624d71aea676dbdd62f7391a42b790c82045e138 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 00:11:48 +0800 Subject: [PATCH 18/49] ci: harden runner preparation caches --- .../2026-07-06-parallel-github-ci-gates.md | 4 ++-- .github/workflows/ci.yml | 15 ++++++++------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md index 35de6cca40..1327e4dd39 100644 --- a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md +++ b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md @@ -16,7 +16,7 @@ The artifact boundary remains load-bearing. `publint`, `verify-node-next-types`, [scripts/run-gates.ts](../../../../scripts/run-gates.ts) remains the common bounded scheduler, but GitHub supplies explicit shard names for the expensive gate families. [scripts/static-shards.ts](../../../../scripts/static-shards.ts) partitions static gates into foundation, documentation-type, API-contract, catalog, prose, documentation-projection, and documentation-build lanes and rejects a missing or duplicate gate assignment. Lint uses disjoint package-source, package-test, and repository-complement lanes; the complement still starts from `.` so a new top-level lint target cannot disappear between shards, and it owns the single cross-file duplication run. [scripts/coverage-shards.ts](../../../../scripts/coverage-shards.ts) assigns every workspace package to exactly one source-coverage lane; its test expands the live package tree, so a new package makes CI red until it has an owner. Directory filters retain a trailing separator because Vitest positional filters match substrings and would otherwise admit prefix-named sibling packages. Each coverage lane includes only its owned source files, repeats the exhaustive companion topology test, and runs without a preceding build because the complete coverage suite passes from a tree with every generated `lib/` removed. -Snapshot replay uses two explicit multi-file lanes and eight scenario partitions of the large ACP file. [scripts/snapshot-shards.ts](../../../../scripts/snapshot-shards.ts) owns that inventory, and its test discovers every file admitted by the snapshot config so a new file cannot land outside CI. Each snapshot job installs dependencies while its Linux runner installs bubblewrap, then builds the shipped runtime and runs only its assigned replay surface. CI explicitly retains the suite's bounded concurrency of five subprocesses instead of clamping it to the runner's two logical CPUs, because replay spends most of its time waiting on child protocol I/O. Fixture guards still inspect the complete ACP scenario table in every partition. Static, lint, coverage, and snapshot sharding changes only GitHub scheduling: the ordinary local package scripts still run their complete suites. +Snapshot replay uses two explicit multi-file lanes and eight scenario partitions of the large ACP file. [scripts/snapshot-shards.ts](../../../../scripts/snapshot-shards.ts) owns that inventory, and its test discovers every file admitted by the snapshot config so a new file cannot land outside CI. Each snapshot job installs dependencies while its Linux runner refreshes APT indexes and installs bubblewrap, then builds the shipped runtime and runs only its assigned replay surface. CI explicitly retains the suite's bounded concurrency of five subprocesses instead of clamping it to the runner's two logical CPUs, because replay spends most of its time waiting on child protocol I/O. Fixture guards still inspect the complete ACP scenario table in every partition. Static, lint, coverage, and snapshot sharding changes only GitHub scheduling: the ordinary local package scripts still run their complete suites. Cold standalone documentation typechecking rebuilds the complete project-reference graph, so a dedicated documentation-type lane builds once and checks Markdown blocks against those declarations. The Linux documentation lane uses VitePress's MPA build to retain page rendering and dead-link validation within the observed non-Windows target; separate blocking Windows build and production-site lanes preserve the emitted-package and shipped-site checks without putting both critical paths in one job. @@ -26,7 +26,7 @@ Artifacts use two lanes: one metadata lane for `publint`, NodeNext declarations, Compatibility lanes run the source worker and Zstandard runtime smokes on every advertised Node line. TypeScript checks the source graph once in a dedicated primary Node 24 lane; repeating the same compiler analysis in runtime compatibility jobs added time without runtime-specific signal. -The workflow caches the pnpm store, preserves native PowerShell for Windows measurements, and retains one aggregate `all checks passed` status for branch protection. Windows reuses the three exhaustive lint partitions and groups foundation/catalog/prose plus documentation-type/API-contract gates behind shared runner setups; only scheduling differs from the Linux partitions. Windows build and production-site validation remain blocking, while the wider Windows static, lint, and artifact matrix remains observational. +The workflow caches the pnpm store, keys each immutable ESLint cache to its owning lint shard, preserves native PowerShell for Windows measurements, and retains one aggregate `all checks passed` status for branch protection. Windows reuses the three exhaustive lint partitions and groups foundation/catalog/prose plus documentation-type/API-contract gates behind shared runner setups; only scheduling differs from the Linux partitions. Windows build and production-site validation remain blocking, while the wider Windows static, lint, and artifact matrix remains observational. ## Alternatives considered diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb946fe117..fef6e6c2d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -239,15 +239,16 @@ jobs: # re-executing their bash calls under a real runner. ubuntu-latest has # no bubblewrap preinstalled and no built Landlock launcher, so without # this the confined executions fail closed (SANDBOX_UNAVAILABLE). Same - # install as sandbox.yml's bwrap leg (incl. the Ubuntu 24.04 AppArmor - # userns knob). Bubblewrap preparation is independent of dependency - # installation and the build, so it runs beside both. + # refreshed install as sandbox.yml's bwrap leg (incl. the Ubuntu 24.04 + # AppArmor userns knob). Bubblewrap preparation is independent of + # dependency installation and the build, so it runs beside both. - name: Install and prepare built snapshot runtime and bubblewrap if: startsWith(matrix.lane, 'snapshot-') run: | pnpm install --frozen-lockfile & install_pid=$! ( + sudo apt-get update -q sudo apt-get install -yq --no-install-recommends bubblewrap sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 \ || echo "apparmor userns knob absent — the functional probe decides" @@ -269,9 +270,9 @@ jobs: if: startsWith(matrix.lane, 'lint-') with: path: .cache/eslint - key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-${{ hashFiles('pnpm-lock.yaml', 'eslint.config.mjs', 'tsconfig.json', 'packages/*/*/tsconfig.json', 'examples/*/tsconfig.json') }} + key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-${{ matrix.lint_shard }}-${{ hashFiles('pnpm-lock.yaml', 'eslint.config.mjs', 'tsconfig.json', 'packages/*/*/tsconfig.json', 'examples/*/tsconfig.json') }} restore-keys: | - ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint- + ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-${{ matrix.lint_shard }}- - name: Run gates run: ${{ matrix.command }} @@ -467,9 +468,9 @@ jobs: if: startsWith(matrix.lane, 'lint-') with: path: .cache/eslint - key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-${{ hashFiles('pnpm-lock.yaml', 'eslint.config.mjs', 'tsconfig.json', 'packages/*/*/tsconfig.json', 'examples/*/tsconfig.json') }} + key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-${{ matrix.lint_shard }}-${{ hashFiles('pnpm-lock.yaml', 'eslint.config.mjs', 'tsconfig.json', 'packages/*/*/tsconfig.json', 'examples/*/tsconfig.json') }} restore-keys: | - ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint- + ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-${{ matrix.lint_shard }}- - name: Run gates shell: pwsh From 070d23d6633dfcb673e5f74409cb9f41b042932d Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 00:22:06 +0800 Subject: [PATCH 19/49] ci: keep cold-cache lanes bounded --- .../2026-07-06-parallel-github-ci-gates.md | 4 +- .github/workflows/ci.yml | 49 ++++++++++++++----- scripts/lint-shards.spec.ts | 33 +++++++++++-- scripts/lint-shards.ts | 12 +++-- 4 files changed, 74 insertions(+), 24 deletions(-) diff --git a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md index 1327e4dd39..184f400a83 100644 --- a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md +++ b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md @@ -14,9 +14,9 @@ The artifact boundary remains load-bearing. `publint`, `verify-node-next-types`, [CI](../../../../.github/workflows/ci.yml) treats one minute for non-Windows jobs and three minutes for Windows jobs as observed performance targets, not cancellation deadlines. Hosted-runner variance should leave complete timing evidence and useful failure logs instead of cancelling an otherwise-correct gate. The [serial cross-platform CI reference](2026-07-21-serial-cross-platform-ci-reference.md) independently runs the complete unsharded primary Node aggregate on Linux, macOS, and Windows so the optimized lane inventory is not its own completeness oracle. -[scripts/run-gates.ts](../../../../scripts/run-gates.ts) remains the common bounded scheduler, but GitHub supplies explicit shard names for the expensive gate families. [scripts/static-shards.ts](../../../../scripts/static-shards.ts) partitions static gates into foundation, documentation-type, API-contract, catalog, prose, documentation-projection, and documentation-build lanes and rejects a missing or duplicate gate assignment. Lint uses disjoint package-source, package-test, and repository-complement lanes; the complement still starts from `.` so a new top-level lint target cannot disappear between shards, and it owns the single cross-file duplication run. [scripts/coverage-shards.ts](../../../../scripts/coverage-shards.ts) assigns every workspace package to exactly one source-coverage lane; its test expands the live package tree, so a new package makes CI red until it has an owner. Directory filters retain a trailing separator because Vitest positional filters match substrings and would otherwise admit prefix-named sibling packages. Each coverage lane includes only its owned source files, repeats the exhaustive companion topology test, and runs without a preceding build because the complete coverage suite passes from a tree with every generated `lib/` removed. +[scripts/run-gates.ts](../../../../scripts/run-gates.ts) remains the common bounded scheduler, but GitHub supplies explicit shard names for the expensive gate families. [scripts/static-shards.ts](../../../../scripts/static-shards.ts) partitions static gates into foundation, documentation-type, API-contract, catalog, prose, documentation-projection, and documentation-build lanes and rejects a missing or duplicate gate assignment. Lint uses disjoint A-M and N-Z package-source and package-test lanes plus a repository complement; the complement still starts from `.` so a new top-level lint target cannot disappear between shards, and it owns the single cross-file duplication run. [scripts/coverage-shards.ts](../../../../scripts/coverage-shards.ts) assigns every workspace package to exactly one source-coverage lane; its test expands the live package tree, so a new package makes CI red until it has an owner. Directory filters retain a trailing separator because Vitest positional filters match substrings and would otherwise admit prefix-named sibling packages. Each coverage lane includes only its owned source files, repeats the exhaustive companion topology test, and runs without a preceding build because the complete coverage suite passes from a tree with every generated `lib/` removed. -Snapshot replay uses two explicit multi-file lanes and eight scenario partitions of the large ACP file. [scripts/snapshot-shards.ts](../../../../scripts/snapshot-shards.ts) owns that inventory, and its test discovers every file admitted by the snapshot config so a new file cannot land outside CI. Each snapshot job installs dependencies while its Linux runner refreshes APT indexes and installs bubblewrap, then builds the shipped runtime and runs only its assigned replay surface. CI explicitly retains the suite's bounded concurrency of five subprocesses instead of clamping it to the runner's two logical CPUs, because replay spends most of its time waiting on child protocol I/O. Fixture guards still inspect the complete ACP scenario table in every partition. Static, lint, coverage, and snapshot sharding changes only GitHub scheduling: the ordinary local package scripts still run their complete suites. +Snapshot replay uses two explicit multi-file lanes and eight scenario partitions of the large ACP file. [scripts/snapshot-shards.ts](../../../../scripts/snapshot-shards.ts) owns that inventory, and its test discovers every file admitted by the snapshot config so a new file cannot land outside CI. Each snapshot job installs dependencies while its Linux runner installs bubblewrap, refreshing APT indexes and retrying if the hosted image's initial package install fails, then builds the shipped runtime and runs only its assigned replay surface. CI explicitly retains the suite's bounded concurrency of five subprocesses instead of clamping it to the runner's two logical CPUs, because replay spends most of its time waiting on child protocol I/O. Fixture guards still inspect the complete ACP scenario table in every partition. Static, lint, coverage, and snapshot sharding changes only GitHub scheduling: the ordinary local package scripts still run their complete suites. Cold standalone documentation typechecking rebuilds the complete project-reference graph, so a dedicated documentation-type lane builds once and checks Markdown blocks against those declarations. The Linux documentation lane uses VitePress's MPA build to retain page rendering and dead-link validation within the observed non-Windows target; separate blocking Windows build and production-site lanes preserve the emitted-package and shipped-site checks without putting both critical paths in one job. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fef6e6c2d9..a280b7316a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,16 +67,26 @@ jobs: static_shard: site-build - lane: typecheck command: pnpm run typecheck - - lane: lint-package-sources + - lane: lint-package-sources-a-m command: pnpm run check:ci:lint gate_concurrency: '1' eslint_cache: '1' - lint_shard: package-sources - - lane: lint-package-tests + lint_shard: package-sources-a-m + - lane: lint-package-sources-n-z command: pnpm run check:ci:lint gate_concurrency: '1' eslint_cache: '1' - lint_shard: package-tests + lint_shard: package-sources-n-z + - lane: lint-package-tests-a-m + command: pnpm run check:ci:lint + gate_concurrency: '1' + eslint_cache: '1' + lint_shard: package-tests-a-m + - lane: lint-package-tests-n-z + command: pnpm run check:ci:lint + gate_concurrency: '1' + eslint_cache: '1' + lint_shard: package-tests-n-z - lane: lint-repository command: pnpm run check:ci:lint gate_concurrency: '1' @@ -238,9 +248,9 @@ jobs: # The snapshot lanes REPLAY the sandbox example's recorded scenarios, # re-executing their bash calls under a real runner. ubuntu-latest has # no bubblewrap preinstalled and no built Landlock launcher, so without - # this the confined executions fail closed (SANDBOX_UNAVAILABLE). Same - # refreshed install as sandbox.yml's bwrap leg (incl. the Ubuntu 24.04 - # AppArmor userns knob). Bubblewrap preparation is independent of + # this the confined executions fail closed (SANDBOX_UNAVAILABLE). The + # install retries after refreshing stale indexes and applies the Ubuntu + # 24.04 AppArmor userns knob. Bubblewrap preparation is independent of # dependency installation and the build, so it runs beside both. - name: Install and prepare built snapshot runtime and bubblewrap if: startsWith(matrix.lane, 'snapshot-') @@ -248,8 +258,11 @@ jobs: pnpm install --frozen-lockfile & install_pid=$! ( - sudo apt-get update -q - sudo apt-get install -yq --no-install-recommends bubblewrap + if ! sudo apt-get install -yq --no-install-recommends bubblewrap; then + echo "initial bubblewrap install failed; refreshing APT indexes and retrying" + sudo apt-get update -q + sudo apt-get install -yq --no-install-recommends bubblewrap + fi sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 \ || echo "apparmor userns knob absent — the functional probe decides" ) & @@ -409,16 +422,26 @@ jobs: command: pnpm run check:ci:static gate_concurrency: '4' static_shard: 'doc-types,api-contracts' - - lane: lint-package-sources + - lane: lint-package-sources-a-m command: pnpm run check:ci:lint gate_concurrency: '1' eslint_cache: '1' - lint_shard: package-sources - - lane: lint-package-tests + lint_shard: package-sources-a-m + - lane: lint-package-sources-n-z command: pnpm run check:ci:lint gate_concurrency: '1' eslint_cache: '1' - lint_shard: package-tests + lint_shard: package-sources-n-z + - lane: lint-package-tests-a-m + command: pnpm run check:ci:lint + gate_concurrency: '1' + eslint_cache: '1' + lint_shard: package-tests-a-m + - lane: lint-package-tests-n-z + command: pnpm run check:ci:lint + gate_concurrency: '1' + eslint_cache: '1' + lint_shard: package-tests-n-z - lane: lint-repository command: pnpm run check:ci:lint gate_concurrency: '1' diff --git a/scripts/lint-shards.spec.ts b/scripts/lint-shards.spec.ts index df50790260..51ac21e0bd 100644 --- a/scripts/lint-shards.spec.ts +++ b/scripts/lint-shards.spec.ts @@ -1,19 +1,31 @@ +import { readdirSync } from 'node:fs' +import { resolve } from 'node:path' import { describe, expect, it } from 'vitest' import { selectLintShard } from './lint-shards.ts' +const packagesRoot = resolve(import.meta.dirname, '..', 'packages') + describe('lint gate shards', () => { it('keeps the unsharded local command complete', () => { expect(selectLintShard()).toEqual({ eslintTargets: ['.'], includeDuplication: true }) expect(selectLintShard('')).toEqual({ eslintTargets: ['.'], includeDuplication: true }) }) - it('partitions package sources, package tests, and their repository complement', () => { - expect(selectLintShard('package-sources')).toEqual({ - eslintTargets: ['packages/*/*/src/**/*.ts'], + it('partitions package sources and tests into alphabetic halves plus their repository complement', () => { + expect(selectLintShard('package-sources-a-m')).toEqual({ + eslintTargets: ['packages/[a-m]*/*/src/**/*.ts'], includeDuplication: false, }) - expect(selectLintShard('package-tests')).toEqual({ - eslintTargets: ['packages/*/*/tests/**/*.ts'], + expect(selectLintShard('package-sources-n-z')).toEqual({ + eslintTargets: ['packages/[n-z]*/*/src/**/*.ts'], + includeDuplication: false, + }) + expect(selectLintShard('package-tests-a-m')).toEqual({ + eslintTargets: ['packages/[a-m]*/*/tests/**/*.ts'], + includeDuplication: false, + }) + expect(selectLintShard('package-tests-n-z')).toEqual({ + eslintTargets: ['packages/[n-z]*/*/tests/**/*.ts'], includeDuplication: false, }) expect(selectLintShard('repository')).toEqual({ @@ -28,6 +40,17 @@ describe('lint gate shards', () => { }) }) + it('assigns every package group to one alphabetic half', () => { + const groups = readdirSync(packagesRoot, { withFileTypes: true }) + .filter(entry => entry.isDirectory()) + .map(entry => entry.name) + .sort() + const firstHalf = groups.filter(group => /^[a-m]/u.test(group)) + const secondHalf = groups.filter(group => /^[n-z]/u.test(group)) + + expect([...firstHalf, ...secondHalf].sort()).toEqual(groups) + }) + it('rejects an unknown lane', () => { expect(() => selectLintShard('missing')).toThrow('unknown DSH_LINT_SHARD') }) diff --git a/scripts/lint-shards.ts b/scripts/lint-shards.ts index ec09a7aca2..a48e9ce257 100644 --- a/scripts/lint-shards.ts +++ b/scripts/lint-shards.ts @@ -19,10 +19,14 @@ export function selectLintShard(name?: string): LintSelection { case undefined: case '': return { eslintTargets: ['.'], includeDuplication: true } - case 'package-sources': - return { eslintTargets: ['packages/*/*/src/**/*.ts'], includeDuplication: false } - case 'package-tests': - return { eslintTargets: ['packages/*/*/tests/**/*.ts'], includeDuplication: false } + case 'package-sources-a-m': + return { eslintTargets: ['packages/[a-m]*/*/src/**/*.ts'], includeDuplication: false } + case 'package-sources-n-z': + return { eslintTargets: ['packages/[n-z]*/*/src/**/*.ts'], includeDuplication: false } + case 'package-tests-a-m': + return { eslintTargets: ['packages/[a-m]*/*/tests/**/*.ts'], includeDuplication: false } + case 'package-tests-n-z': + return { eslintTargets: ['packages/[n-z]*/*/tests/**/*.ts'], includeDuplication: false } case 'repository': return { eslintTargets: [ From 109224af5d66ec3c35b6cdb4c22c6f98a48b0168 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 00:31:25 +0800 Subject: [PATCH 20/49] ci: quarter cold-cache lint lanes --- .../2026-07-06-parallel-github-ci-gates.md | 2 +- .github/workflows/ci.yml | 72 ++++++++++++++----- scripts/lint-shards.spec.ts | 42 +++++++---- scripts/lint-shards.ts | 24 ++++--- 4 files changed, 102 insertions(+), 38 deletions(-) diff --git a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md index 184f400a83..aeab7cbfbb 100644 --- a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md +++ b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md @@ -14,7 +14,7 @@ The artifact boundary remains load-bearing. `publint`, `verify-node-next-types`, [CI](../../../../.github/workflows/ci.yml) treats one minute for non-Windows jobs and three minutes for Windows jobs as observed performance targets, not cancellation deadlines. Hosted-runner variance should leave complete timing evidence and useful failure logs instead of cancelling an otherwise-correct gate. The [serial cross-platform CI reference](2026-07-21-serial-cross-platform-ci-reference.md) independently runs the complete unsharded primary Node aggregate on Linux, macOS, and Windows so the optimized lane inventory is not its own completeness oracle. -[scripts/run-gates.ts](../../../../scripts/run-gates.ts) remains the common bounded scheduler, but GitHub supplies explicit shard names for the expensive gate families. [scripts/static-shards.ts](../../../../scripts/static-shards.ts) partitions static gates into foundation, documentation-type, API-contract, catalog, prose, documentation-projection, and documentation-build lanes and rejects a missing or duplicate gate assignment. Lint uses disjoint A-M and N-Z package-source and package-test lanes plus a repository complement; the complement still starts from `.` so a new top-level lint target cannot disappear between shards, and it owns the single cross-file duplication run. [scripts/coverage-shards.ts](../../../../scripts/coverage-shards.ts) assigns every workspace package to exactly one source-coverage lane; its test expands the live package tree, so a new package makes CI red until it has an owner. Directory filters retain a trailing separator because Vitest positional filters match substrings and would otherwise admit prefix-named sibling packages. Each coverage lane includes only its owned source files, repeats the exhaustive companion topology test, and runs without a preceding build because the complete coverage suite passes from a tree with every generated `lib/` removed. +[scripts/run-gates.ts](../../../../scripts/run-gates.ts) remains the common bounded scheduler, but GitHub supplies explicit shard names for the expensive gate families. [scripts/static-shards.ts](../../../../scripts/static-shards.ts) partitions static gates into foundation, documentation-type, API-contract, catalog, prose, documentation-projection, and documentation-build lanes and rejects a missing or duplicate gate assignment. Lint uses disjoint A-C, D-M, N-S, and T-Z package-source and package-test lanes plus a repository complement; the complement still starts from `.` so a new top-level lint target cannot disappear between shards, and it owns the single cross-file duplication run. [scripts/coverage-shards.ts](../../../../scripts/coverage-shards.ts) assigns every workspace package to exactly one source-coverage lane; its test expands the live package tree, so a new package makes CI red until it has an owner. Directory filters retain a trailing separator because Vitest positional filters match substrings and would otherwise admit prefix-named sibling packages. Each coverage lane includes only its owned source files, repeats the exhaustive companion topology test, and runs without a preceding build because the complete coverage suite passes from a tree with every generated `lib/` removed. Snapshot replay uses two explicit multi-file lanes and eight scenario partitions of the large ACP file. [scripts/snapshot-shards.ts](../../../../scripts/snapshot-shards.ts) owns that inventory, and its test discovers every file admitted by the snapshot config so a new file cannot land outside CI. Each snapshot job installs dependencies while its Linux runner installs bubblewrap, refreshing APT indexes and retrying if the hosted image's initial package install fails, then builds the shipped runtime and runs only its assigned replay surface. CI explicitly retains the suite's bounded concurrency of five subprocesses instead of clamping it to the runner's two logical CPUs, because replay spends most of its time waiting on child protocol I/O. Fixture guards still inspect the complete ACP scenario table in every partition. Static, lint, coverage, and snapshot sharding changes only GitHub scheduling: the ordinary local package scripts still run their complete suites. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a280b7316a..79dcfcffcf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,26 +67,46 @@ jobs: static_shard: site-build - lane: typecheck command: pnpm run typecheck - - lane: lint-package-sources-a-m + - lane: lint-package-sources-a-c command: pnpm run check:ci:lint gate_concurrency: '1' eslint_cache: '1' - lint_shard: package-sources-a-m - - lane: lint-package-sources-n-z + lint_shard: package-sources-a-c + - lane: lint-package-sources-d-m command: pnpm run check:ci:lint gate_concurrency: '1' eslint_cache: '1' - lint_shard: package-sources-n-z - - lane: lint-package-tests-a-m + lint_shard: package-sources-d-m + - lane: lint-package-sources-n-s command: pnpm run check:ci:lint gate_concurrency: '1' eslint_cache: '1' - lint_shard: package-tests-a-m - - lane: lint-package-tests-n-z + lint_shard: package-sources-n-s + - lane: lint-package-sources-t-z command: pnpm run check:ci:lint gate_concurrency: '1' eslint_cache: '1' - lint_shard: package-tests-n-z + lint_shard: package-sources-t-z + - lane: lint-package-tests-a-c + command: pnpm run check:ci:lint + gate_concurrency: '1' + eslint_cache: '1' + lint_shard: package-tests-a-c + - lane: lint-package-tests-d-m + command: pnpm run check:ci:lint + gate_concurrency: '1' + eslint_cache: '1' + lint_shard: package-tests-d-m + - lane: lint-package-tests-n-s + command: pnpm run check:ci:lint + gate_concurrency: '1' + eslint_cache: '1' + lint_shard: package-tests-n-s + - lane: lint-package-tests-t-z + command: pnpm run check:ci:lint + gate_concurrency: '1' + eslint_cache: '1' + lint_shard: package-tests-t-z - lane: lint-repository command: pnpm run check:ci:lint gate_concurrency: '1' @@ -422,26 +442,46 @@ jobs: command: pnpm run check:ci:static gate_concurrency: '4' static_shard: 'doc-types,api-contracts' - - lane: lint-package-sources-a-m + - lane: lint-package-sources-a-c command: pnpm run check:ci:lint gate_concurrency: '1' eslint_cache: '1' - lint_shard: package-sources-a-m - - lane: lint-package-sources-n-z + lint_shard: package-sources-a-c + - lane: lint-package-sources-d-m command: pnpm run check:ci:lint gate_concurrency: '1' eslint_cache: '1' - lint_shard: package-sources-n-z - - lane: lint-package-tests-a-m + lint_shard: package-sources-d-m + - lane: lint-package-sources-n-s command: pnpm run check:ci:lint gate_concurrency: '1' eslint_cache: '1' - lint_shard: package-tests-a-m - - lane: lint-package-tests-n-z + lint_shard: package-sources-n-s + - lane: lint-package-sources-t-z command: pnpm run check:ci:lint gate_concurrency: '1' eslint_cache: '1' - lint_shard: package-tests-n-z + lint_shard: package-sources-t-z + - lane: lint-package-tests-a-c + command: pnpm run check:ci:lint + gate_concurrency: '1' + eslint_cache: '1' + lint_shard: package-tests-a-c + - lane: lint-package-tests-d-m + command: pnpm run check:ci:lint + gate_concurrency: '1' + eslint_cache: '1' + lint_shard: package-tests-d-m + - lane: lint-package-tests-n-s + command: pnpm run check:ci:lint + gate_concurrency: '1' + eslint_cache: '1' + lint_shard: package-tests-n-s + - lane: lint-package-tests-t-z + command: pnpm run check:ci:lint + gate_concurrency: '1' + eslint_cache: '1' + lint_shard: package-tests-t-z - lane: lint-repository command: pnpm run check:ci:lint gate_concurrency: '1' diff --git a/scripts/lint-shards.spec.ts b/scripts/lint-shards.spec.ts index 51ac21e0bd..540c4b88c4 100644 --- a/scripts/lint-shards.spec.ts +++ b/scripts/lint-shards.spec.ts @@ -11,21 +11,37 @@ describe('lint gate shards', () => { expect(selectLintShard('')).toEqual({ eslintTargets: ['.'], includeDuplication: true }) }) - it('partitions package sources and tests into alphabetic halves plus their repository complement', () => { - expect(selectLintShard('package-sources-a-m')).toEqual({ - eslintTargets: ['packages/[a-m]*/*/src/**/*.ts'], + it('partitions package sources and tests into alphabetic ranges plus their repository complement', () => { + expect(selectLintShard('package-sources-a-c')).toEqual({ + eslintTargets: ['packages/[a-c]*/*/src/**/*.ts'], includeDuplication: false, }) - expect(selectLintShard('package-sources-n-z')).toEqual({ - eslintTargets: ['packages/[n-z]*/*/src/**/*.ts'], + expect(selectLintShard('package-sources-d-m')).toEqual({ + eslintTargets: ['packages/[d-m]*/*/src/**/*.ts'], includeDuplication: false, }) - expect(selectLintShard('package-tests-a-m')).toEqual({ - eslintTargets: ['packages/[a-m]*/*/tests/**/*.ts'], + expect(selectLintShard('package-sources-n-s')).toEqual({ + eslintTargets: ['packages/[n-s]*/*/src/**/*.ts'], includeDuplication: false, }) - expect(selectLintShard('package-tests-n-z')).toEqual({ - eslintTargets: ['packages/[n-z]*/*/tests/**/*.ts'], + expect(selectLintShard('package-sources-t-z')).toEqual({ + eslintTargets: ['packages/[t-z]*/*/src/**/*.ts'], + includeDuplication: false, + }) + expect(selectLintShard('package-tests-a-c')).toEqual({ + eslintTargets: ['packages/[a-c]*/*/tests/**/*.ts'], + includeDuplication: false, + }) + expect(selectLintShard('package-tests-d-m')).toEqual({ + eslintTargets: ['packages/[d-m]*/*/tests/**/*.ts'], + includeDuplication: false, + }) + expect(selectLintShard('package-tests-n-s')).toEqual({ + eslintTargets: ['packages/[n-s]*/*/tests/**/*.ts'], + includeDuplication: false, + }) + expect(selectLintShard('package-tests-t-z')).toEqual({ + eslintTargets: ['packages/[t-z]*/*/tests/**/*.ts'], includeDuplication: false, }) expect(selectLintShard('repository')).toEqual({ @@ -40,15 +56,15 @@ describe('lint gate shards', () => { }) }) - it('assigns every package group to one alphabetic half', () => { + it('assigns every package group to one alphabetic range', () => { const groups = readdirSync(packagesRoot, { withFileTypes: true }) .filter(entry => entry.isDirectory()) .map(entry => entry.name) .sort() - const firstHalf = groups.filter(group => /^[a-m]/u.test(group)) - const secondHalf = groups.filter(group => /^[n-z]/u.test(group)) + const ranges = [/^[a-c]/u, /^[d-m]/u, /^[n-s]/u, /^[t-z]/u] + const assignments = ranges.flatMap(range => groups.filter(group => range.test(group))).sort() - expect([...firstHalf, ...secondHalf].sort()).toEqual(groups) + expect(assignments).toEqual(groups) }) it('rejects an unknown lane', () => { diff --git a/scripts/lint-shards.ts b/scripts/lint-shards.ts index a48e9ce257..e9a12c3e80 100644 --- a/scripts/lint-shards.ts +++ b/scripts/lint-shards.ts @@ -19,14 +19,22 @@ export function selectLintShard(name?: string): LintSelection { case undefined: case '': return { eslintTargets: ['.'], includeDuplication: true } - case 'package-sources-a-m': - return { eslintTargets: ['packages/[a-m]*/*/src/**/*.ts'], includeDuplication: false } - case 'package-sources-n-z': - return { eslintTargets: ['packages/[n-z]*/*/src/**/*.ts'], includeDuplication: false } - case 'package-tests-a-m': - return { eslintTargets: ['packages/[a-m]*/*/tests/**/*.ts'], includeDuplication: false } - case 'package-tests-n-z': - return { eslintTargets: ['packages/[n-z]*/*/tests/**/*.ts'], includeDuplication: false } + case 'package-sources-a-c': + return { eslintTargets: ['packages/[a-c]*/*/src/**/*.ts'], includeDuplication: false } + case 'package-sources-d-m': + return { eslintTargets: ['packages/[d-m]*/*/src/**/*.ts'], includeDuplication: false } + case 'package-sources-n-s': + return { eslintTargets: ['packages/[n-s]*/*/src/**/*.ts'], includeDuplication: false } + case 'package-sources-t-z': + return { eslintTargets: ['packages/[t-z]*/*/src/**/*.ts'], includeDuplication: false } + case 'package-tests-a-c': + return { eslintTargets: ['packages/[a-c]*/*/tests/**/*.ts'], includeDuplication: false } + case 'package-tests-d-m': + return { eslintTargets: ['packages/[d-m]*/*/tests/**/*.ts'], includeDuplication: false } + case 'package-tests-n-s': + return { eslintTargets: ['packages/[n-s]*/*/tests/**/*.ts'], includeDuplication: false } + case 'package-tests-t-z': + return { eslintTargets: ['packages/[t-z]*/*/tests/**/*.ts'], includeDuplication: false } case 'repository': return { eslintTargets: [ From 5bb594c4005dedea5d5a51414b50510556f6151e Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 00:39:58 +0800 Subject: [PATCH 21/49] ci: avoid hosted runner setup saturation --- .../2026-07-06-parallel-github-ci-gates.md | 2 +- .github/workflows/ci.yml | 52 +++++-------------- scripts/lint-shards.spec.ts | 29 +++++++++-- scripts/lint-shards.ts | 8 +++ 4 files changed, 46 insertions(+), 45 deletions(-) diff --git a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md index aeab7cbfbb..ab705d9450 100644 --- a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md +++ b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md @@ -14,7 +14,7 @@ The artifact boundary remains load-bearing. `publint`, `verify-node-next-types`, [CI](../../../../.github/workflows/ci.yml) treats one minute for non-Windows jobs and three minutes for Windows jobs as observed performance targets, not cancellation deadlines. Hosted-runner variance should leave complete timing evidence and useful failure logs instead of cancelling an otherwise-correct gate. The [serial cross-platform CI reference](2026-07-21-serial-cross-platform-ci-reference.md) independently runs the complete unsharded primary Node aggregate on Linux, macOS, and Windows so the optimized lane inventory is not its own completeness oracle. -[scripts/run-gates.ts](../../../../scripts/run-gates.ts) remains the common bounded scheduler, but GitHub supplies explicit shard names for the expensive gate families. [scripts/static-shards.ts](../../../../scripts/static-shards.ts) partitions static gates into foundation, documentation-type, API-contract, catalog, prose, documentation-projection, and documentation-build lanes and rejects a missing or duplicate gate assignment. Lint uses disjoint A-C, D-M, N-S, and T-Z package-source and package-test lanes plus a repository complement; the complement still starts from `.` so a new top-level lint target cannot disappear between shards, and it owns the single cross-file duplication run. [scripts/coverage-shards.ts](../../../../scripts/coverage-shards.ts) assigns every workspace package to exactly one source-coverage lane; its test expands the live package tree, so a new package makes CI red until it has an owner. Directory filters retain a trailing separator because Vitest positional filters match substrings and would otherwise admit prefix-named sibling packages. Each coverage lane includes only its owned source files, repeats the exhaustive companion topology test, and runs without a preceding build because the complete coverage suite passes from a tree with every generated `lib/` removed. +[scripts/run-gates.ts](../../../../scripts/run-gates.ts) remains the common bounded scheduler, but GitHub supplies explicit shard names for the expensive gate families. [scripts/static-shards.ts](../../../../scripts/static-shards.ts) partitions static gates into foundation, documentation-type, API-contract, catalog, prose, documentation-projection, and documentation-build ownership and rejects a missing or duplicate gate assignment; low-cost ownership groups may share one hosted lane to stay below the runner-provisioning concurrency wave. Linux lint uses disjoint A-C, D-M, N-S, and T-Z package-source and package-test lanes, while the longer Windows target uses A-M and N-Z halves; both topologies include a repository complement that still starts from `.` so a new top-level lint target cannot disappear between shards and owns the single cross-file duplication run. [scripts/coverage-shards.ts](../../../../scripts/coverage-shards.ts) assigns every workspace package to exactly one source-coverage lane; its test expands the live package tree, so a new package makes CI red until it has an owner. Directory filters retain a trailing separator because Vitest positional filters match substrings and would otherwise admit prefix-named sibling packages. Each coverage lane includes only its owned source files, repeats the exhaustive companion topology test, and runs without a preceding build because the complete coverage suite passes from a tree with every generated `lib/` removed. Snapshot replay uses two explicit multi-file lanes and eight scenario partitions of the large ACP file. [scripts/snapshot-shards.ts](../../../../scripts/snapshot-shards.ts) owns that inventory, and its test discovers every file admitted by the snapshot config so a new file cannot land outside CI. Each snapshot job installs dependencies while its Linux runner installs bubblewrap, refreshing APT indexes and retrying if the hosted image's initial package install fails, then builds the shipped runtime and runs only its assigned replay surface. CI explicitly retains the suite's bounded concurrency of five subprocesses instead of clamping it to the runner's two logical CPUs, because replay spends most of its time waiting on child protocol I/O. Fixture guards still inspect the complete ACP scenario table in every partition. Static, lint, coverage, and snapshot sharding changes only GitHub scheduling: the ordinary local package scripts still run their complete suites. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 79dcfcffcf..6b6ede348b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,30 +37,22 @@ jobs: fail-fast: false matrix: include: - - lane: static-foundation + - lane: static-foundation-projection command: pnpm run check:ci:static gate_concurrency: '4' - static_shard: foundation + static_shard: 'foundation,site-projection' - lane: static-doc-types command: pnpm run check:ci:static gate_concurrency: '1' static_shard: doc-types - - lane: static-api-contracts + - lane: static-api-catalogs command: pnpm run check:ci:static gate_concurrency: '4' - static_shard: api-contracts - - lane: static-catalogs - command: pnpm run check:ci:static - gate_concurrency: '4' - static_shard: catalogs + static_shard: 'api-contracts,catalogs' - lane: static-prose command: pnpm run check:ci:static gate_concurrency: '4' static_shard: prose - - lane: static-site-projection - command: pnpm run check:ci:static - gate_concurrency: '1' - static_shard: site-projection - lane: static-site-mpa command: pnpm run check:ci:static gate_concurrency: '1' @@ -442,46 +434,26 @@ jobs: command: pnpm run check:ci:static gate_concurrency: '4' static_shard: 'doc-types,api-contracts' - - lane: lint-package-sources-a-c + - lane: lint-package-sources command: pnpm run check:ci:lint gate_concurrency: '1' eslint_cache: '1' - lint_shard: package-sources-a-c - - lane: lint-package-sources-d-m + lint_shard: package-sources-a-m + - lane: lint-package-sources-n-z command: pnpm run check:ci:lint gate_concurrency: '1' eslint_cache: '1' - lint_shard: package-sources-d-m - - lane: lint-package-sources-n-s + lint_shard: package-sources-n-z + - lane: lint-package-tests command: pnpm run check:ci:lint gate_concurrency: '1' eslint_cache: '1' - lint_shard: package-sources-n-s - - lane: lint-package-sources-t-z + lint_shard: package-tests-a-m + - lane: lint-package-tests-n-z command: pnpm run check:ci:lint gate_concurrency: '1' eslint_cache: '1' - lint_shard: package-sources-t-z - - lane: lint-package-tests-a-c - command: pnpm run check:ci:lint - gate_concurrency: '1' - eslint_cache: '1' - lint_shard: package-tests-a-c - - lane: lint-package-tests-d-m - command: pnpm run check:ci:lint - gate_concurrency: '1' - eslint_cache: '1' - lint_shard: package-tests-d-m - - lane: lint-package-tests-n-s - command: pnpm run check:ci:lint - gate_concurrency: '1' - eslint_cache: '1' - lint_shard: package-tests-n-s - - lane: lint-package-tests-t-z - command: pnpm run check:ci:lint - gate_concurrency: '1' - eslint_cache: '1' - lint_shard: package-tests-t-z + lint_shard: package-tests-n-z - lane: lint-repository command: pnpm run check:ci:lint gate_concurrency: '1' diff --git a/scripts/lint-shards.spec.ts b/scripts/lint-shards.spec.ts index 540c4b88c4..e14a15b9ad 100644 --- a/scripts/lint-shards.spec.ts +++ b/scripts/lint-shards.spec.ts @@ -44,6 +44,22 @@ describe('lint gate shards', () => { eslintTargets: ['packages/[t-z]*/*/tests/**/*.ts'], includeDuplication: false, }) + expect(selectLintShard('package-sources-a-m')).toEqual({ + eslintTargets: ['packages/[a-m]*/*/src/**/*.ts'], + includeDuplication: false, + }) + expect(selectLintShard('package-sources-n-z')).toEqual({ + eslintTargets: ['packages/[n-z]*/*/src/**/*.ts'], + includeDuplication: false, + }) + expect(selectLintShard('package-tests-a-m')).toEqual({ + eslintTargets: ['packages/[a-m]*/*/tests/**/*.ts'], + includeDuplication: false, + }) + expect(selectLintShard('package-tests-n-z')).toEqual({ + eslintTargets: ['packages/[n-z]*/*/tests/**/*.ts'], + includeDuplication: false, + }) expect(selectLintShard('repository')).toEqual({ eslintTargets: [ '.', @@ -56,15 +72,20 @@ describe('lint gate shards', () => { }) }) - it('assigns every package group to one alphabetic range', () => { + it('assigns every package group once in the Linux and Windows topologies', () => { const groups = readdirSync(packagesRoot, { withFileTypes: true }) .filter(entry => entry.isDirectory()) .map(entry => entry.name) .sort() - const ranges = [/^[a-c]/u, /^[d-m]/u, /^[n-s]/u, /^[t-z]/u] - const assignments = ranges.flatMap(range => groups.filter(group => range.test(group))).sort() + const topologies = [ + [/^[a-c]/u, /^[d-m]/u, /^[n-s]/u, /^[t-z]/u], + [/^[a-m]/u, /^[n-z]/u], + ] - expect(assignments).toEqual(groups) + for (const ranges of topologies) { + const assignments = ranges.flatMap(range => groups.filter(group => range.test(group))).sort() + expect(assignments).toEqual(groups) + } }) it('rejects an unknown lane', () => { diff --git a/scripts/lint-shards.ts b/scripts/lint-shards.ts index e9a12c3e80..3f1ee9dd54 100644 --- a/scripts/lint-shards.ts +++ b/scripts/lint-shards.ts @@ -27,6 +27,10 @@ export function selectLintShard(name?: string): LintSelection { return { eslintTargets: ['packages/[n-s]*/*/src/**/*.ts'], includeDuplication: false } case 'package-sources-t-z': return { eslintTargets: ['packages/[t-z]*/*/src/**/*.ts'], includeDuplication: false } + case 'package-sources-a-m': + return { eslintTargets: ['packages/[a-m]*/*/src/**/*.ts'], includeDuplication: false } + case 'package-sources-n-z': + return { eslintTargets: ['packages/[n-z]*/*/src/**/*.ts'], includeDuplication: false } case 'package-tests-a-c': return { eslintTargets: ['packages/[a-c]*/*/tests/**/*.ts'], includeDuplication: false } case 'package-tests-d-m': @@ -35,6 +39,10 @@ export function selectLintShard(name?: string): LintSelection { return { eslintTargets: ['packages/[n-s]*/*/tests/**/*.ts'], includeDuplication: false } case 'package-tests-t-z': return { eslintTargets: ['packages/[t-z]*/*/tests/**/*.ts'], includeDuplication: false } + case 'package-tests-a-m': + return { eslintTargets: ['packages/[a-m]*/*/tests/**/*.ts'], includeDuplication: false } + case 'package-tests-n-z': + return { eslintTargets: ['packages/[n-z]*/*/tests/**/*.ts'], includeDuplication: false } case 'repository': return { eslintTargets: [ From ab1c24301a423a81eb909c4e65d2710c134d2c7c Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 00:44:48 +0800 Subject: [PATCH 22/49] ci: rebalance hosted concurrency budget --- .../2026-07-06-parallel-github-ci-gates.md | 2 +- .github/workflows/ci.yml | 22 +++++--------- scripts/lint-shards.spec.ts | 29 +++++-------------- scripts/lint-shards.ts | 12 +++----- 4 files changed, 21 insertions(+), 44 deletions(-) diff --git a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md index ab705d9450..156e3691d1 100644 --- a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md +++ b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md @@ -14,7 +14,7 @@ The artifact boundary remains load-bearing. `publint`, `verify-node-next-types`, [CI](../../../../.github/workflows/ci.yml) treats one minute for non-Windows jobs and three minutes for Windows jobs as observed performance targets, not cancellation deadlines. Hosted-runner variance should leave complete timing evidence and useful failure logs instead of cancelling an otherwise-correct gate. The [serial cross-platform CI reference](2026-07-21-serial-cross-platform-ci-reference.md) independently runs the complete unsharded primary Node aggregate on Linux, macOS, and Windows so the optimized lane inventory is not its own completeness oracle. -[scripts/run-gates.ts](../../../../scripts/run-gates.ts) remains the common bounded scheduler, but GitHub supplies explicit shard names for the expensive gate families. [scripts/static-shards.ts](../../../../scripts/static-shards.ts) partitions static gates into foundation, documentation-type, API-contract, catalog, prose, documentation-projection, and documentation-build ownership and rejects a missing or duplicate gate assignment; low-cost ownership groups may share one hosted lane to stay below the runner-provisioning concurrency wave. Linux lint uses disjoint A-C, D-M, N-S, and T-Z package-source and package-test lanes, while the longer Windows target uses A-M and N-Z halves; both topologies include a repository complement that still starts from `.` so a new top-level lint target cannot disappear between shards and owns the single cross-file duplication run. [scripts/coverage-shards.ts](../../../../scripts/coverage-shards.ts) assigns every workspace package to exactly one source-coverage lane; its test expands the live package tree, so a new package makes CI red until it has an owner. Directory filters retain a trailing separator because Vitest positional filters match substrings and would otherwise admit prefix-named sibling packages. Each coverage lane includes only its owned source files, repeats the exhaustive companion topology test, and runs without a preceding build because the complete coverage suite passes from a tree with every generated `lib/` removed. +[scripts/run-gates.ts](../../../../scripts/run-gates.ts) remains the common bounded scheduler, but GitHub supplies explicit shard names for the expensive gate families. [scripts/static-shards.ts](../../../../scripts/static-shards.ts) partitions static gates into foundation, documentation-type, API-contract, catalog, prose, documentation-projection, and documentation-build ownership and rejects a missing or duplicate gate assignment; low-cost ownership groups may share one hosted lane to stay below the runner-provisioning concurrency wave. Linux lint uses disjoint A-C, D-M, N-S, and T-Z package-source and package-test lanes, while the longer Windows target uses one complete package-source lane and one complete package-test lane; both topologies include a repository complement that still starts from `.` so a new top-level lint target cannot disappear between shards and owns the single cross-file duplication run. [scripts/coverage-shards.ts](../../../../scripts/coverage-shards.ts) assigns every workspace package to exactly one source-coverage lane; its test expands the live package tree, so a new package makes CI red until it has an owner. Directory filters retain a trailing separator because Vitest positional filters match substrings and would otherwise admit prefix-named sibling packages. Each coverage lane includes only its owned source files, repeats the exhaustive companion topology test, and runs without a preceding build because the complete coverage suite passes from a tree with every generated `lib/` removed. Snapshot replay uses two explicit multi-file lanes and eight scenario partitions of the large ACP file. [scripts/snapshot-shards.ts](../../../../scripts/snapshot-shards.ts) owns that inventory, and its test discovers every file admitted by the snapshot config so a new file cannot land outside CI. Each snapshot job installs dependencies while its Linux runner installs bubblewrap, refreshing APT indexes and retrying if the hosted image's initial package install fails, then builds the shipped runtime and runs only its assigned replay surface. CI explicitly retains the suite's bounded concurrency of five subprocesses instead of clamping it to the runner's two logical CPUs, because replay spends most of its time waiting on child protocol I/O. Fixture guards still inspect the complete ACP scenario table in every partition. Static, lint, coverage, and snapshot sharding changes only GitHub scheduling: the ordinary local package scripts still run their complete suites. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6b6ede348b..0e4a895b85 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,10 +45,14 @@ jobs: command: pnpm run check:ci:static gate_concurrency: '1' static_shard: doc-types - - lane: static-api-catalogs + - lane: static-api-contracts command: pnpm run check:ci:static gate_concurrency: '4' - static_shard: 'api-contracts,catalogs' + static_shard: api-contracts + - lane: static-catalogs + command: pnpm run check:ci:static + gate_concurrency: '4' + static_shard: catalogs - lane: static-prose command: pnpm run check:ci:static gate_concurrency: '4' @@ -438,22 +442,12 @@ jobs: command: pnpm run check:ci:lint gate_concurrency: '1' eslint_cache: '1' - lint_shard: package-sources-a-m - - lane: lint-package-sources-n-z - command: pnpm run check:ci:lint - gate_concurrency: '1' - eslint_cache: '1' - lint_shard: package-sources-n-z + lint_shard: package-sources - lane: lint-package-tests command: pnpm run check:ci:lint gate_concurrency: '1' eslint_cache: '1' - lint_shard: package-tests-a-m - - lane: lint-package-tests-n-z - command: pnpm run check:ci:lint - gate_concurrency: '1' - eslint_cache: '1' - lint_shard: package-tests-n-z + lint_shard: package-tests - lane: lint-repository command: pnpm run check:ci:lint gate_concurrency: '1' diff --git a/scripts/lint-shards.spec.ts b/scripts/lint-shards.spec.ts index e14a15b9ad..d3020fa6b2 100644 --- a/scripts/lint-shards.spec.ts +++ b/scripts/lint-shards.spec.ts @@ -44,20 +44,12 @@ describe('lint gate shards', () => { eslintTargets: ['packages/[t-z]*/*/tests/**/*.ts'], includeDuplication: false, }) - expect(selectLintShard('package-sources-a-m')).toEqual({ - eslintTargets: ['packages/[a-m]*/*/src/**/*.ts'], + expect(selectLintShard('package-sources')).toEqual({ + eslintTargets: ['packages/*/*/src/**/*.ts'], includeDuplication: false, }) - expect(selectLintShard('package-sources-n-z')).toEqual({ - eslintTargets: ['packages/[n-z]*/*/src/**/*.ts'], - includeDuplication: false, - }) - expect(selectLintShard('package-tests-a-m')).toEqual({ - eslintTargets: ['packages/[a-m]*/*/tests/**/*.ts'], - includeDuplication: false, - }) - expect(selectLintShard('package-tests-n-z')).toEqual({ - eslintTargets: ['packages/[n-z]*/*/tests/**/*.ts'], + expect(selectLintShard('package-tests')).toEqual({ + eslintTargets: ['packages/*/*/tests/**/*.ts'], includeDuplication: false, }) expect(selectLintShard('repository')).toEqual({ @@ -72,20 +64,15 @@ describe('lint gate shards', () => { }) }) - it('assigns every package group once in the Linux and Windows topologies', () => { + it('assigns every package group once in the Linux topology', () => { const groups = readdirSync(packagesRoot, { withFileTypes: true }) .filter(entry => entry.isDirectory()) .map(entry => entry.name) .sort() - const topologies = [ - [/^[a-c]/u, /^[d-m]/u, /^[n-s]/u, /^[t-z]/u], - [/^[a-m]/u, /^[n-z]/u], - ] + const ranges = [/^[a-c]/u, /^[d-m]/u, /^[n-s]/u, /^[t-z]/u] + const assignments = ranges.flatMap(range => groups.filter(group => range.test(group))).sort() - for (const ranges of topologies) { - const assignments = ranges.flatMap(range => groups.filter(group => range.test(group))).sort() - expect(assignments).toEqual(groups) - } + expect(assignments).toEqual(groups) }) it('rejects an unknown lane', () => { diff --git a/scripts/lint-shards.ts b/scripts/lint-shards.ts index 3f1ee9dd54..05c806c49f 100644 --- a/scripts/lint-shards.ts +++ b/scripts/lint-shards.ts @@ -27,10 +27,8 @@ export function selectLintShard(name?: string): LintSelection { return { eslintTargets: ['packages/[n-s]*/*/src/**/*.ts'], includeDuplication: false } case 'package-sources-t-z': return { eslintTargets: ['packages/[t-z]*/*/src/**/*.ts'], includeDuplication: false } - case 'package-sources-a-m': - return { eslintTargets: ['packages/[a-m]*/*/src/**/*.ts'], includeDuplication: false } - case 'package-sources-n-z': - return { eslintTargets: ['packages/[n-z]*/*/src/**/*.ts'], includeDuplication: false } + case 'package-sources': + return { eslintTargets: ['packages/*/*/src/**/*.ts'], includeDuplication: false } case 'package-tests-a-c': return { eslintTargets: ['packages/[a-c]*/*/tests/**/*.ts'], includeDuplication: false } case 'package-tests-d-m': @@ -39,10 +37,8 @@ export function selectLintShard(name?: string): LintSelection { return { eslintTargets: ['packages/[n-s]*/*/tests/**/*.ts'], includeDuplication: false } case 'package-tests-t-z': return { eslintTargets: ['packages/[t-z]*/*/tests/**/*.ts'], includeDuplication: false } - case 'package-tests-a-m': - return { eslintTargets: ['packages/[a-m]*/*/tests/**/*.ts'], includeDuplication: false } - case 'package-tests-n-z': - return { eslintTargets: ['packages/[n-z]*/*/tests/**/*.ts'], includeDuplication: false } + case 'package-tests': + return { eslintTargets: ['packages/*/*/tests/**/*.ts'], includeDuplication: false } case 'repository': return { eslintTargets: [ From 753d168bb1db705323ee7e58bda29569e47db0cc Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 13:55:51 +0800 Subject: [PATCH 23/49] ci: benchmark larger hosted runners --- .github/workflows/ci.yml | 115 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 112 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e4a895b85..574685cd45 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,15 @@ on: branches: [main, master] pull_request: workflow_dispatch: + inputs: + suite: + description: Manual CI suite to run + required: true + default: serial-reference + type: choice + options: + - serial-reference + - larger-runner-benchmark concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -510,7 +519,7 @@ jobs: # gate worker, giving reviewers a simple cross-platform oracle for completeness # and timing. serial-linux: - if: github.event_name == 'workflow_dispatch' + if: github.event_name == 'workflow_dispatch' && inputs.suite == 'serial-reference' name: serial / linux runs-on: ubuntu-latest steps: @@ -543,7 +552,7 @@ jobs: run: pnpm run check:ci serial-macos: - if: github.event_name == 'workflow_dispatch' + if: github.event_name == 'workflow_dispatch' && inputs.suite == 'serial-reference' name: serial / macos runs-on: macos-latest steps: @@ -569,7 +578,7 @@ jobs: run: pnpm run check:ci serial-windows: - if: github.event_name == 'workflow_dispatch' + if: github.event_name == 'workflow_dispatch' && inputs.suite == 'serial-reference' name: serial / windows runs-on: windows-2025 steps: @@ -603,6 +612,106 @@ jobs: DSH_SNAPSHOT_MAX_CONCURRENCY: '1' run: pnpm run check:ci + # Manual, bounded comparison of the actual critical Linux and Windows lanes. + # The named pools are restricted at the organization level to this repository. + larger-runner-benchmark: + if: github.event_name == 'workflow_dispatch' && inputs.suite == 'larger-runner-benchmark' + name: larger runner / ${{ matrix.platform }} / ${{ matrix.cores }} cores / ${{ matrix.workload }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 15 + strategy: + fail-fast: false + max-parallel: 12 + matrix: + include: + - platform: linux + cores: '4' + runner: dsh-ubuntu-24-04-4core + workload: typecheck + - platform: linux + cores: '8' + runner: dsh-ubuntu-24-04-8core + workload: typecheck + - platform: linux + cores: '16' + runner: dsh-ubuntu-24-04-16core + workload: typecheck + - platform: linux + cores: '32' + runner: dsh-ubuntu-24-04-32core + workload: typecheck + - platform: linux + cores: '64' + runner: dsh-ubuntu-24-04-64core + workload: typecheck + - platform: linux + cores: '96' + runner: dsh-ubuntu-24-04-96core + workload: typecheck + - platform: windows + cores: '4' + runner: dsh-windows-2025-4core + workload: production-site + - platform: windows + cores: '8' + runner: dsh-windows-2025-8core + workload: production-site + - platform: windows + cores: '16' + runner: dsh-windows-2025-16core + workload: production-site + - platform: windows + cores: '32' + runner: dsh-windows-2025-32core + workload: production-site + - platform: windows + cores: '64' + runner: dsh-windows-2025-64core + workload: production-site + - platform: windows + cores: '96' + runner: dsh-windows-2025-96core + workload: production-site + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-node@v6 + with: + node-version: ${{ env.PRIMARY_NODE_VERSION }} + + - name: Report runner capacity + run: >- + node -e "const os = require('node:os'); + console.log(JSON.stringify({ arch: process.arch, cpus: os.cpus().length, + memoryGiB: Math.round(os.totalmem() / 2 ** 30) }))" + + - name: Enable corepack (pnpm) + run: corepack enable + + - name: Resolve pnpm store path + if: matrix.platform == 'linux' + id: pnpm-store + run: echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" + + - uses: actions/cache@v4 + if: matrix.platform == 'linux' + with: + path: ${{ steps.pnpm-store.outputs.path }} + key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm- + + - name: Install (immutable) + run: pnpm install --frozen-lockfile + + - name: Run critical Linux typecheck lane + if: matrix.platform == 'linux' + run: pnpm run typecheck + + - name: Run critical Windows production-site lane + if: matrix.platform == 'windows' + run: pnpm run docs:build + # Single stable required check for branch protection: require "all checks # passed" instead of enumerating matrix legs whose names change as lanes and # node versions evolve. Every blocking job in THIS workflow must be listed in From da1bee87c79ba64ef5b30b6f11a34da66377257d Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 14:13:49 +0800 Subject: [PATCH 24/49] ci: use four-core hosted runners --- ...ence-based-larger-hosted-runners.i18n.yaml | 6 +++ ...22-evidence-based-larger-hosted-runners.md | 44 +++++++++++++++++++ ...evidence-based-larger-hosted-runners.zh.md | 44 +++++++++++++++++++ .github/workflows/ci.yml | 23 +++++----- 4 files changed, 106 insertions(+), 11 deletions(-) create mode 100644 .agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml create mode 100644 .agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md create mode 100644 .agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml new file mode 100644 index 0000000000..303a021a38 --- /dev/null +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml @@ -0,0 +1,6 @@ +# Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each +# side as of the last confirmed-consistent state. Both languages carry equal authority; +# after editing either side, bring the other along and re-record with: +# pnpm run verify-translation-pairing --write +2026-07-22-evidence-based-larger-hosted-runners.md: 992a622f0d7a2662cc65353d6c4cd1350e1f3f5f +2026-07-22-evidence-based-larger-hosted-runners.zh.md: 622911af8400fbe159038b69b440c0e97fb1f2cc diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md new file mode 100644 index 0000000000..992a622f0d --- /dev/null +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md @@ -0,0 +1,44 @@ +# Agent Note: Evidence-based larger hosted runners + +Status: implemented + +English | [中文](2026-07-22-evidence-based-larger-hosted-runners.zh.md) + +## Problem + +The sharded primary CI fits its latency targets on standard GitHub-hosted runners, but the margin depends on cold setup and install variance. Larger runners may add useful headroom, yet their per-minute price rises much faster than these short lanes can use extra cores. Selecting a size from machine specifications or a synthetic benchmark would spend more without proving that repository CI becomes faster. + +## Decision + +The organization keeps twelve x64 larger-runner pools in the repo-restricted `dsh-larger-ci` group: Ubuntu 24.04 and Windows 2025 at 4, 8, 16, 32, 64, and 96 cores. Public IPs are disabled. Each pool has an autoscaling ceiling of 256, while the repository bounds actual demand through its workflow matrices; an idle ceiling does not allocate machines. + +The `CI` workflow exposes `suite=larger-runner-benchmark` only through manual dispatch. Its six Linux legs run the critical typecheck lane, and its six Windows legs run the critical production-site lane. Every leg reports the observed CPU and memory, has a 15-minute timeout, and uses the same setup and caching policy as the production lane it represents. Push and pull-request events skip this benchmark. + +The [twelve-size benchmark](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29895295659) used a workflow-only commit on top of the standard-runner [baseline](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29850033610), so the code, lockfile, and critical commands were identical: + +| Critical job | Standard | 4 cores | 8 cores | 16 cores | 32 cores | 64 cores | 96 cores | +|---|---:|---:|---:|---:|---:|---:|---:| +| Linux typecheck | 56 s | 38 s | 35 s | 40 s | 35 s | 44 s | 40 s | +| Windows production site | 160 s | 117 s | 103 s | 113 s | 75 s | 105 s | 108 s | + +The repository therefore uses the 4-core pools for the primary Node matrix and all Windows jobs. Linux 4-core finished within three seconds of the fastest measured size at the lowest larger-runner rate. Windows 4-core stayed below two minutes; the isolated 32-core result was faster, but adjacent larger sizes regressed and the production-site command itself varied only from 28 to 36 seconds, so setup and install noise—not scalable compute—created most of the spread. Node compatibility, Python, and the final aggregator remain on standard runners because their baseline jobs already finish well below one minute. + +The workflow also exposes `suite=optimized-larger-runners` through manual dispatch. That path runs the production matrices against the branch ref itself, providing an exact-head timing check when a pull request cannot form a merge commit. + +## Alternatives considered + +**Keep every job on standard runners.** This meets the threshold but gives the critical lanes no cold-run margin and leaves the larger-runner suggestion untested. + +**Select 8 or 32 cores from the fastest individual result.** The small differences were not monotonic, while billing grows sharply with size. Treating one noisy minimum as scaling evidence would make recurring CI substantially more expensive. + +**Move every job to a larger runner.** Compatibility, Python, and aggregation were already short; paying the larger-runner premium there cannot improve the critical path enough to justify the dependency or cost. + +**Use a synthetic CPU benchmark.** A microbenchmark would not include checkout, action startup, package installation, cache restore, or the repository command mix that dominates these jobs. + +## Consequences + +The benchmark incurred $2.936 across dedicated larger-runner SKUs, as recorded by organization billing immediately after the run. The existing zero-dollar Actions budget did not block those jobs, so the repo-only runner group, manual benchmark trigger, matrix width, and timeout are the observed cost controls; the budget is not treated as an execution guard. + +Production CI now depends on the organization-owned runner names in this note and in `.github/workflows/ci.yml`. Missing or renamed pools leave jobs queued instead of silently falling back to standard capacity. The 256 autoscaling ceiling permits future concurrency but does not override the bounded matrices in the current workflow. + +The selected pools buy latency headroom at a recurring per-job premium. The manual benchmark retains all sizes so a future image, dependency graph, or workload change can be measured before changing the production labels again. diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md new file mode 100644 index 0000000000..622911af84 --- /dev/null +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md @@ -0,0 +1,44 @@ +# Agent Note: 基于实证选用 GitHub 托管大型运行器 + +Status: implemented + +[English](2026-07-22-evidence-based-larger-hosted-runners.md) | 中文 + +## 问题 + +分片后的主 CI 在标准 GitHub 托管运行器上能够达到延迟目标，但余量会受冷启动设置和安装耗时波动影响。大型运行器或许能提供更多有效余量，但随着核心数增加，其每分钟单价增长得远比这些短通道能够利用额外核心的速度快。仅根据机器规格或合成基准测试选择配置，只会增加支出，无法证明仓库 CI 确实变快。 + +## 决策 + +组织在仅限本仓库使用的 `dsh-larger-ci` 运行器组中保留 12 个 x64 大型运行器池：Ubuntu 24.04 和 Windows 2025 各设 4、8、16、32、64、96 核规格。公网 IP 已禁用。每个池的自动扩缩容上限为 256，仓库则通过工作流矩阵限制实际需求；空闲状态下的上限不会分配机器。 + +`CI` 工作流仅在手动触发时提供 `suite=larger-runner-benchmark`。其中 6 个 Linux 矩阵分支运行关键类型检查通道，6 个 Windows 矩阵分支运行生产网站关键通道。每个矩阵分支都会报告实测 CPU 和内存，超时限制为 15 分钟，并采用与其所代表的生产通道相同的设置和缓存策略。push 和拉取请求事件会跳过这项基准测试。 + +这项[涵盖 12 种规格的基准测试](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29895295659)以标准运行器[基线](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29850033610)为基础，只叠加了一个仅修改工作流的提交，因此代码、锁文件和关键命令完全相同： + +| 关键作业 | 标准 | 4 核 | 8 核 | 16 核 | 32 核 | 64 核 | 96 核 | +|---|---:|---:|---:|---:|---:|---:|---:| +| Linux 类型检查 | 56 秒 | 38 秒 | 35 秒 | 40 秒 | 35 秒 | 44 秒 | 40 秒 | +| Windows 生产网站 | 160 秒 | 117 秒 | 103 秒 | 113 秒 | 75 秒 | 105 秒 | 108 秒 | + +因此，仓库让主 Node 矩阵和所有 Windows 作业使用 4 核池。Linux 4 核规格比实测最快规格至多慢 3 秒，同时采用大型运行器中的最低费率。Windows 4 核规格保持在 2 分钟以内；单次测得的 32 核结果虽然更快，但相邻的大型运行器规格反而耗时回升，而生产网站命令本身的耗时仅在 28 至 36 秒之间波动，因此大部分差异来自设置和安装噪声，而非可随核心数扩展的计算负载。Node 兼容性、Python 和最终聚合作业仍使用标准运行器，因为它们的基线耗时本就远低于 1 分钟。 + +工作流还在手动触发时提供 `suite=optimized-larger-runners`。这一路径直接以分支引用本身运行生产矩阵；当拉取请求无法生成合并提交时，它仍能针对该分支实际的头部提交提供精确计时检查。 + +## 曾考虑的替代方案 + +**所有作业均保留在标准运行器上。** 此方案能达到时长目标，但关键通道面对冷启动时没有余量，也无法验证大型运行器方案。 + +**根据单次最快结果选择 8 核或 32 核。** 各规格间的微小差异并不呈单调变化，计费却随规格急剧增长。把一次带有噪声的最低值当作扩展性证据，会大幅提高 CI 的经常性成本。 + +**将所有作业迁移到大型运行器。** 兼容性、Python 和聚合作业原本就很短；为这些作业支付大型运行器溢价，无法充分缩短关键路径，因而不足以证明这种依赖或成本合理。 + +**使用合成 CPU 基准测试。** 微基准测试不会包含代码检出、操作启动、包安装、缓存恢复或主导这些作业耗时的仓库命令组合。 + +## 后果 + +这次基准测试使用专用大型运行器 SKU，共产生 $2.936 费用，组织计费系统在运行结束后立即记录了这笔费用。现有的零美元 Actions 预算并未阻止这些作业，因此经实测有效的成本控制机制是仅限本仓库的运行器组、手动基准测试触发、矩阵宽度和超时限制；该预算不被视为执行防护措施。 + +生产 CI 目前依赖本 Agent Note 和 `.github/workflows/ci.yml` 中由组织持有的运行器名称。池缺失或改名会让作业一直排队，不会静默回退到标准容量。256 的自动扩缩容上限允许未来提高并发量，但不会突破当前工作流对矩阵规模的限制。 + +选定的运行器池以每次作业都需支付的溢价换取延迟余量。手动基准测试保留了所有规格，以便未来映像、依赖图或工作负载发生变化时，先完成测量，再次调整生产标签。 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 574685cd45..060078cc1f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,6 +14,7 @@ on: options: - serial-reference - larger-runner-benchmark + - optimized-larger-runners concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -27,8 +28,8 @@ env: jobs: node-24: - if: github.event_name != 'workflow_dispatch' - runs-on: ubuntu-latest + if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' + runs-on: dsh-ubuntu-24-04-4core name: node 24 / ${{ matrix.lane }} env: DSH_GATE_CONCURRENCY: ${{ matrix.gate_concurrency }} @@ -316,7 +317,7 @@ jobs: run: ${{ matrix.command }} node-compat: - if: github.event_name != 'workflow_dispatch' + if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' runs-on: ubuntu-latest name: node ${{ matrix.node }} env: @@ -359,7 +360,7 @@ jobs: run: pnpm run check:node-compat python-sdk: - if: github.event_name != 'workflow_dispatch' + if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' runs-on: ubuntu-latest name: python 3.10 / keyless SDK steps: @@ -381,8 +382,8 @@ jobs: # The broader observational gate matrix below exposes the remaining # portability work without blocking mainline merges. windows-build: - if: github.event_name != 'workflow_dispatch' - runs-on: windows-2025 + if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' + runs-on: dsh-windows-2025-4core name: windows / build steps: - uses: actions/checkout@v6 @@ -400,8 +401,8 @@ jobs: run: pnpm run build windows-site: - if: github.event_name != 'workflow_dispatch' - runs-on: windows-2025 + if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' + runs-on: dsh-windows-2025-4core name: windows / production site steps: - uses: actions/checkout@v6 @@ -424,9 +425,9 @@ jobs: # would change the environment being measured. This job intentionally stays # out of all-checks-passed.needs. windows-gates: - if: github.event_name != 'workflow_dispatch' + if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' continue-on-error: true - runs-on: windows-2025 + runs-on: dsh-windows-2025-4core name: windows node 24 / ${{ matrix.lane }} env: DSH_GATE_CONCURRENCY: ${{ matrix.gate_concurrency }} @@ -725,7 +726,7 @@ jobs: name: all checks passed runs-on: ubuntu-latest needs: [node-24, node-compat, python-sdk, windows-build, windows-site] - if: always() && github.event_name != 'workflow_dispatch' + if: always() && (github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners') steps: - name: Fail if any needed job did not succeed if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'skipped') From 95abe24cb79b680692cfddc488b8e7440047af3e Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 14:28:32 +0800 Subject: [PATCH 25/49] ci: give snapshot lanes eight cores --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 060078cc1f..f601bc1a2c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,7 @@ env: jobs: node-24: if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' - runs-on: dsh-ubuntu-24-04-4core + runs-on: ${{ startsWith(matrix.lane, 'snapshot-') && 'dsh-ubuntu-24-04-8core' || 'dsh-ubuntu-24-04-4core' }} name: node 24 / ${{ matrix.lane }} env: DSH_GATE_CONCURRENCY: ${{ matrix.gate_concurrency }} From e65ea9bdbc2b58fc6b2640c087ee215db935ee93 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 14:42:43 +0800 Subject: [PATCH 26/49] ci: benchmark consolidated hosted gates --- .github/workflows/ci.yml | 148 +++++++++++++++++++++++++++++++++++++++ package.json | 2 + scripts/run-gates.ts | 32 ++++++++- 3 files changed, 181 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f601bc1a2c..a41eb480d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,6 +14,7 @@ on: options: - serial-reference - larger-runner-benchmark + - consolidated-runner-benchmark - optimized-larger-runners concurrency: @@ -713,6 +714,153 @@ jobs: if: matrix.platform == 'windows' run: pnpm run docs:build + # Manual comparison of the intended low-fanout topology. Linux runs the + # complete unsharded primary aggregate with bounded in-runner parallelism; + # Windows runs both blocking build surfaces concurrently through run-gates. + consolidated-runner-benchmark: + if: github.event_name == 'workflow_dispatch' && inputs.suite == 'consolidated-runner-benchmark' + name: consolidated / ${{ matrix.platform }} / ${{ matrix.cores }} cores + runs-on: ${{ matrix.runner }} + timeout-minutes: 15 + strategy: + fail-fast: false + max-parallel: 12 + matrix: + include: + - platform: linux + cores: '4' + runner: dsh-ubuntu-24-04-4core + workers: '4' + - platform: linux + cores: '8' + runner: dsh-ubuntu-24-04-8core + workers: '8' + - platform: linux + cores: '16' + runner: dsh-ubuntu-24-04-16core + workers: '16' + - platform: linux + cores: '32' + runner: dsh-ubuntu-24-04-32core + workers: '32' + - platform: linux + cores: '64' + runner: dsh-ubuntu-24-04-64core + workers: '32' + - platform: linux + cores: '96' + runner: dsh-ubuntu-24-04-96core + workers: '32' + - platform: windows + cores: '4' + runner: dsh-windows-2025-4core + workers: '2' + - platform: windows + cores: '8' + runner: dsh-windows-2025-8core + workers: '2' + - platform: windows + cores: '16' + runner: dsh-windows-2025-16core + workers: '2' + - platform: windows + cores: '32' + runner: dsh-windows-2025-32core + workers: '2' + - platform: windows + cores: '64' + runner: dsh-windows-2025-64core + workers: '2' + - platform: windows + cores: '96' + runner: dsh-windows-2025-96core + workers: '2' + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-node@v6 + with: + node-version: ${{ env.PRIMARY_NODE_VERSION }} + + - name: Report runner capacity + run: >- + node -e "const os = require('node:os'); + console.log(JSON.stringify({ arch: process.arch, cpus: os.cpus().length, + memoryGiB: Math.round(os.totalmem() / 2 ** 30) }))" + + - name: Enable corepack (pnpm) + run: corepack enable + + - name: Resolve pnpm store path (Linux) + if: matrix.platform == 'linux' + id: pnpm-store-linux + run: echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" + + - name: Resolve pnpm store path (Windows) + if: matrix.platform == 'windows' + id: pnpm-store-windows + shell: pwsh + run: '"path=$(pnpm store path --silent)" >> $env:GITHUB_OUTPUT' + + - uses: actions/cache@v4 + with: + path: ${{ steps.pnpm-store-linux.outputs.path || steps.pnpm-store-windows.outputs.path }} + key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm- + + - uses: actions/cache@v4 + if: matrix.platform == 'linux' + with: + path: .cache/eslint + key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-full-${{ hashFiles('pnpm-lock.yaml', 'eslint.config.mjs', 'tsconfig.json', 'packages/*/*/tsconfig.json', 'examples/*/tsconfig.json') }} + restore-keys: | + ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-full- + + - name: Install and prepare Linux + if: matrix.platform == 'linux' + run: | + pnpm install --frozen-lockfile & + install_pid=$! + ( + if ! sudo apt-get install -yq --no-install-recommends bubblewrap; then + echo "initial bubblewrap install failed; refreshing APT indexes and retrying" + sudo apt-get update -q + sudo apt-get install -yq --no-install-recommends bubblewrap + fi + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 \ + || echo "apparmor userns knob absent — the functional probe decides" + ) & + sandbox_pid=$! + install_status=0 + wait "$install_pid" || install_status=$? + sandbox_status=0 + wait "$sandbox_pid" || sandbox_status=$? + if (( install_status != 0 )); then exit "$install_status"; fi + exit "$sandbox_status" + + - name: Install (immutable) + if: matrix.platform == 'windows' + shell: pwsh + run: pnpm install --frozen-lockfile + + - name: Run complete unsharded primary Node CI concurrently + if: matrix.platform == 'linux' + env: + DSH_COVERAGE_MAX_WORKERS: ${{ matrix.workers }} + DSH_ESLINT_CACHE: '1' + DSH_GATE_CONCURRENCY: ${{ matrix.workers }} + DSH_PUBLINT_CONCURRENCY: ${{ matrix.workers }} + DSH_SNAPSHOT_MAX_CONCURRENCY: ${{ matrix.workers }} + run: pnpm run check:ci + + - name: Run blocking Windows builds concurrently + if: matrix.platform == 'windows' + shell: pwsh + env: + DSH_GATE_CONCURRENCY: ${{ matrix.workers }} + run: pnpm run check:ci:windows-blocking + # Single stable required check for branch protection: require "all checks # passed" instead of enumerating matrix legs whose names change as lanes and # node versions evolve. Every blocking job in THIS workflow must be listed in diff --git a/package.json b/package.json index 9dca83dc6a..b758974ef7 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,8 @@ "check:ci:coverage": "tsx scripts/run-gates.ts ci-coverage", "check:ci:snapshot": "tsx scripts/run-gates.ts ci-snapshot", "check:ci:artifacts": "tsx scripts/run-gates.ts ci-artifacts", + "check:ci:windows-blocking": "tsx scripts/run-gates.ts ci-windows-blocking", + "check:ci:windows-observational": "tsx scripts/run-gates.ts ci-windows-observational", "check:node-compat": "tsx scripts/run-gates.ts node-compat", "check:pre-push": "tsx scripts/run-gates.ts pre-push", "knip": "knip --treat-config-hints-as-errors", diff --git a/scripts/run-gates.ts b/scripts/run-gates.ts index 504dc26ba7..b117c62d11 100644 --- a/scripts/run-gates.ts +++ b/scripts/run-gates.ts @@ -20,6 +20,8 @@ type Mode = | 'ci-coverage' | 'ci-snapshot' | 'ci-artifacts' + | 'ci-windows-blocking' + | 'ci-windows-observational' | 'node-compat' | 'pre-push' | 'doc-sync' @@ -90,13 +92,15 @@ function parseMode(raw: string | undefined): Mode { case 'ci-coverage': case 'ci-snapshot': case 'ci-artifacts': + case 'ci-windows-blocking': + case 'ci-windows-observational': case 'node-compat': case 'pre-push': case 'doc-sync': return raw default: throw new Error( - `run-gates: expected mode ci-primary | ci-static | ci-lint | ci-coverage | ci-snapshot | ci-artifacts | node-compat | pre-push | doc-sync, got ${JSON.stringify(raw)}.`, + `run-gates: expected mode ci-primary | ci-static | ci-lint | ci-coverage | ci-snapshot | ci-artifacts | ci-windows-blocking | ci-windows-observational | node-compat | pre-push | doc-sync, got ${JSON.stringify(raw)}.`, ) } } @@ -179,6 +183,10 @@ function gatesForMode(selected: Mode): Gate[] { : [pnpmScript('build', 'build'), snapshotGate()] case 'ci-artifacts': return ciArtifactGates() + case 'ci-windows-blocking': + return ciWindowsBlockingGates() + case 'ci-windows-observational': + return ciWindowsObservationalGates() case 'node-compat': return [ ...flagEnabled('DSH_NODE_COMPAT_SKIP_TYPECHECK') ? [] : [pnpmScript('typecheck', 'typecheck')], @@ -275,6 +283,28 @@ function ciArtifactGates(): Gate[] { return [...metadataGates, builtBinSmokeGate()] } +function ciWindowsBlockingGates(): Gate[] { + return [ + pnpmScript('windows-build', 'build', { label: 'build' }), + pnpmScript('windows-site', 'docs:build', { label: 'production site' }), + ] +} + +function ciWindowsObservationalGates(): Gate[] { + return [ + ...ciStaticGates(), + lintGate(), + pnpmScript('duplication', 'duplication'), + pnpmScript('publint', 'publint', { needs: ['build'] }), + pnpmScript('node-next-types', 'verify-node-next-types', { + label: 'node-next types', + needs: ['build'], + }), + builtPackageInvariantsGate(['build']), + builtBinSmokeGate(), + ] +} + function lintGate(eslintTargets: readonly string[] = ['.']): Gate { if (process.env.DSH_ESLINT_CACHE === '1') { return pnpmExec('lint', [ From 03348aa382c5775fbe92eadc0854079e2d277121 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 14:53:46 +0800 Subject: [PATCH 27/49] ci: parallelize aggregate lint --- .github/workflows/ci.yml | 1 + scripts/run-gates.ts | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a41eb480d5..719e2f8e9b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -849,6 +849,7 @@ jobs: env: DSH_COVERAGE_MAX_WORKERS: ${{ matrix.workers }} DSH_ESLINT_CACHE: '1' + DSH_ESLINT_CONCURRENCY: ${{ matrix.workers }} DSH_GATE_CONCURRENCY: ${{ matrix.workers }} DSH_PUBLINT_CONCURRENCY: ${{ matrix.workers }} DSH_SNAPSHOT_MAX_CONCURRENCY: ${{ matrix.workers }} diff --git a/scripts/run-gates.ts b/scripts/run-gates.ts index b117c62d11..98326d6db9 100644 --- a/scripts/run-gates.ts +++ b/scripts/run-gates.ts @@ -306,10 +306,12 @@ function ciWindowsObservationalGates(): Gate[] { } function lintGate(eslintTargets: readonly string[] = ['.']): Gate { + const concurrencyArgs = eslintConcurrencyArgs() if (process.env.DSH_ESLINT_CACHE === '1') { return pnpmExec('lint', [ 'eslint', ...eslintTargets, + ...concurrencyArgs, '--cache', '--cache-location', '.cache/eslint/', @@ -320,11 +322,28 @@ function lintGate(eslintTargets: readonly string[] = ['.']): Gate { env: { NODE_OPTIONS: nodeOptions('--max-old-space-size=8192') }, }) } + if (concurrencyArgs.length > 0) { + return pnpmExec('lint', ['eslint', ...eslintTargets, ...concurrencyArgs], { + label: 'lint', + env: { NODE_OPTIONS: nodeOptions('--max-old-space-size=8192') }, + }) + } return pnpmScript('lint', 'lint', { env: { NODE_OPTIONS: nodeOptions('--max-old-space-size=8192') }, }) } +function eslintConcurrencyArgs(): string[] { + const raw = process.env.DSH_ESLINT_CONCURRENCY + if (raw === undefined || raw === '') return [] + if (raw === 'auto') return ['--concurrency=auto'] + const parsed = Number.parseInt(raw, 10) + if (!Number.isSafeInteger(parsed) || parsed < 1 || String(parsed) !== raw) { + throw new Error(`run-gates: DSH_ESLINT_CONCURRENCY must be a positive integer or auto, got ${JSON.stringify(raw)}.`) + } + return [`--concurrency=${raw}`] +} + function coverageGate(): Gate { const shard = process.env.DSH_COVERAGE_SHARD return pnpmExec('coverage', [ From 3db0b71f91e1c90b73c36fddae74c169627b6966 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 15:04:24 +0800 Subject: [PATCH 28/49] ci: consolidate gates on large runners --- .github/workflows/ci.yml | 190 ++++++++++++++++++++++++++++++++++++--- package.json | 1 + scripts/run-gates.ts | 14 ++- 3 files changed, 192 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 719e2f8e9b..0e1135f447 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,7 @@ on: - serial-reference - larger-runner-benchmark - consolidated-runner-benchmark + - sharded-reference - optimized-larger-runners concurrency: @@ -28,8 +29,8 @@ env: PRIMARY_NODE_VERSION: '24' jobs: - node-24: - if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' + node-24-sharded: + if: github.event_name == 'workflow_dispatch' && inputs.suite == 'sharded-reference' runs-on: ${{ startsWith(matrix.lane, 'snapshot-') && 'dsh-ubuntu-24-04-8core' || 'dsh-ubuntu-24-04-4core' }} name: node 24 / ${{ matrix.lane }} env: @@ -317,6 +318,75 @@ jobs: - name: Run gates run: ${{ matrix.command }} + # One large runner pays hosted setup once, then the repository scheduler + # overlaps the complete unsharded primary gate inventory inside that box. + node-24: + if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' + runs-on: dsh-ubuntu-24-04-96core + name: node 24 / complete + env: + DSH_COVERAGE_MAX_WORKERS: '32' + DSH_ESLINT_CACHE: '1' + DSH_ESLINT_CONCURRENCY: '32' + DSH_GATE_CONCURRENCY: '32' + DSH_PUBLINT_CONCURRENCY: '32' + DSH_SNAPSHOT_MAX_CONCURRENCY: '32' + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-node@v6 + with: + node-version: ${{ env.PRIMARY_NODE_VERSION }} + + - name: Enable corepack and resolve pnpm store path + id: pnpm-store + run: | + corepack enable + echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" + + - uses: actions/cache@v4 + with: + path: ${{ steps.pnpm-store.outputs.path }} + key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm- + + - uses: actions/cache@v4 + with: + path: .cache/eslint + key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-full-${{ hashFiles('pnpm-lock.yaml', 'eslint.config.mjs', 'tsconfig.json', 'packages/*/*/tsconfig.json', 'examples/*/tsconfig.json') }} + restore-keys: | + ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-full- + + - name: Install, build, and prepare bubblewrap + run: | + pnpm install --frozen-lockfile & + install_pid=$! + ( + if ! sudo apt-get install -yq --no-install-recommends bubblewrap; then + echo "initial bubblewrap install failed; refreshing APT indexes and retrying" + sudo apt-get update -q + sudo apt-get install -yq --no-install-recommends bubblewrap + fi + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 \ + || echo "apparmor userns knob absent — the functional probe decides" + ) & + sandbox_pid=$! + install_status=0 + wait "$install_pid" || install_status=$? + build_status=0 + if (( install_status == 0 )); then + pnpm run build || build_status=$? + fi + sandbox_status=0 + wait "$sandbox_pid" || sandbox_status=$? + if (( install_status != 0 )); then exit "$install_status"; fi + if (( build_status != 0 )); then exit "$build_status"; fi + exit "$sandbox_status" + + - name: Run complete unsharded primary Node CI concurrently + run: pnpm run check:ci:prebuilt + node-compat: if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' runs-on: ubuntu-latest @@ -378,12 +448,13 @@ jobs: - name: Run complete keyless Python suite run: uv run --python 3.10 --group test --project python/sdk pytest - # Blocking Windows build and production-site lanes keep the already-green + # Manual sharded Windows reference matching the former production topology. + # Blocking build and production-site lanes keep the already-green # native outputs protected without putting both critical paths in one job. # The broader observational gate matrix below exposes the remaining # portability work without blocking mainline merges. - windows-build: - if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' + windows-build-sharded: + if: github.event_name == 'workflow_dispatch' && inputs.suite == 'sharded-reference' runs-on: dsh-windows-2025-4core name: windows / build steps: @@ -401,8 +472,8 @@ jobs: - name: Build (tsc -b + tsdown) run: pnpm run build - windows-site: - if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' + windows-site-sharded: + if: github.event_name == 'workflow_dispatch' && inputs.suite == 'sharded-reference' runs-on: dsh-windows-2025-4core name: windows / production site steps: @@ -425,8 +496,8 @@ jobs: # have dedicated support. Run the gates from native PowerShell: an MSYS parent # would change the environment being measured. This job intentionally stays # out of all-checks-passed.needs. - windows-gates: - if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' + windows-gates-sharded: + if: github.event_name == 'workflow_dispatch' && inputs.suite == 'sharded-reference' continue-on-error: true runs-on: dsh-windows-2025-4core name: windows node 24 / ${{ matrix.lane }} @@ -516,7 +587,97 @@ jobs: shell: pwsh run: ${{ matrix.command }} - # Manual reference runs deliberately avoid the optimized matrices above. + # The two blocking Windows build surfaces share one setup and run concurrently. + windows-build: + if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' + runs-on: dsh-windows-2025-16core + name: windows / blocking builds + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-node@v6 + with: + node-version: ${{ env.PRIMARY_NODE_VERSION }} + + - name: Enable corepack and resolve pnpm store path + id: pnpm-store + shell: pwsh + run: | + corepack enable + "path=$(pnpm store path --silent)" >> $env:GITHUB_OUTPUT + + - uses: actions/cache@v4 + with: + path: ${{ steps.pnpm-store.outputs.path }} + key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm- + + - name: Install (immutable) + shell: pwsh + run: pnpm install --frozen-lockfile + + - name: Run blocking Windows builds concurrently + shell: pwsh + env: + DSH_GATE_CONCURRENCY: '2' + run: pnpm run check:ci:windows-blocking + + # Wider Windows portability gates remain observational, but one larger box + # now runs their complete unsharded inventory with in-runner concurrency. + windows-gates: + if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' + continue-on-error: true + runs-on: dsh-windows-2025-16core + name: windows node 24 / observational + env: + DSH_ESLINT_CACHE: '1' + DSH_ESLINT_CONCURRENCY: '16' + DSH_GATE_CONCURRENCY: '16' + DSH_PUBLINT_CONCURRENCY: '16' + steps: + - uses: actions/checkout@v6 + + - name: Enable Developer Mode (symlink support) + shell: pwsh + run: >- + reg add "HKLM\SOFTWARE\Microsoft\Windows\CurrentVersion\AppModelUnlock" + /t REG_DWORD /f /v "AllowDevelopmentWithoutDevLicense" /d "1" + + - uses: actions/setup-node@v6 + with: + node-version: ${{ env.PRIMARY_NODE_VERSION }} + + - name: Enable corepack and resolve pnpm store path + id: pnpm-store + shell: pwsh + run: | + corepack enable + "path=$(pnpm store path --silent)" >> $env:GITHUB_OUTPUT + + - uses: actions/cache@v4 + with: + path: ${{ steps.pnpm-store.outputs.path }} + key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm- + + - uses: actions/cache@v4 + with: + path: .cache/eslint + key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-full-${{ hashFiles('pnpm-lock.yaml', 'eslint.config.mjs', 'tsconfig.json', 'packages/*/*/tsconfig.json', 'examples/*/tsconfig.json') }} + restore-keys: | + ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-full- + + - name: Install (immutable) + shell: pwsh + run: pnpm install --frozen-lockfile + + - name: Run observational Windows gates concurrently + shell: pwsh + run: pnpm run check:ci:windows-observational + + # Manual reference runs deliberately avoid the optimized jobs above. # Each host executes the complete, unsharded primary Node aggregate with one # gate worker, giving reviewers a simple cross-platform oracle for completeness # and timing. @@ -834,9 +995,14 @@ jobs: sandbox_pid=$! install_status=0 wait "$install_pid" || install_status=$? + build_status=0 + if (( install_status == 0 )); then + pnpm run build || build_status=$? + fi sandbox_status=0 wait "$sandbox_pid" || sandbox_status=$? if (( install_status != 0 )); then exit "$install_status"; fi + if (( build_status != 0 )); then exit "$build_status"; fi exit "$sandbox_status" - name: Install (immutable) @@ -853,7 +1019,7 @@ jobs: DSH_GATE_CONCURRENCY: ${{ matrix.workers }} DSH_PUBLINT_CONCURRENCY: ${{ matrix.workers }} DSH_SNAPSHOT_MAX_CONCURRENCY: ${{ matrix.workers }} - run: pnpm run check:ci + run: pnpm run check:ci:prebuilt - name: Run blocking Windows builds concurrently if: matrix.platform == 'windows' @@ -874,7 +1040,7 @@ jobs: all-checks-passed: name: all checks passed runs-on: ubuntu-latest - needs: [node-24, node-compat, python-sdk, windows-build, windows-site] + needs: [node-24, node-compat, python-sdk, windows-build] if: always() && (github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners') steps: - name: Fail if any needed job did not succeed diff --git a/package.json b/package.json index b758974ef7..a8ee301da3 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "test:snapshot:record": "DSH_SNAPSHOT=record vitest run --config vitest.snapshot.config.ts --update", "test:snapshot:refresh": "DSH_SNAPSHOT=refresh vitest run --config vitest.snapshot.config.ts", "check:ci": "tsx scripts/run-gates.ts ci-primary", + "check:ci:prebuilt": "tsx scripts/run-gates.ts ci-primary-prebuilt", "check:ci:static": "tsx scripts/run-gates.ts ci-static", "check:ci:lint": "tsx scripts/run-gates.ts ci-lint", "check:ci:coverage": "tsx scripts/run-gates.ts ci-coverage", diff --git a/scripts/run-gates.ts b/scripts/run-gates.ts index 98326d6db9..c9d058518b 100644 --- a/scripts/run-gates.ts +++ b/scripts/run-gates.ts @@ -15,6 +15,7 @@ import { selectStaticGates } from './static-shards.ts' type Mode = | 'ci-primary' + | 'ci-primary-prebuilt' | 'ci-static' | 'ci-lint' | 'ci-coverage' @@ -87,6 +88,7 @@ if (results.some(result => result.status === 'failed' || result.status === 'skip function parseMode(raw: string | undefined): Mode { switch (raw) { case 'ci-primary': + case 'ci-primary-prebuilt': case 'ci-static': case 'ci-lint': case 'ci-coverage': @@ -100,7 +102,7 @@ function parseMode(raw: string | undefined): Mode { return raw default: throw new Error( - `run-gates: expected mode ci-primary | ci-static | ci-lint | ci-coverage | ci-snapshot | ci-artifacts | ci-windows-blocking | ci-windows-observational | node-compat | pre-push | doc-sync, got ${JSON.stringify(raw)}.`, + `run-gates: expected mode ci-primary | ci-primary-prebuilt | ci-static | ci-lint | ci-coverage | ci-snapshot | ci-artifacts | ci-windows-blocking | ci-windows-observational | node-compat | pre-push | doc-sync, got ${JSON.stringify(raw)}.`, ) } } @@ -166,6 +168,8 @@ function gatesForMode(selected: Mode): Gate[] { switch (selected) { case 'ci-primary': return ciPrimaryGates() + case 'ci-primary-prebuilt': + return ciPrimaryPrebuiltGates() case 'ci-static': return ciStaticGates() case 'ci-lint': { @@ -246,6 +250,14 @@ function ciPrimaryGates(): Gate[] { ] } +function ciPrimaryPrebuiltGates(): Gate[] { + return ciPrimaryGates() + .filter(gate => gate.id !== 'build') + .map(gate => gate.needs?.includes('build') === true + ? { ...gate, needs: gate.needs.filter(id => id !== 'build') } + : gate) +} + function ciStaticGates(): Gate[] { const gates = [ pnpmScript('runtime-closure', 'verify-runtime-closure', { label: 'runtime closure' }), From 990f82b85cf748638c04272f8cada333ca2dee39 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 15:17:17 +0800 Subject: [PATCH 29/49] ci: tune consolidated runner concurrency --- .github/workflows/ci.yml | 28 ++++++++++------------------ package.json | 2 +- scripts/run-gates.ts | 24 ++++++++++++++---------- 3 files changed, 25 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e1135f447..121d82d701 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -325,7 +325,8 @@ jobs: runs-on: dsh-ubuntu-24-04-96core name: node 24 / complete env: - DSH_COVERAGE_MAX_WORKERS: '32' + # Thirty-two Vitest forks intermittently crash Node's CJS lexer on this image. + DSH_COVERAGE_MAX_WORKERS: '16' DSH_ESLINT_CACHE: '1' DSH_ESLINT_CONCURRENCY: '32' DSH_GATE_CONCURRENCY: '32' @@ -358,7 +359,7 @@ jobs: restore-keys: | ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-full- - - name: Install, build, and prepare bubblewrap + - name: Install and prepare bubblewrap run: | pnpm install --frozen-lockfile & install_pid=$! @@ -374,18 +375,13 @@ jobs: sandbox_pid=$! install_status=0 wait "$install_pid" || install_status=$? - build_status=0 - if (( install_status == 0 )); then - pnpm run build || build_status=$? - fi sandbox_status=0 wait "$sandbox_pid" || sandbox_status=$? if (( install_status != 0 )); then exit "$install_status"; fi - if (( build_status != 0 )); then exit "$build_status"; fi exit "$sandbox_status" - name: Run complete unsharded primary Node CI concurrently - run: pnpm run check:ci:prebuilt + run: pnpm run check:ci:large-runner node-compat: if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' @@ -628,13 +624,14 @@ jobs: windows-gates: if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' continue-on-error: true - runs-on: dsh-windows-2025-16core + runs-on: dsh-windows-2025-32core name: windows node 24 / observational env: + # Keep ESLint itself single-threaded: 16 ESLint workers took 174 seconds on + # this image. The outer scheduler still overlaps lint with the other gates. DSH_ESLINT_CACHE: '1' - DSH_ESLINT_CONCURRENCY: '16' - DSH_GATE_CONCURRENCY: '16' - DSH_PUBLINT_CONCURRENCY: '16' + DSH_GATE_CONCURRENCY: '32' + DSH_PUBLINT_CONCURRENCY: '32' steps: - uses: actions/checkout@v6 @@ -995,14 +992,9 @@ jobs: sandbox_pid=$! install_status=0 wait "$install_pid" || install_status=$? - build_status=0 - if (( install_status == 0 )); then - pnpm run build || build_status=$? - fi sandbox_status=0 wait "$sandbox_pid" || sandbox_status=$? if (( install_status != 0 )); then exit "$install_status"; fi - if (( build_status != 0 )); then exit "$build_status"; fi exit "$sandbox_status" - name: Install (immutable) @@ -1019,7 +1011,7 @@ jobs: DSH_GATE_CONCURRENCY: ${{ matrix.workers }} DSH_PUBLINT_CONCURRENCY: ${{ matrix.workers }} DSH_SNAPSHOT_MAX_CONCURRENCY: ${{ matrix.workers }} - run: pnpm run check:ci:prebuilt + run: pnpm run check:ci:large-runner - name: Run blocking Windows builds concurrently if: matrix.platform == 'windows' diff --git a/package.json b/package.json index a8ee301da3..c3aaa3c29d 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "test:snapshot:record": "DSH_SNAPSHOT=record vitest run --config vitest.snapshot.config.ts --update", "test:snapshot:refresh": "DSH_SNAPSHOT=refresh vitest run --config vitest.snapshot.config.ts", "check:ci": "tsx scripts/run-gates.ts ci-primary", - "check:ci:prebuilt": "tsx scripts/run-gates.ts ci-primary-prebuilt", + "check:ci:large-runner": "tsx scripts/run-gates.ts ci-primary-large-runner", "check:ci:static": "tsx scripts/run-gates.ts ci-static", "check:ci:lint": "tsx scripts/run-gates.ts ci-lint", "check:ci:coverage": "tsx scripts/run-gates.ts ci-coverage", diff --git a/scripts/run-gates.ts b/scripts/run-gates.ts index c9d058518b..80c9c9bc5c 100644 --- a/scripts/run-gates.ts +++ b/scripts/run-gates.ts @@ -15,7 +15,7 @@ import { selectStaticGates } from './static-shards.ts' type Mode = | 'ci-primary' - | 'ci-primary-prebuilt' + | 'ci-primary-large-runner' | 'ci-static' | 'ci-lint' | 'ci-coverage' @@ -88,7 +88,7 @@ if (results.some(result => result.status === 'failed' || result.status === 'skip function parseMode(raw: string | undefined): Mode { switch (raw) { case 'ci-primary': - case 'ci-primary-prebuilt': + case 'ci-primary-large-runner': case 'ci-static': case 'ci-lint': case 'ci-coverage': @@ -102,7 +102,7 @@ function parseMode(raw: string | undefined): Mode { return raw default: throw new Error( - `run-gates: expected mode ci-primary | ci-primary-prebuilt | ci-static | ci-lint | ci-coverage | ci-snapshot | ci-artifacts | ci-windows-blocking | ci-windows-observational | node-compat | pre-push | doc-sync, got ${JSON.stringify(raw)}.`, + `run-gates: expected mode ci-primary | ci-primary-large-runner | ci-static | ci-lint | ci-coverage | ci-snapshot | ci-artifacts | ci-windows-blocking | ci-windows-observational | node-compat | pre-push | doc-sync, got ${JSON.stringify(raw)}.`, ) } } @@ -168,8 +168,8 @@ function gatesForMode(selected: Mode): Gate[] { switch (selected) { case 'ci-primary': return ciPrimaryGates() - case 'ci-primary-prebuilt': - return ciPrimaryPrebuiltGates() + case 'ci-primary-large-runner': + return ciPrimaryLargeRunnerGates() case 'ci-static': return ciStaticGates() case 'ci-lint': { @@ -250,12 +250,16 @@ function ciPrimaryGates(): Gate[] { ] } -function ciPrimaryPrebuiltGates(): Gate[] { +function ciPrimaryLargeRunnerGates(): Gate[] { + // Typecheck does not consume build output, so a large runner can start both + // together while snapshot and artifact consumers still wait for the build. return ciPrimaryGates() - .filter(gate => gate.id !== 'build') - .map(gate => gate.needs?.includes('build') === true - ? { ...gate, needs: gate.needs.filter(id => id !== 'build') } - : gate) + .map((gate) => { + if (gate.id !== 'build') return gate + const eagerBuild = { ...gate } + delete eagerBuild.needs + return eagerBuild + }) } function ciStaticGates(): Gate[] { From cc22492ddd1c40973b903db950a5b897a9e9fb88 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 15:30:02 +0800 Subject: [PATCH 30/49] docs: record consolidated runner decision --- .../2026-07-06-parallel-github-ci-gates.md | 2 + ...ence-based-larger-hosted-runners.i18n.yaml | 4 +- ...22-evidence-based-larger-hosted-runners.md | 54 ++++++++++++++----- ...evidence-based-larger-hosted-runners.zh.md | 54 ++++++++++++++----- 4 files changed, 86 insertions(+), 28 deletions(-) diff --git a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md index 156e3691d1..c7904ae378 100644 --- a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md +++ b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md @@ -12,6 +12,8 @@ The artifact boundary remains load-bearing. `publint`, `verify-node-next-types`, ## Decision +The production topology below was superseded by [Evidence-based larger hosted runners](2026-07-22-evidence-based-larger-hosted-runners.md). Its shard selectors and former job layout remain available only through manual `suite=sharded-reference`; this note preserves why that earlier topology was implemented. + [CI](../../../../.github/workflows/ci.yml) treats one minute for non-Windows jobs and three minutes for Windows jobs as observed performance targets, not cancellation deadlines. Hosted-runner variance should leave complete timing evidence and useful failure logs instead of cancelling an otherwise-correct gate. The [serial cross-platform CI reference](2026-07-21-serial-cross-platform-ci-reference.md) independently runs the complete unsharded primary Node aggregate on Linux, macOS, and Windows so the optimized lane inventory is not its own completeness oracle. [scripts/run-gates.ts](../../../../scripts/run-gates.ts) remains the common bounded scheduler, but GitHub supplies explicit shard names for the expensive gate families. [scripts/static-shards.ts](../../../../scripts/static-shards.ts) partitions static gates into foundation, documentation-type, API-contract, catalog, prose, documentation-projection, and documentation-build ownership and rejects a missing or duplicate gate assignment; low-cost ownership groups may share one hosted lane to stay below the runner-provisioning concurrency wave. Linux lint uses disjoint A-C, D-M, N-S, and T-Z package-source and package-test lanes, while the longer Windows target uses one complete package-source lane and one complete package-test lane; both topologies include a repository complement that still starts from `.` so a new top-level lint target cannot disappear between shards and owns the single cross-file duplication run. [scripts/coverage-shards.ts](../../../../scripts/coverage-shards.ts) assigns every workspace package to exactly one source-coverage lane; its test expands the live package tree, so a new package makes CI red until it has an owner. Directory filters retain a trailing separator because Vitest positional filters match substrings and would otherwise admit prefix-named sibling packages. Each coverage lane includes only its owned source files, repeats the exhaustive companion topology test, and runs without a preceding build because the complete coverage suite passes from a tree with every generated `lib/` removed. diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml index 303a021a38..da19d5558b 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -2026-07-22-evidence-based-larger-hosted-runners.md: 992a622f0d7a2662cc65353d6c4cd1350e1f3f5f -2026-07-22-evidence-based-larger-hosted-runners.zh.md: 622911af8400fbe159038b69b440c0e97fb1f2cc +2026-07-22-evidence-based-larger-hosted-runners.md: 8d3852c56e2289bd222a260f2685f6a941a0cb19 +2026-07-22-evidence-based-larger-hosted-runners.zh.md: 4e7f846db1be2d68a080381943012583636ed2c4 diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md index 992a622f0d..8d3852c56e 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md @@ -6,39 +6,67 @@ English | [中文](2026-07-22-evidence-based-larger-hosted-runners.zh.md) ## Problem -The sharded primary CI fits its latency targets on standard GitHub-hosted runners, but the margin depends on cold setup and install variance. Larger runners may add useful headroom, yet their per-minute price rises much faster than these short lanes can use extra cores. Selecting a size from machine specifications or a synthetic benchmark would spend more without proving that repository CI becomes faster. +The shard-heavy CI topology met its latency targets by spreading primary Node work across 40 Linux jobs and Windows work across nine jobs. Most gates were shorter than checkout, runner setup, cache restore, and dependency installation, so repeated setup waves created both cost and latency variance. One hosted run finished its slowest Linux job in 49 seconds yet took 231 seconds for a Windows lint shard whose checkout, cache restore, and install alone consumed 158 seconds. + +Larger runners make it possible to pay setup once and parallelize inside the repository scheduler, but the useful size cannot be selected from core counts alone. Critical-lane benchmarks did not scale monotonically, and a whole-repository aggregate exposed different bottlenecks from isolated typecheck or site builds. ## Decision -The organization keeps twelve x64 larger-runner pools in the repo-restricted `dsh-larger-ci` group: Ubuntu 24.04 and Windows 2025 at 4, 8, 16, 32, 64, and 96 cores. Public IPs are disabled. Each pool has an autoscaling ceiling of 256, while the repository bounds actual demand through its workflow matrices; an idle ceiling does not allocate machines. +The organization keeps twelve x64 larger-runner pools in the repo-restricted `dsh-larger-ci` group: Ubuntu 24.04 and Windows 2025 at 4, 8, 16, 32, 64, and 96 cores. Public IPs are disabled. Each pool has an autoscaling ceiling of 256; the ceiling does not allocate idle machines or remove the need to bound workflow demand. -The `CI` workflow exposes `suite=larger-runner-benchmark` only through manual dispatch. Its six Linux legs run the critical typecheck lane, and its six Windows legs run the critical production-site lane. Every leg reports the observed CPU and memory, has a 15-minute timeout, and uses the same setup and caching policy as the production lane it represents. Push and pull-request events skip this benchmark. +Production CI uses three larger-runner jobs and keeps Node compatibility, Python, and the final aggregator on standard runners: -The [twelve-size benchmark](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29895295659) used a workflow-only commit on top of the standard-runner [baseline](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29850033610), so the code, lockfile, and critical commands were identical: +- `node 24 / complete` uses the 96-core Linux pool. One checkout, setup, cache restore, and install feeds the complete unsharded 40-gate primary inventory. `run-gates` starts up to 32 independent gates, ESLint uses 32 workers, snapshots use up to 32 subprocesses, and coverage uses 16 forks. Build starts beside typecheck; snapshot and artifact consumers still wait for emitted output. Coverage stays below 32 forks because that setting twice caused Node 24's CJS lexer to terminate a Vitest worker and invalidate coverage. +- `windows / blocking builds` uses the 16-core Windows pool. Build and the production VitePress site run concurrently after one setup. +- `windows node 24 / observational` uses the 32-core Windows pool. Its complete unsharded 37-gate static, lint, and artifact inventory runs with 32 outer scheduler slots and remains non-blocking. ESLint itself stays single-threaded because 16 ESLint worker threads increased full-lint time to 174.54 seconds; with outer concurrency and no ESLint workers, the same full lint took 31.67 seconds. + +The two Windows jobs deliberately use different pools. A first candidate put both on the 16-core pool and GitHub took 93 seconds to provision the second same-label runner despite the configured autoscaling ceiling. Using the 16-core and 32-core pools let every production job begin within two seconds in the final validation run. + +The workflow retains four manual diagnostics. `suite=larger-runner-benchmark` compares isolated critical lanes across every size, `suite=consolidated-runner-benchmark` compares whole aggregates, `suite=sharded-reference` preserves the former production shard topology, and `suite=serial-reference` remains the unsharded cross-platform completeness oracle. `suite=optimized-larger-runners` runs the exact production topology against a branch ref when a pull request cannot form a merge commit. + +The first [twelve-size critical-lane benchmark](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29895295659) used a workflow-only commit on top of the standard-runner [baseline](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29850033610), so the code, lockfile, and commands were identical: | Critical job | Standard | 4 cores | 8 cores | 16 cores | 32 cores | 64 cores | 96 cores | |---|---:|---:|---:|---:|---:|---:|---:| | Linux typecheck | 56 s | 38 s | 35 s | 40 s | 35 s | 44 s | 40 s | | Windows production site | 160 s | 117 s | 103 s | 113 s | 75 s | 105 s | 108 s | -The repository therefore uses the 4-core pools for the primary Node matrix and all Windows jobs. Linux 4-core finished within three seconds of the fastest measured size at the lowest larger-runner rate. Windows 4-core stayed below two minutes; the isolated 32-core result was faster, but adjacent larger sizes regressed and the production-site command itself varied only from 28 to 36 seconds, so setup and install noise—not scalable compute—created most of the spread. Node compatibility, Python, and the final aggregator remain on standard runners because their baseline jobs already finish well below one minute. +Those isolated results showed that setup dominated but did not identify the production size. A [whole-aggregate benchmark without native ESLint concurrency](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29897826082) found a 69-second single-threaded Linux lint gate. After enabling native Linux ESLint concurrency, the [second whole-aggregate benchmark](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29898331705) produced these active job times: -The workflow also exposes `suite=optimized-larger-runners` through manual dispatch. That path runs the production matrices against the branch ref itself, providing an exact-head timing check when a pull request cannot form a merge commit. +| Aggregate job | 4 cores | 8 cores | 16 cores | 32 cores | 64 cores | 96 cores | +|---|---:|---:|---:|---:|---:|---:| +| Linux complete primary | 147 s | 104 s | 95 s | failed at 57 s | 66 s | 60 s | +| Windows blocking builds | 137 s | 127 s | 113 s | 107 s | 105 s | 131 s | + +The Linux 32-core failure was the first CJS-lexer worker crash. The 96-core aggregate was the only successful all-size result at the one-minute boundary. Windows gained little above 16 cores, so the blocking job uses 16 cores; the observational job uses a separate 32-core pool to avoid same-label provisioning delay and to start all outer gates together. + +The exact production [validation run](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29899733584) passed every job at the tested branch head: + +| Production job | Active time | Repository work | Result | +|---|---:|---:|---:| +| Linux complete primary | 50 s | 40 gates in 23.23 s | passed | +| Slowest standard non-Windows job | 40 s | Node 26 compatibility | passed | +| Windows blocking builds | 91 s | 2 gates in 28.69 s | passed | +| Windows observational | 153 s | 37 gates in 31.83 s | passed | + +The Windows observational job spent 57 seconds restoring its pnpm cache, so its remaining margin measures hosted setup variance as well as repository work. The final run still stays below one minute for every non-Windows job and below three minutes for both Windows jobs. ## Alternatives considered -**Keep every job on standard runners.** This meets the threshold but gives the critical lanes no cold-run margin and leaves the larger-runner suggestion untested. +**Keep the former shard topology in production.** The shards can be fast when provisioned together, but 49 larger-runner jobs repeat setup and create more chances for a cold outlier. The 231-second Windows control demonstrated that a short lint shard does not protect the end-to-end job target. -**Select 8 or 32 cores from the fastest individual result.** The small differences were not monotonic, while billing grows sharply with size. Treating one noisy minimum as scaling evidence would make recurring CI substantially more expensive. +**Select a production size from the critical-lane benchmark.** Four cores looked cost-effective for isolated typecheck and site builds, but the full aggregate found repository-wide lint and dependent artifact work that those commands did not represent. -**Move every job to a larger runner.** Compatibility, Python, and aggregation were already short; paying the larger-runner premium there cannot improve the critical path enough to justify the dependency or cost. +**Prebuild before starting the Linux aggregate.** This moved build onto the setup path and produced a 66-second candidate. Starting build eagerly inside `run-gates` preserves artifact dependencies while overlapping it with unrelated checks; the final aggregate completed in 23.23 seconds. -**Use a synthetic CPU benchmark.** A microbenchmark would not include checkout, action startup, package installation, cache restore, or the repository command mix that dominates these jobs. +**Use native ESLint worker concurrency on Windows.** Sixteen workers made lint more than five times slower than the final single-threaded result. Outer gate parallelism uses the 32-core runner without multiplying ESLint's Windows worker startup and TypeScript project loading. + +**Move compatibility, Python, and aggregation to larger runners.** These standard-runner jobs all completed in 40 seconds or less. Paid capacity would not shorten the critical path. ## Consequences -The benchmark incurred $2.936 across dedicated larger-runner SKUs, as recorded by organization billing immediately after the run. The existing zero-dollar Actions budget did not block those jobs, so the repo-only runner group, manual benchmark trigger, matrix width, and timeout are the observed cost controls; the budget is not treated as an execution guard. +The final production validation consumed one billed 96-core Linux minute, two billed 16-core Windows minutes, and three billed 32-core Windows minutes. At the configured larger-runner rates, its larger-runner cost was $0.902. The all-size critical benchmark cost $2.936. GitHub rounds each larger-runner job up to a whole minute, so reducing paid job count from 49 to three matters as much as shortening repository work. -Production CI now depends on the organization-owned runner names in this note and in `.github/workflows/ci.yml`. Missing or renamed pools leave jobs queued instead of silently falling back to standard capacity. The 256 autoscaling ceiling permits future concurrency but does not override the bounded matrices in the current workflow. +The existing zero-dollar Actions budget did not block larger-runner jobs. The repo-only runner group, bounded workflow topology, manual benchmark triggers, and job timeouts are the observed cost controls; the budget is not treated as an execution guard. -The selected pools buy latency headroom at a recurring per-job premium. The manual benchmark retains all sizes so a future image, dependency graph, or workload change can be measured before changing the production labels again. +Production CI depends on the organization-owned runner names in this note and in `.github/workflows/ci.yml`. Missing or renamed pools leave jobs queued instead of falling back to standard capacity. Manual all-size and former-shard suites remain available so image, dependency, scheduler, or pricing changes can be remeasured before changing production labels. diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md index 622911af84..4e7f846db1 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md @@ -6,39 +6,67 @@ Status: implemented ## 问题 -分片后的主 CI 在标准 GitHub 托管运行器上能够达到延迟目标，但余量会受冷启动设置和安装耗时波动影响。大型运行器或许能提供更多有效余量，但随着核心数增加，其每分钟单价增长得远比这些短通道能够利用额外核心的速度快。仅根据机器规格或合成基准测试选择配置，只会增加支出，无法证明仓库 CI 确实变快。 +高度分片的 CI 拓扑通过把主 Node 工作分散到 40 个 Linux 作业、把 Windows 工作分散到 9 个作业来达到延迟目标。大多数门禁本身的耗时短于代码检出、运行器设置、缓存恢复和依赖安装这些准备阶段，因此反复执行多轮设置既增加成本，也带来延迟波动。一次托管运行中最慢的 Linux 作业用时 49 秒，而一个 Windows lint 分片却耗时 231 秒，其中仅代码检出、缓存恢复和安装就占了 158 秒。 + +大型运行器可以让 CI 只承担一次设置开销，再由仓库调度器在内部并行执行，但无法仅凭核心数选出有实际价值的规格。关键通道基准测试的性能提升不呈单调变化，完整仓库聚合流程暴露出的瓶颈也不同于单独运行类型检查或网站构建时的瓶颈。 ## 决策 -组织在仅限本仓库使用的 `dsh-larger-ci` 运行器组中保留 12 个 x64 大型运行器池：Ubuntu 24.04 和 Windows 2025 各设 4、8、16、32、64、96 核规格。公网 IP 已禁用。每个池的自动扩缩容上限为 256，仓库则通过工作流矩阵限制实际需求；空闲状态下的上限不会分配机器。 +组织在仅限本仓库使用的 `dsh-larger-ci` 运行器组中保留 12 个 x64 大型运行器池：Ubuntu 24.04 和 Windows 2025 各设 4、8、16、32、64、96 核规格。公网 IP 已禁用。每个池的自动扩缩容上限为 256；该上限既不会分配闲置机器，也不能免除限制工作流需求的必要性。 -`CI` 工作流仅在手动触发时提供 `suite=larger-runner-benchmark`。其中 6 个 Linux 矩阵分支运行关键类型检查通道，6 个 Windows 矩阵分支运行生产网站关键通道。每个矩阵分支都会报告实测 CPU 和内存，超时限制为 15 分钟，并采用与其所代表的生产通道相同的设置和缓存策略。push 和拉取请求事件会跳过这项基准测试。 +生产 CI 使用 3 个大型运行器作业，并让 Node 兼容性、Python 和最终聚合作业继续使用标准运行器： -这项[涵盖 12 种规格的基准测试](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29895295659)以标准运行器[基线](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29850033610)为基础，只叠加了一个仅修改工作流的提交，因此代码、锁文件和关键命令完全相同： +- `node 24 / complete` 使用 96 核 Linux 池。只需执行一次代码检出、设置、缓存恢复和安装，即可供完整且未分片的 40 项主门禁清单使用。`run-gates` 最多同时启动 32 项相互独立的门禁，ESLint 使用 32 个工作线程，快照最多使用 32 个子进程，覆盖率运行使用 16 个 fork。构建与类型检查同时启动；快照和产物消费方仍会等待生成的输出。覆盖率运行的 fork 数保持低于 32，因为将其设为 32 曾两次导致 Node 24 的 CJS 词法分析器终止一个 Vitest 工作进程，使覆盖率结果失效。 +- `windows / blocking builds` 使用 16 核 Windows 池。一次设置完成后，构建任务与生产版 VitePress 网站任务并发运行。 +- `windows node 24 / observational` 使用 32 核 Windows 池。其完整且未分片的 37 项静态、lint 和产物门禁清单使用 32 个外层调度器槽位运行，并保持非阻塞。ESLint 本身仍采用单线程，因为启用 16 个 ESLint 工作线程会把完整 lint 的耗时增加至 174.54 秒；在启用外层并发且不使用 ESLint 工作线程时，同一项完整 lint 只需 31.67 秒。 + +两个 Windows 作业有意使用不同的运行器池。首个候选方案让二者都使用 16 核池；尽管已配置自动扩缩容上限，GitHub 仍花费 93 秒才预配好第二台同标签运行器。分别使用 16 核池和 32 核池后，最终验证运行中的每个生产作业都在 2 秒内开始运行。 + +工作流保留 4 项手动诊断。`suite=larger-runner-benchmark` 比较所有规格下相互独立的关键通道，`suite=consolidated-runner-benchmark` 比较完整聚合流程，`suite=sharded-reference` 保留原生产分片拓扑，`suite=serial-reference` 则继续作为未分片的跨平台完整性判定基准。当拉取请求无法生成合并提交时，`suite=optimized-larger-runners` 会直接针对分支引用运行与生产环境完全相同的拓扑。 + +首次[涵盖 12 种规格的关键通道基准测试](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29895295659)以标准运行器[基线](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29850033610)为基础，只叠加了一个仅修改工作流的提交，因此代码、锁文件和命令完全相同： | 关键作业 | 标准 | 4 核 | 8 核 | 16 核 | 32 核 | 64 核 | 96 核 | |---|---:|---:|---:|---:|---:|---:|---:| | Linux 类型检查 | 56 秒 | 38 秒 | 35 秒 | 40 秒 | 35 秒 | 44 秒 | 40 秒 | | Windows 生产网站 | 160 秒 | 117 秒 | 103 秒 | 113 秒 | 75 秒 | 105 秒 | 108 秒 | -因此，仓库让主 Node 矩阵和所有 Windows 作业使用 4 核池。Linux 4 核规格比实测最快规格至多慢 3 秒，同时采用大型运行器中的最低费率。Windows 4 核规格保持在 2 分钟以内；单次测得的 32 核结果虽然更快，但相邻的大型运行器规格反而耗时回升，而生产网站命令本身的耗时仅在 28 至 36 秒之间波动，因此大部分差异来自设置和安装噪声，而非可随核心数扩展的计算负载。Node 兼容性、Python 和最终聚合作业仍使用标准运行器，因为它们的基线耗时本就远低于 1 分钟。 +这些单项结果表明设置开销占主导地位，却无法确定生产环境应选用的规格。一项[未启用 ESLint 原生并发的完整聚合基准测试](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29897826082)发现，Linux 单线程 lint 门禁耗时 69 秒。启用 Linux ESLint 原生并发后，[第二次完整聚合基准测试](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29898331705)得到了以下作业活动耗时： -工作流还在手动触发时提供 `suite=optimized-larger-runners`。这一路径直接以分支引用本身运行生产矩阵；当拉取请求无法生成合并提交时，它仍能针对该分支实际的头部提交提供精确计时检查。 +| 聚合作业 | 4 核 | 8 核 | 16 核 | 32 核 | 64 核 | 96 核 | +|---|---:|---:|---:|---:|---:|---:| +| Linux 完整主流程 | 147 秒 | 104 秒 | 95 秒 | 57 秒时失败 | 66 秒 | 60 秒 | +| Windows 阻塞性构建 | 137 秒 | 127 秒 | 113 秒 | 107 秒 | 105 秒 | 131 秒 | + +Linux 32 核作业的失败是首次发生的 CJS 词法分析器工作进程崩溃。在所有规格的结果中，96 核聚合作业是唯一成功达到 1 分钟边界的结果。Windows 超过 16 核后的收益很小，因此阻塞性作业使用 16 核；观测作业则使用单独的 32 核池，以避免同标签运行器的预配延迟，并让全部外层门禁同时启动。 + +生产环境的精确[验证运行](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29899733584)在受测分支头通过了所有作业： + +| 生产作业 | 活动耗时 | 仓库工作 | 结果 | +|---|---:|---:|---:| +| Linux 完整主流程 | 50 秒 | 40 项门禁耗时 23.23 秒 | 通过 | +| 最慢的标准非 Windows 作业 | 40 秒 | Node 26 兼容性 | 通过 | +| Windows 阻塞性构建 | 91 秒 | 2 项门禁耗时 28.69 秒 | 通过 | +| Windows 观测作业 | 153 秒 | 37 项门禁耗时 31.83 秒 | 通过 | + +Windows 观测作业花费 57 秒恢复 pnpm 缓存，因此其剩余余量既反映托管环境的设置波动，也反映仓库工作耗时。最终运行中每个非 Windows 作业仍低于 1 分钟，两个 Windows 作业也都低于 3 分钟。 ## 曾考虑的替代方案 -**所有作业均保留在标准运行器上。** 此方案能达到时长目标，但关键通道面对冷启动时没有余量，也无法验证大型运行器方案。 +**在生产环境中保留原分片拓扑。** 各分片在一同完成预配时可以很快，但 49 个大型运行器作业会重复设置，也增加了出现冷启动异常值的机会。耗时 231 秒的 Windows 对照结果表明，短小的 lint 分片并不能保障端到端作业达到时长目标。 -**根据单次最快结果选择 8 核或 32 核。** 各规格间的微小差异并不呈单调变化，计费却随规格急剧增长。把一次带有噪声的最低值当作扩展性证据，会大幅提高 CI 的经常性成本。 +**根据关键通道基准测试选择生产规格。** 对单独的类型检查和网站构建而言，4 核看起来具备成本效益，但完整聚合流程发现了这些命令未覆盖的全仓库 lint 和存在依赖关系的产物工作。 -**将所有作业迁移到大型运行器。** 兼容性、Python 和聚合作业原本就很短；为这些作业支付大型运行器溢价，无法充分缩短关键路径，因而不足以证明这种依赖或成本合理。 +**在启动 Linux 聚合流程前预先构建。** 此方案让构建成为设置路径的一部分，并产生了一个耗时 66 秒的候选结果。在 `run-gates` 内尽早启动构建，既能保留产物依赖关系，又能让构建与无关检查重叠执行；最终聚合流程耗时 23.23 秒。 -**使用合成 CPU 基准测试。** 微基准测试不会包含代码检出、操作启动、包安装、缓存恢复或主导这些作业耗时的仓库命令组合。 +**在 Windows 上使用 ESLint 原生工作线程并发。** 16 个工作线程让 lint 比最终的单线程结果慢 5 倍以上。外层门禁并发能够利用 32 核运行器，同时不会成倍增加 ESLint 在 Windows 上启动工作线程和加载 TypeScript 项目的开销。 + +**将兼容性、Python 和聚合作业迁移到大型运行器。** 这些标准运行器作业都在 40 秒以内完成。付费容量不会缩短关键路径。 ## 后果 -这次基准测试使用专用大型运行器 SKU，共产生 $2.936 费用，组织计费系统在运行结束后立即记录了这笔费用。现有的零美元 Actions 预算并未阻止这些作业，因此经实测有效的成本控制机制是仅限本仓库的运行器组、手动基准测试触发、矩阵宽度和超时限制；该预算不被视为执行防护措施。 +最终生产验证产生的计费时长为：96 核 Linux 1 分钟、16 核 Windows 2 分钟和 32 核 Windows 3 分钟。按已配置的大型运行器费率计算，其大型运行器成本为 $0.902。全规格关键通道基准测试的成本为 $2.936。GitHub 会把每个大型运行器作业向上取整到整分钟计费，因此把付费作业数从 49 个减少到 3 个，与缩短仓库工作耗时同样重要。 -生产 CI 目前依赖本 Agent Note 和 `.github/workflows/ci.yml` 中由组织持有的运行器名称。池缺失或改名会让作业一直排队，不会静默回退到标准容量。256 的自动扩缩容上限允许未来提高并发量，但不会突破当前工作流对矩阵规模的限制。 +现有的零美元 Actions 预算并未阻止大型运行器作业。仅限本仓库的运行器组、有界的工作流拓扑、手动基准测试触发和作业超时限制才是经实测有效的成本控制机制；该预算不被视为执行防护措施。 -选定的运行器池以每次作业都需支付的溢价换取延迟余量。手动基准测试保留了所有规格，以便未来映像、依赖图或工作负载发生变化时，先完成测量，再次调整生产标签。 +生产 CI 依赖本 Agent Note 和 `.github/workflows/ci.yml` 中由组织持有的运行器名称。池缺失或改名会让作业一直排队，不会回退到标准容量。手动全规格套件和原分片套件均予以保留，以便在映像、依赖、调度器或定价发生变化后重新测量，再调整生产标签。 From 0f57eafc364ca26b533d38279f2fa4c754ea9b27 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 15:44:11 +0800 Subject: [PATCH 31/49] ci: collapse cold setup waves --- .github/workflows/ci.yml | 93 +++++++++++++++++++--------------------- package.json | 1 + scripts/run-gates.ts | 28 ++++++++++-- 3 files changed, 71 insertions(+), 51 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 121d82d701..45adb82b57 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -319,7 +319,7 @@ jobs: run: ${{ matrix.command }} # One large runner pays hosted setup once, then the repository scheduler - # overlaps the complete unsharded primary gate inventory inside that box. + # overlaps the unsharded primary inventory except the production site build. node-24: if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' runs-on: dsh-ubuntu-24-04-96core @@ -383,6 +383,38 @@ jobs: - name: Run complete unsharded primary Node CI concurrently run: pnpm run check:ci:large-runner + # Keep only the longest independent Linux gate on a second coarse-grained + # runner so cold install variance does not push the primary box over a minute. + node-24-site: + if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' + runs-on: dsh-ubuntu-24-04-16core + name: node 24 / production site + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-node@v6 + with: + node-version: ${{ env.PRIMARY_NODE_VERSION }} + + - name: Enable corepack and resolve pnpm store path + id: pnpm-store + run: | + corepack enable + echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" + + - uses: actions/cache@v4 + with: + path: ${{ steps.pnpm-store.outputs.path }} + key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm- + + - name: Install (immutable) + run: pnpm install --frozen-lockfile + + - name: Build documentation site (production SPA) + run: pnpm run docs:build + node-compat: if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' runs-on: ubuntu-latest @@ -583,49 +615,13 @@ jobs: shell: pwsh run: ${{ matrix.command }} - # The two blocking Windows build surfaces share one setup and run concurrently. - windows-build: + # One Windows box shares setup across the required build/site checks and the + # complete observational portability inventory. run-gates reports failures + # from observational gates without allowing them to fail the required job. + windows: if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' - runs-on: dsh-windows-2025-16core - name: windows / blocking builds - steps: - - uses: actions/checkout@v6 - - - uses: actions/setup-node@v6 - with: - node-version: ${{ env.PRIMARY_NODE_VERSION }} - - - name: Enable corepack and resolve pnpm store path - id: pnpm-store - shell: pwsh - run: | - corepack enable - "path=$(pnpm store path --silent)" >> $env:GITHUB_OUTPUT - - - uses: actions/cache@v4 - with: - path: ${{ steps.pnpm-store.outputs.path }} - key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm- - - - name: Install (immutable) - shell: pwsh - run: pnpm install --frozen-lockfile - - - name: Run blocking Windows builds concurrently - shell: pwsh - env: - DSH_GATE_CONCURRENCY: '2' - run: pnpm run check:ci:windows-blocking - - # Wider Windows portability gates remain observational, but one larger box - # now runs their complete unsharded inventory with in-runner concurrency. - windows-gates: - if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' - continue-on-error: true runs-on: dsh-windows-2025-32core - name: windows node 24 / observational + name: windows node 24 / complete env: # Keep ESLint itself single-threaded: 16 ESLint workers took 174 seconds on # this image. The outer scheduler still overlaps lint with the other gates. @@ -670,9 +666,9 @@ jobs: shell: pwsh run: pnpm install --frozen-lockfile - - name: Run observational Windows gates concurrently + - name: Run blocking and observational Windows gates concurrently shell: pwsh - run: pnpm run check:ci:windows-observational + run: pnpm run check:ci:windows-complete # Manual reference runs deliberately avoid the optimized jobs above. # Each host executes the complete, unsharded primary Node aggregate with one @@ -1011,7 +1007,7 @@ jobs: DSH_GATE_CONCURRENCY: ${{ matrix.workers }} DSH_PUBLINT_CONCURRENCY: ${{ matrix.workers }} DSH_SNAPSHOT_MAX_CONCURRENCY: ${{ matrix.workers }} - run: pnpm run check:ci:large-runner + run: pnpm run check:ci - name: Run blocking Windows builds concurrently if: matrix.platform == 'windows' @@ -1023,8 +1019,9 @@ jobs: # Single stable required check for branch protection: require "all checks # passed" instead of enumerating matrix legs whose names change as lanes and # node versions evolve. Every blocking job in THIS workflow must be listed in - # `needs`; explicitly observational jobs such as windows-gates stay out - # (`needs` cannot reach across workflow files; e2e.yml stays its own check). + # `needs`; observational Windows gates share the required Windows job but are + # marked non-blocking inside run-gates. (`needs` cannot reach across workflow + # files; e2e.yml stays its own check.) # `if: always()` is load-bearing: without it a failed dependency # would SKIP this job, and GitHub counts a skipped required check as passing # — so this job always runs and fails on any non-success result, including @@ -1032,7 +1029,7 @@ jobs: all-checks-passed: name: all checks passed runs-on: ubuntu-latest - needs: [node-24, node-compat, python-sdk, windows-build] + needs: [node-24, node-24-site, node-compat, python-sdk, windows] if: always() && (github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners') steps: - name: Fail if any needed job did not succeed diff --git a/package.json b/package.json index c3aaa3c29d..c9e5e01a71 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "check:ci:snapshot": "tsx scripts/run-gates.ts ci-snapshot", "check:ci:artifacts": "tsx scripts/run-gates.ts ci-artifacts", "check:ci:windows-blocking": "tsx scripts/run-gates.ts ci-windows-blocking", + "check:ci:windows-complete": "tsx scripts/run-gates.ts ci-windows-complete", "check:ci:windows-observational": "tsx scripts/run-gates.ts ci-windows-observational", "check:node-compat": "tsx scripts/run-gates.ts node-compat", "check:pre-push": "tsx scripts/run-gates.ts pre-push", diff --git a/scripts/run-gates.ts b/scripts/run-gates.ts index 80c9c9bc5c..11155551db 100644 --- a/scripts/run-gates.ts +++ b/scripts/run-gates.ts @@ -22,6 +22,7 @@ type Mode = | 'ci-snapshot' | 'ci-artifacts' | 'ci-windows-blocking' + | 'ci-windows-complete' | 'ci-windows-observational' | 'node-compat' | 'pre-push' @@ -38,6 +39,7 @@ interface Gate { env?: Record input?: string verify?: (result: GateResult) => Promise + allowFailure?: boolean } interface GateResult { @@ -83,7 +85,9 @@ console.log(`run-gates: ${mode} running ${gates.length} gate(s) with ${maxConcur const results = await runGates(gates, maxConcurrency) printSummary(results, performance.now() - startedAt) -if (results.some(result => result.status === 'failed' || result.status === 'skipped')) process.exit(1) +if (results.some(result => result.gate.allowFailure !== true && (result.status === 'failed' || result.status === 'skipped'))) { + process.exit(1) +} function parseMode(raw: string | undefined): Mode { switch (raw) { @@ -95,6 +99,7 @@ function parseMode(raw: string | undefined): Mode { case 'ci-snapshot': case 'ci-artifacts': case 'ci-windows-blocking': + case 'ci-windows-complete': case 'ci-windows-observational': case 'node-compat': case 'pre-push': @@ -102,7 +107,7 @@ function parseMode(raw: string | undefined): Mode { return raw default: throw new Error( - `run-gates: expected mode ci-primary | ci-primary-large-runner | ci-static | ci-lint | ci-coverage | ci-snapshot | ci-artifacts | ci-windows-blocking | ci-windows-observational | node-compat | pre-push | doc-sync, got ${JSON.stringify(raw)}.`, + `run-gates: expected mode ci-primary | ci-primary-large-runner | ci-static | ci-lint | ci-coverage | ci-snapshot | ci-artifacts | ci-windows-blocking | ci-windows-complete | ci-windows-observational | node-compat | pre-push | doc-sync, got ${JSON.stringify(raw)}.`, ) } } @@ -189,6 +194,8 @@ function gatesForMode(selected: Mode): Gate[] { return ciArtifactGates() case 'ci-windows-blocking': return ciWindowsBlockingGates() + case 'ci-windows-complete': + return ciWindowsCompleteGates() case 'ci-windows-observational': return ciWindowsObservationalGates() case 'node-compat': @@ -254,6 +261,7 @@ function ciPrimaryLargeRunnerGates(): Gate[] { // Typecheck does not consume build output, so a large runner can start both // together while snapshot and artifact consumers still wait for the build. return ciPrimaryGates() + .filter(gate => gate.id !== 'docs-site-build') .map((gate) => { if (gate.id !== 'build') return gate const eagerBuild = { ...gate } @@ -306,6 +314,19 @@ function ciWindowsBlockingGates(): Gate[] { ] } +function ciWindowsCompleteGates(): Gate[] { + const observational = ciWindowsObservationalGates() + // The required production site replaces the observational MPA build; both + // VitePress modes write the same output directory and cannot overlap. + .filter(gate => gate.id !== 'build' && gate.id !== 'docs-site-build') + .map(gate => ({ ...gate, allowFailure: true })) + return [ + pnpmScript('build', 'build'), + pnpmScript('windows-site', 'docs:build', { label: 'production site' }), + ...observational, + ] +} + function ciWindowsObservationalGates(): Gate[] { return [ ...ciStaticGates(), @@ -638,7 +659,8 @@ function printSummary(results: GateResult[], durationMs: number): void { for (const result of unsuccessful) { const duration = (result.durationMs / 1000).toFixed(2) const reason = result.error ?? (result.exitCode === null ? 'no exit code' : `exit ${result.exitCode}`) - console.error(` - ${result.status.toUpperCase()} ${result.gate.label} (${duration}s, ${reason})`) + const disposition = result.gate.allowFailure === true ? 'NON-BLOCKING ' : '' + console.error(` - ${disposition}${result.status.toUpperCase()} ${result.gate.label} (${duration}s, ${reason})`) console.error(` ${result.gate.displayCommand}`) } } From cd19f87c3f97ea4546eca003c8a450e89f75edaf Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 15:57:32 +0800 Subject: [PATCH 32/49] ci: name split Linux lane precisely --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 45adb82b57..09d61f845d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -323,7 +323,7 @@ jobs: node-24: if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' runs-on: dsh-ubuntu-24-04-96core - name: node 24 / complete + name: node 24 / core env: # Thirty-two Vitest forks intermittently crash Node's CJS lexer on this image. DSH_COVERAGE_MAX_WORKERS: '16' @@ -380,7 +380,7 @@ jobs: if (( install_status != 0 )); then exit "$install_status"; fi exit "$sandbox_status" - - name: Run complete unsharded primary Node CI concurrently + - name: Run unsharded primary Node core CI concurrently run: pnpm run check:ci:large-runner # Keep only the longest independent Linux gate on a second coarse-grained From d7945da13e2de2e527ceb06b4963bbac184424df Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 15:57:39 +0800 Subject: [PATCH 33/49] docs: record reduced-fanout validation --- ...ence-based-larger-hosted-runners.i18n.yaml | 4 +- ...22-evidence-based-larger-hosted-runners.md | 38 ++++++++++--------- ...evidence-based-larger-hosted-runners.zh.md | 38 ++++++++++--------- 3 files changed, 44 insertions(+), 36 deletions(-) diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml index da19d5558b..d5b558837f 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -2026-07-22-evidence-based-larger-hosted-runners.md: 8d3852c56e2289bd222a260f2685f6a941a0cb19 -2026-07-22-evidence-based-larger-hosted-runners.zh.md: 4e7f846db1be2d68a080381943012583636ed2c4 +2026-07-22-evidence-based-larger-hosted-runners.md: 7d38552d23f7a3fb8abca45cbb4cf4e9c82fdd78 +2026-07-22-evidence-based-larger-hosted-runners.zh.md: f16db8eeb9000fe9a7671345efa9b1fd7fc744e1 diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md index 8d3852c56e..7d38552d23 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md @@ -14,13 +14,13 @@ Larger runners make it possible to pay setup once and parallelize inside the rep The organization keeps twelve x64 larger-runner pools in the repo-restricted `dsh-larger-ci` group: Ubuntu 24.04 and Windows 2025 at 4, 8, 16, 32, 64, and 96 cores. Public IPs are disabled. Each pool has an autoscaling ceiling of 256; the ceiling does not allocate idle machines or remove the need to bound workflow demand. -Production CI uses three larger-runner jobs and keeps Node compatibility, Python, and the final aggregator on standard runners: +Production CI uses three coarse larger-runner jobs and keeps Node compatibility, Python, and the final aggregator on standard runners: -- `node 24 / complete` uses the 96-core Linux pool. One checkout, setup, cache restore, and install feeds the complete unsharded 40-gate primary inventory. `run-gates` starts up to 32 independent gates, ESLint uses 32 workers, snapshots use up to 32 subprocesses, and coverage uses 16 forks. Build starts beside typecheck; snapshot and artifact consumers still wait for emitted output. Coverage stays below 32 forks because that setting twice caused Node 24's CJS lexer to terminate a Vitest worker and invalidate coverage. -- `windows / blocking builds` uses the 16-core Windows pool. Build and the production VitePress site run concurrently after one setup. -- `windows node 24 / observational` uses the 32-core Windows pool. Its complete unsharded 37-gate static, lint, and artifact inventory runs with 32 outer scheduler slots and remains non-blocking. ESLint itself stays single-threaded because 16 ESLint worker threads increased full-lint time to 174.54 seconds; with outer concurrency and no ESLint workers, the same full lint took 31.67 seconds. +- `node 24 / core` uses the 96-core Linux pool. One checkout, setup, cache restore, and install feeds the unsharded 39-gate primary inventory other than the production site build. `run-gates` starts up to 32 independent gates, ESLint uses 32 workers, snapshots use up to 32 subprocesses, and coverage uses 16 forks. Build starts beside typecheck; snapshot and artifact consumers still wait for emitted output. Coverage stays below 32 forks because that setting twice caused Node 24's CJS lexer to terminate a Vitest worker and invalidate coverage. +- `node 24 / production site` uses the 16-core Linux pool for the longest independent primary gate. This is one coarse split, not a shard matrix: the job performs one setup and one production VitePress build. +- `windows node 24 / complete` uses the 32-core Windows pool. One setup feeds the required package build, the required production site build, and the complete observational portability inventory. The outer scheduler has 32 slots. Required failures fail the job; observational failures are printed as non-blocking and preserve their former advisory status. ESLint itself stays single-threaded because 16 ESLint worker threads increased full-lint time to 174.54 seconds; outer gate concurrency uses the runner without multiplying Windows worker startup and TypeScript project loading. -The two Windows jobs deliberately use different pools. A first candidate put both on the 16-core pool and GitHub took 93 seconds to provision the second same-label runner despite the configured autoscaling ceiling. Using the 16-core and 32-core pools let every production job begin within two seconds in the final validation run. +The final shape followed two cold-path observations. A first candidate used two 16-core Windows jobs, and GitHub took 93 seconds to provision the second same-label runner despite the configured autoscaling ceiling. A later [documentation-head validation](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29900502413) took 64 seconds on the combined Linux job because its production site gate lasted 29.05 seconds, while a separate Windows blocking job took 266 seconds after spending 138 seconds restoring a 153 MB pnpm cache. Moving that independent Linux gate to one coarse 16-core lane and combining all Windows work on one 32-core box removed both duplicate setup waves. The workflow retains four manual diagnostics. `suite=larger-runner-benchmark` compares isolated critical lanes across every size, `suite=consolidated-runner-benchmark` compares whole aggregates, `suite=sharded-reference` preserves the former production shard topology, and `suite=serial-reference` remains the unsharded cross-platform completeness oracle. `suite=optimized-larger-runners` runs the exact production topology against a branch ref when a pull request cannot form a merge commit. @@ -33,23 +33,23 @@ The first [twelve-size critical-lane benchmark](https://github.com/deepseek-harn Those isolated results showed that setup dominated but did not identify the production size. A [whole-aggregate benchmark without native ESLint concurrency](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29897826082) found a 69-second single-threaded Linux lint gate. After enabling native Linux ESLint concurrency, the [second whole-aggregate benchmark](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29898331705) produced these active job times: -| Aggregate job | 4 cores | 8 cores | 16 cores | 32 cores | 64 cores | 96 cores | +| Job | 4 cores | 8 cores | 16 cores | 32 cores | 64 cores | 96 cores | |---|---:|---:|---:|---:|---:|---:| | Linux complete primary | 147 s | 104 s | 95 s | failed at 57 s | 66 s | 60 s | | Windows blocking builds | 137 s | 127 s | 113 s | 107 s | 105 s | 131 s | -The Linux 32-core failure was the first CJS-lexer worker crash. The 96-core aggregate was the only successful all-size result at the one-minute boundary. Windows gained little above 16 cores, so the blocking job uses 16 cores; the observational job uses a separate 32-core pool to avoid same-label provisioning delay and to start all outer gates together. +The Linux 32-core failure was the first CJS-lexer worker crash. The 96-core aggregate was the only successful all-size result at the one-minute boundary. Although Windows repository work gained little above 16 cores, the 32-core pool can start the complete outer inventory together and, more importantly, removes an entire paid setup from production. -The exact production [validation run](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29899733584) passed every job at the tested branch head: +The exact [reduced-fanout validation run](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29901539360) passed every job at the tested branch head: | Production job | Active time | Repository work | Result | |---|---:|---:|---:| -| Linux complete primary | 50 s | 40 gates in 23.23 s | passed | -| Slowest standard non-Windows job | 40 s | Node 26 compatibility | passed | -| Windows blocking builds | 91 s | 2 gates in 28.69 s | passed | -| Windows observational | 153 s | 37 gates in 31.83 s | passed | +| Linux core | 52 s | 39 gates in 25.84 s | passed | +| Linux production site | 44 s | VitePress in 22.42 s | passed | +| Slowest standard non-Windows job | 43 s | Node 26 compatibility | passed | +| Windows complete | 96 s | 37 gates in 31.13 s | passed | -The Windows observational job spent 57 seconds restoring its pnpm cache, so its remaining margin measures hosted setup variance as well as repository work. The final run still stays below one minute for every non-Windows job and below three minutes for both Windows jobs. +All three paid jobs began together. The Windows job spent 21 seconds restoring its pnpm cache and 25 seconds installing dependencies, so its margin measures hosted setup variance as well as repository work. Every non-Windows job stays below one minute and the sole Windows job stays below three minutes. ## Alternatives considered @@ -57,16 +57,20 @@ The Windows observational job spent 57 seconds restoring its pnpm cache, so its **Select a production size from the critical-lane benchmark.** Four cores looked cost-effective for isolated typecheck and site builds, but the full aggregate found repository-wide lint and dependent artifact work that those commands did not represent. -**Prebuild before starting the Linux aggregate.** This moved build onto the setup path and produced a 66-second candidate. Starting build eagerly inside `run-gates` preserves artifact dependencies while overlapping it with unrelated checks; the final aggregate completed in 23.23 seconds. +**Run the production site inside the Linux core aggregate.** This reached 50 seconds with warm hosted setup, then crossed the threshold at 64 seconds when the site gate took 29.05 seconds. One coarse independent site job protects the target without returning to gate-level sharding. + +**Keep required and observational Windows checks in separate jobs.** The split preserved status semantics at the workflow level but paid setup twice, and a cold cache pushed the required job to 266 seconds. `run-gates` now preserves those semantics inside one process: build and production site are required, while the remaining inventory is explicitly non-blocking. + +**Prebuild before starting the Linux aggregate.** This moved build onto the setup path and produced a 66-second candidate. Starting build eagerly inside `run-gates` preserves artifact dependencies while overlapping it with unrelated checks. **Use native ESLint worker concurrency on Windows.** Sixteen workers made lint more than five times slower than the final single-threaded result. Outer gate parallelism uses the 32-core runner without multiplying ESLint's Windows worker startup and TypeScript project loading. -**Move compatibility, Python, and aggregation to larger runners.** These standard-runner jobs all completed in 40 seconds or less. Paid capacity would not shorten the critical path. +**Move compatibility, Python, and aggregation to larger runners.** These standard-runner jobs completed in 43 seconds or less. Paid capacity would not shorten the critical path. ## Consequences -The final production validation consumed one billed 96-core Linux minute, two billed 16-core Windows minutes, and three billed 32-core Windows minutes. At the configured larger-runner rates, its larger-runner cost was $0.902. The all-size critical benchmark cost $2.936. GitHub rounds each larger-runner job up to a whole minute, so reducing paid job count from 49 to three matters as much as shortening repository work. +The reduced-fanout validation consumed one billed 96-core Linux minute, one billed 16-core Linux minute, and two billed 32-core Windows minutes. At the configured larger-runner rates, its larger-runner cost was $0.618. The all-size critical benchmark cost $2.936. GitHub rounds each larger-runner job up to a whole minute, so reducing paid job count from 49 to three matters as much as shortening repository work. The existing zero-dollar Actions budget did not block larger-runner jobs. The repo-only runner group, bounded workflow topology, manual benchmark triggers, and job timeouts are the observed cost controls; the budget is not treated as an execution guard. -Production CI depends on the organization-owned runner names in this note and in `.github/workflows/ci.yml`. Missing or renamed pools leave jobs queued instead of falling back to standard capacity. Manual all-size and former-shard suites remain available so image, dependency, scheduler, or pricing changes can be remeasured before changing production labels. +Production CI depends on the organization-owned runner names in this note and in `.github/workflows/ci.yml`. Missing or renamed pools leave jobs queued instead of falling back to standard capacity. Manual all-size, consolidated, former-shard, and serial suites remain available so image, dependency, scheduler, or pricing changes can be remeasured before changing production labels. diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md index 4e7f846db1..f16db8eeb9 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md @@ -14,13 +14,13 @@ Status: implemented 组织在仅限本仓库使用的 `dsh-larger-ci` 运行器组中保留 12 个 x64 大型运行器池：Ubuntu 24.04 和 Windows 2025 各设 4、8、16、32、64、96 核规格。公网 IP 已禁用。每个池的自动扩缩容上限为 256；该上限既不会分配闲置机器，也不能免除限制工作流需求的必要性。 -生产 CI 使用 3 个大型运行器作业，并让 Node 兼容性、Python 和最终聚合作业继续使用标准运行器： +生产 CI 使用 3 个粗粒度大型运行器作业，并让 Node 兼容性、Python 和最终聚合作业继续使用标准运行器： -- `node 24 / complete` 使用 96 核 Linux 池。只需执行一次代码检出、设置、缓存恢复和安装，即可供完整且未分片的 40 项主门禁清单使用。`run-gates` 最多同时启动 32 项相互独立的门禁，ESLint 使用 32 个工作线程，快照最多使用 32 个子进程，覆盖率运行使用 16 个 fork。构建与类型检查同时启动；快照和产物消费方仍会等待生成的输出。覆盖率运行的 fork 数保持低于 32，因为将其设为 32 曾两次导致 Node 24 的 CJS 词法分析器终止一个 Vitest 工作进程，使覆盖率结果失效。 -- `windows / blocking builds` 使用 16 核 Windows 池。一次设置完成后，构建任务与生产版 VitePress 网站任务并发运行。 -- `windows node 24 / observational` 使用 32 核 Windows 池。其完整且未分片的 37 项静态、lint 和产物门禁清单使用 32 个外层调度器槽位运行，并保持非阻塞。ESLint 本身仍采用单线程，因为启用 16 个 ESLint 工作线程会把完整 lint 的耗时增加至 174.54 秒；在启用外层并发且不使用 ESLint 工作线程时，同一项完整 lint 只需 31.67 秒。 +- `node 24 / core` 使用 96 核 Linux 池。只需执行一次代码检出、设置、缓存恢复和安装，即可供生产网站构建以外、未分片的 39 项主门禁清单使用。`run-gates` 最多同时启动 32 项相互独立的门禁，ESLint 使用 32 个工作线程，快照最多使用 32 个子进程，覆盖率运行使用 16 个 fork。构建与类型检查同时启动；快照和产物消费方仍会等待生成的输出。覆盖率运行的 fork 数保持低于 32，因为将其设为 32 曾两次导致 Node 24 的 CJS 词法分析器终止一个 Vitest 工作进程，使覆盖率结果失效。 +- `node 24 / production site` 使用 16 核 Linux 池运行耗时最长的独立主门禁。这是一项粗粒度拆分，而非分片矩阵：该作业执行一次设置和一次生产版 VitePress 构建。 +- `windows node 24 / complete` 使用 32 核 Windows 池。一次设置供必需的包构建、必需的生产网站构建以及完整的观测性可移植性清单共用。外层调度器设有 32 个槽位。任何必需项失败都会使作业失败；观测项失败会明确打印为非阻塞，并保留其原有的建议性状态。ESLint 本身仍采用单线程，因为启用 16 个 ESLint 工作线程会把完整 lint 的耗时增加至 174.54 秒；外层门禁并发能够利用该运行器，同时不会成倍增加 Windows 工作线程的启动和 TypeScript 项目加载开销。 -两个 Windows 作业有意使用不同的运行器池。首个候选方案让二者都使用 16 核池；尽管已配置自动扩缩容上限，GitHub 仍花费 93 秒才预配好第二台同标签运行器。分别使用 16 核池和 32 核池后，最终验证运行中的每个生产作业都在 2 秒内开始运行。 +最终结构由两项冷路径观测结果确定。首个候选方案使用两个 16 核 Windows 作业；尽管已配置自动扩缩容上限，GitHub 仍花费 93 秒才预配好第二台同标签运行器。随后一次[文档分支头验证](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29900502413)中，合并后的 Linux 作业耗时 64 秒，因为其中的生产网站门禁耗时 29.05 秒；另一个独立的 Windows 阻塞性作业则耗时 266 秒，其中恢复 153 MB pnpm 缓存就用了 138 秒。把这项独立的 Linux 门禁移入一个粗粒度 16 核通道，并把所有 Windows 工作合并到一台 32 核机器上，消除了这两轮重复设置。 工作流保留 4 项手动诊断。`suite=larger-runner-benchmark` 比较所有规格下相互独立的关键通道，`suite=consolidated-runner-benchmark` 比较完整聚合流程，`suite=sharded-reference` 保留原生产分片拓扑，`suite=serial-reference` 则继续作为未分片的跨平台完整性判定基准。当拉取请求无法生成合并提交时，`suite=optimized-larger-runners` 会直接针对分支引用运行与生产环境完全相同的拓扑。 @@ -33,23 +33,23 @@ Status: implemented 这些单项结果表明设置开销占主导地位，却无法确定生产环境应选用的规格。一项[未启用 ESLint 原生并发的完整聚合基准测试](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29897826082)发现，Linux 单线程 lint 门禁耗时 69 秒。启用 Linux ESLint 原生并发后，[第二次完整聚合基准测试](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29898331705)得到了以下作业活动耗时： -| 聚合作业 | 4 核 | 8 核 | 16 核 | 32 核 | 64 核 | 96 核 | +| 作业 | 4 核 | 8 核 | 16 核 | 32 核 | 64 核 | 96 核 | |---|---:|---:|---:|---:|---:|---:| | Linux 完整主流程 | 147 秒 | 104 秒 | 95 秒 | 57 秒时失败 | 66 秒 | 60 秒 | | Windows 阻塞性构建 | 137 秒 | 127 秒 | 113 秒 | 107 秒 | 105 秒 | 131 秒 | -Linux 32 核作业的失败是首次发生的 CJS 词法分析器工作进程崩溃。在所有规格的结果中，96 核聚合作业是唯一成功达到 1 分钟边界的结果。Windows 超过 16 核后的收益很小，因此阻塞性作业使用 16 核；观测作业则使用单独的 32 核池，以避免同标签运行器的预配延迟，并让全部外层门禁同时启动。 +Linux 32 核作业的失败是首次发生的 CJS 词法分析器工作进程崩溃。在所有规格的结果中，96 核聚合作业是唯一成功达到 1 分钟边界的结果。尽管 Windows 仓库工作在超过 16 核后收益很小，32 核池可以让完整的外层清单同时启动；更重要的是，它能从生产环境中消除整整一次付费设置。 -生产环境的精确[验证运行](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29899733584)在受测分支头通过了所有作业： +精确的[低扇出验证运行](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29901539360)在受测分支头通过了所有作业： | 生产作业 | 活动耗时 | 仓库工作 | 结果 | |---|---:|---:|---:| -| Linux 完整主流程 | 50 秒 | 40 项门禁耗时 23.23 秒 | 通过 | -| 最慢的标准非 Windows 作业 | 40 秒 | Node 26 兼容性 | 通过 | -| Windows 阻塞性构建 | 91 秒 | 2 项门禁耗时 28.69 秒 | 通过 | -| Windows 观测作业 | 153 秒 | 37 项门禁耗时 31.83 秒 | 通过 | +| Linux 核心 | 52 秒 | 39 项门禁耗时 25.84 秒 | 通过 | +| Linux 生产网站 | 44 秒 | VitePress 耗时 22.42 秒 | 通过 | +| 最慢的标准非 Windows 作业 | 43 秒 | Node 26 兼容性 | 通过 | +| Windows 完整流程 | 96 秒 | 37 项门禁耗时 31.13 秒 | 通过 | -Windows 观测作业花费 57 秒恢复 pnpm 缓存，因此其剩余余量既反映托管环境的设置波动，也反映仓库工作耗时。最终运行中每个非 Windows 作业仍低于 1 分钟，两个 Windows 作业也都低于 3 分钟。 +3 个付费作业同时开始运行。Windows 作业花费 21 秒恢复 pnpm 缓存、25 秒安装依赖，因此其余量既反映托管环境的设置波动，也反映仓库工作耗时。每个非 Windows 作业均低于 1 分钟，唯一的 Windows 作业也低于 3 分钟。 ## 曾考虑的替代方案 @@ -57,16 +57,20 @@ Windows 观测作业花费 57 秒恢复 pnpm 缓存，因此其剩余余量既 **根据关键通道基准测试选择生产规格。** 对单独的类型检查和网站构建而言，4 核看起来具备成本效益，但完整聚合流程发现了这些命令未覆盖的全仓库 lint 和存在依赖关系的产物工作。 -**在启动 Linux 聚合流程前预先构建。** 此方案让构建成为设置路径的一部分，并产生了一个耗时 66 秒的候选结果。在 `run-gates` 内尽早启动构建，既能保留产物依赖关系，又能让构建与无关检查重叠执行；最终聚合流程耗时 23.23 秒。 +**在 Linux 核心聚合流程内运行生产网站。** 托管环境设置处于热状态时，此方案用时 50 秒；当网站门禁耗时 29.05 秒时，总耗时增至 64 秒，超过了目标。一项独立的粗粒度网站作业能够保障目标，而无需重新采用门禁级分片。 + +**将必需的 Windows 检查和观测性 Windows 检查保留在不同作业中。** 这种拆分在工作流层保留了状态语义，却要支付两次设置开销，而且冷缓存让必需作业的耗时达到 266 秒。`run-gates` 目前在一个进程内保留这些语义：构建和生产网站属于必需项，其余清单则明确为非阻塞。 + +**在启动 Linux 聚合流程前预先构建。** 此方案让构建成为设置路径的一部分，并产生了一个耗时 66 秒的候选结果。在 `run-gates` 内尽早启动构建，既能保留产物依赖关系，又能让构建与无关检查重叠执行。 **在 Windows 上使用 ESLint 原生工作线程并发。** 16 个工作线程让 lint 比最终的单线程结果慢 5 倍以上。外层门禁并发能够利用 32 核运行器，同时不会成倍增加 ESLint 在 Windows 上启动工作线程和加载 TypeScript 项目的开销。 -**将兼容性、Python 和聚合作业迁移到大型运行器。** 这些标准运行器作业都在 40 秒以内完成。付费容量不会缩短关键路径。 +**将兼容性、Python 和聚合作业迁移到大型运行器。** 这些标准运行器作业都在 43 秒以内完成。付费容量不会缩短关键路径。 ## 后果 -最终生产验证产生的计费时长为：96 核 Linux 1 分钟、16 核 Windows 2 分钟和 32 核 Windows 3 分钟。按已配置的大型运行器费率计算，其大型运行器成本为 $0.902。全规格关键通道基准测试的成本为 $2.936。GitHub 会把每个大型运行器作业向上取整到整分钟计费，因此把付费作业数从 49 个减少到 3 个，与缩短仓库工作耗时同样重要。 +低扇出验证产生的计费时长为：96 核 Linux 1 分钟、16 核 Linux 1 分钟和 32 核 Windows 2 分钟。按已配置的大型运行器费率计算，其大型运行器成本为 $0.618。全规格关键通道基准测试的成本为 $2.936。GitHub 会把每个大型运行器作业向上取整到整分钟计费，因此把付费作业数从 49 个减少到 3 个，与缩短仓库工作耗时同样重要。 现有的零美元 Actions 预算并未阻止大型运行器作业。仅限本仓库的运行器组、有界的工作流拓扑、手动基准测试触发和作业超时限制才是经实测有效的成本控制机制；该预算不被视为执行防护措施。 -生产 CI 依赖本 Agent Note 和 `.github/workflows/ci.yml` 中由组织持有的运行器名称。池缺失或改名会让作业一直排队，不会回退到标准容量。手动全规格套件和原分片套件均予以保留，以便在映像、依赖、调度器或定价发生变化后重新测量，再调整生产标签。 +生产 CI 依赖本 Agent Note 和 `.github/workflows/ci.yml` 中由组织持有的运行器名称。池缺失或改名会让作业一直排队，不会回退到标准容量。手动全规格、完整聚合、原分片和串行套件均予以保留，以便在映像、依赖、调度器或定价发生变化后重新测量，再调整生产标签。 From c67ae11f78fffd4a36ae524b6246a708920d9a07 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 16:02:52 +0800 Subject: [PATCH 34/49] ci: stabilize compatibility runner setup --- .github/workflows/ci.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 09d61f845d..41a49c4269 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -417,7 +417,9 @@ jobs: node-compat: if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' - runs-on: ubuntu-latest + # Distinct larger-runner pools avoid both standard-runner setup outliers and + # the delayed second allocation observed when two jobs shared one pool. + runs-on: ${{ matrix.runner }} name: node ${{ matrix.node }} env: DSH_GATE_CONCURRENCY: '2' @@ -427,10 +429,13 @@ jobs: matrix: include: - node: '22.19' + runner: dsh-ubuntu-24-04-4core skip_typecheck: '1' - node: 24 + runner: dsh-ubuntu-24-04-8core skip_typecheck: '1' - node: 26 + runner: dsh-ubuntu-24-04-32core skip_typecheck: '1' steps: - uses: actions/checkout@v6 @@ -460,7 +465,7 @@ jobs: python-sdk: if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' - runs-on: ubuntu-latest + runs-on: dsh-ubuntu-24-04-64core name: python 3.10 / keyless SDK steps: - uses: actions/checkout@v6 From 9d81078f2f8476f60a22bde645f4a11c69b16bbc Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 16:10:52 +0800 Subject: [PATCH 35/49] ci: split primary cold-path budget --- .github/workflows/ci.yml | 61 +++++++++++++++++++++++++++++++--------- package.json | 1 + scripts/run-gates.ts | 59 +++++++++++++++++++++++++++----------- 3 files changed, 91 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41a49c4269..17cd71ffde 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -319,19 +319,16 @@ jobs: run: ${{ matrix.command }} # One large runner pays hosted setup once, then the repository scheduler - # overlaps the unsharded primary inventory except the production site build. + # overlaps the unsharded core inventory. CPU and site chains use coarse lanes. node-24: if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' runs-on: dsh-ubuntu-24-04-96core name: node 24 / core env: - # Thirty-two Vitest forks intermittently crash Node's CJS lexer on this image. - DSH_COVERAGE_MAX_WORKERS: '16' DSH_ESLINT_CACHE: '1' DSH_ESLINT_CONCURRENCY: '32' DSH_GATE_CONCURRENCY: '32' DSH_PUBLINT_CONCURRENCY: '32' - DSH_SNAPSHOT_MAX_CONCURRENCY: '32' steps: - uses: actions/checkout@v6 @@ -383,8 +380,8 @@ jobs: - name: Run unsharded primary Node core CI concurrently run: pnpm run check:ci:large-runner - # Keep only the longest independent Linux gate on a second coarse-grained - # runner so cold install variance does not push the primary box over a minute. + # Keep the longest independent Linux gate on its own coarse-grained runner so + # cold install variance does not push either primary aggregate over a minute. node-24-site: if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' runs-on: dsh-ubuntu-24-04-16core @@ -418,24 +415,39 @@ jobs: node-compat: if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' # Distinct larger-runner pools avoid both standard-runner setup outliers and - # the delayed second allocation observed when two jobs shared one pool. + # the delayed second allocation observed when two jobs shared one pool. The + # primary Node row also owns the CPU/dependency-critical primary gate chain. runs-on: ${{ matrix.runner }} - name: node ${{ matrix.node }} + name: ${{ matrix.name }} env: - DSH_GATE_CONCURRENCY: '2' + DSH_COVERAGE_MAX_WORKERS: ${{ matrix.coverage_workers }} + DSH_GATE_CONCURRENCY: ${{ matrix.gate_concurrency }} DSH_NODE_COMPAT_SKIP_TYPECHECK: ${{ matrix.skip_typecheck }} + DSH_SNAPSHOT_MAX_CONCURRENCY: ${{ matrix.snapshot_workers }} strategy: fail-fast: false matrix: include: - node: '22.19' + name: node 22.19 runner: dsh-ubuntu-24-04-4core + command: pnpm run check:node-compat + gate_concurrency: '2' skip_typecheck: '1' - node: 24 - runner: dsh-ubuntu-24-04-8core + name: node 24 / cpu + runner: dsh-ubuntu-24-04-64core + command: pnpm run check:ci:primary-cpu + coverage_workers: '16' + gate_concurrency: '6' + primary_cpu: true + snapshot_workers: '16' skip_typecheck: '1' - node: 26 + name: node 26 runner: dsh-ubuntu-24-04-32core + command: pnpm run check:node-compat + gate_concurrency: '2' skip_typecheck: '1' steps: - uses: actions/checkout@v6 @@ -458,14 +470,37 @@ jobs: ${{ runner.os }}-node-${{ matrix.node }}-pnpm- - name: Install (immutable) + if: matrix.primary_cpu != true run: pnpm install --frozen-lockfile - - name: Run compatibility gates - run: pnpm run check:node-compat + - name: Install and prepare bubblewrap + if: matrix.primary_cpu == true + run: | + pnpm install --frozen-lockfile & + install_pid=$! + ( + if ! sudo apt-get install -yq --no-install-recommends bubblewrap; then + echo "initial bubblewrap install failed; refreshing APT indexes and retrying" + sudo apt-get update -q + sudo apt-get install -yq --no-install-recommends bubblewrap + fi + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 \ + || echo "apparmor userns knob absent — the functional probe decides" + ) & + sandbox_pid=$! + install_status=0 + wait "$install_pid" || install_status=$? + sandbox_status=0 + wait "$sandbox_pid" || sandbox_status=$? + if (( install_status != 0 )); then exit "$install_status"; fi + exit "$sandbox_status" + + - name: Run assigned gates + run: ${{ matrix.command }} python-sdk: if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' - runs-on: dsh-ubuntu-24-04-64core + runs-on: dsh-ubuntu-24-04-8core name: python 3.10 / keyless SDK steps: - uses: actions/checkout@v6 diff --git a/package.json b/package.json index c9e5e01a71..c318a3a94d 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "test:snapshot:record": "DSH_SNAPSHOT=record vitest run --config vitest.snapshot.config.ts --update", "test:snapshot:refresh": "DSH_SNAPSHOT=refresh vitest run --config vitest.snapshot.config.ts", "check:ci": "tsx scripts/run-gates.ts ci-primary", + "check:ci:primary-cpu": "tsx scripts/run-gates.ts ci-primary-cpu", "check:ci:large-runner": "tsx scripts/run-gates.ts ci-primary-large-runner", "check:ci:static": "tsx scripts/run-gates.ts ci-static", "check:ci:lint": "tsx scripts/run-gates.ts ci-lint", diff --git a/scripts/run-gates.ts b/scripts/run-gates.ts index 11155551db..ff4b1aab2d 100644 --- a/scripts/run-gates.ts +++ b/scripts/run-gates.ts @@ -15,6 +15,7 @@ import { selectStaticGates } from './static-shards.ts' type Mode = | 'ci-primary' + | 'ci-primary-cpu' | 'ci-primary-large-runner' | 'ci-static' | 'ci-lint' @@ -92,6 +93,7 @@ if (results.some(result => result.gate.allowFailure !== true && (result.status = function parseMode(raw: string | undefined): Mode { switch (raw) { case 'ci-primary': + case 'ci-primary-cpu': case 'ci-primary-large-runner': case 'ci-static': case 'ci-lint': @@ -107,7 +109,7 @@ function parseMode(raw: string | undefined): Mode { return raw default: throw new Error( - `run-gates: expected mode ci-primary | ci-primary-large-runner | ci-static | ci-lint | ci-coverage | ci-snapshot | ci-artifacts | ci-windows-blocking | ci-windows-complete | ci-windows-observational | node-compat | pre-push | doc-sync, got ${JSON.stringify(raw)}.`, + `run-gates: expected mode ci-primary | ci-primary-cpu | ci-primary-large-runner | ci-static | ci-lint | ci-coverage | ci-snapshot | ci-artifacts | ci-windows-blocking | ci-windows-complete | ci-windows-observational | node-compat | pre-push | doc-sync, got ${JSON.stringify(raw)}.`, ) } } @@ -173,6 +175,8 @@ function gatesForMode(selected: Mode): Gate[] { switch (selected) { case 'ci-primary': return ciPrimaryGates() + case 'ci-primary-cpu': + return ciPrimaryCpuGates() case 'ci-primary-large-runner': return ciPrimaryLargeRunnerGates() case 'ci-static': @@ -199,19 +203,7 @@ function gatesForMode(selected: Mode): Gate[] { case 'ci-windows-observational': return ciWindowsObservationalGates() case 'node-compat': - return [ - ...flagEnabled('DSH_NODE_COMPAT_SKIP_TYPECHECK') ? [] : [pnpmScript('typecheck', 'typecheck')], - pnpmExec('source-worker-smoke', [ - 'vitest', - 'run', - 'packages/workflow/workflow-workerthread/tests/source-worker.compat.spec.ts', - ], { label: 'source worker smoke' }), - pnpmExec('jsonl-zstd-smoke', [ - 'vitest', - 'run', - 'packages/session-persistence/session-persistence-jsonl/tests/zstd.compat.spec.ts', - ], { label: 'JSONL Zstandard smoke' }), - ] + return nodeCompatGates() case 'pre-push': return [ pnpmScript('runtime-closure', 'verify-runtime-closure', { label: 'runtime closure' }), @@ -258,10 +250,10 @@ function ciPrimaryGates(): Gate[] { } function ciPrimaryLargeRunnerGates(): Gate[] { - // Typecheck does not consume build output, so a large runner can start both - // together while snapshot and artifact consumers still wait for the build. + // The CPU lane owns typecheck, coverage, and the build-to-snapshot chain. + // This core lane starts its own build eagerly for the remaining artifact consumers. return ciPrimaryGates() - .filter(gate => gate.id !== 'docs-site-build') + .filter(gate => !['coverage', 'docs-site-build', 'snapshot', 'typecheck'].includes(gate.id)) .map((gate) => { if (gate.id !== 'build') return gate const eagerBuild = { ...gate } @@ -270,6 +262,39 @@ function ciPrimaryLargeRunnerGates(): Gate[] { }) } +function ciPrimaryCpuGates(): Gate[] { + // Build and snapshot stay together so the dependent replay consumes this lane's output. + return [ + pnpmScript('typecheck', 'typecheck'), + coverageGate(), + pnpmScript('build', 'build'), + snapshotGate(), + ...nodeCompatSmokeGates(), + ] +} + +function nodeCompatGates(): Gate[] { + return [ + ...flagEnabled('DSH_NODE_COMPAT_SKIP_TYPECHECK') ? [] : [pnpmScript('typecheck', 'typecheck')], + ...nodeCompatSmokeGates(), + ] +} + +function nodeCompatSmokeGates(): Gate[] { + return [ + pnpmExec('source-worker-smoke', [ + 'vitest', + 'run', + 'packages/workflow/workflow-workerthread/tests/source-worker.compat.spec.ts', + ], { label: 'source worker smoke' }), + pnpmExec('jsonl-zstd-smoke', [ + 'vitest', + 'run', + 'packages/session-persistence/session-persistence-jsonl/tests/zstd.compat.spec.ts', + ], { label: 'JSONL Zstandard smoke' }), + ] +} + function ciStaticGates(): Gate[] { const gates = [ pnpmScript('runtime-closure', 'verify-runtime-closure', { label: 'runtime closure' }), From 2bd466a4a68acf0cd78db03a1256ed3f7f69a83f Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 16:21:17 +0800 Subject: [PATCH 36/49] docs: record all-pool validation --- ...ence-based-larger-hosted-runners.i18n.yaml | 4 +-- ...22-evidence-based-larger-hosted-runners.md | 33 ++++++++++++------- ...evidence-based-larger-hosted-runners.zh.md | 33 ++++++++++++------- 3 files changed, 44 insertions(+), 26 deletions(-) diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml index d5b558837f..5e7792d08d 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -2026-07-22-evidence-based-larger-hosted-runners.md: 7d38552d23f7a3fb8abca45cbb4cf4e9c82fdd78 -2026-07-22-evidence-based-larger-hosted-runners.zh.md: f16db8eeb9000fe9a7671345efa9b1fd7fc744e1 +2026-07-22-evidence-based-larger-hosted-runners.md: 5390cce001f5d493fc4fb3eaee9203c34af1175d +2026-07-22-evidence-based-larger-hosted-runners.zh.md: 04a32f9ac1e0b131a695695b8f6cece3a8ffd3fa diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md index 7d38552d23..5390cce001 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md @@ -14,13 +14,17 @@ Larger runners make it possible to pay setup once and parallelize inside the rep The organization keeps twelve x64 larger-runner pools in the repo-restricted `dsh-larger-ci` group: Ubuntu 24.04 and Windows 2025 at 4, 8, 16, 32, 64, and 96 cores. Public IPs are disabled. Each pool has an autoscaling ceiling of 256; the ceiling does not allocate idle machines or remove the need to bound workflow demand. -Production CI uses three coarse larger-runner jobs and keeps Node compatibility, Python, and the final aggregator on standard runners: +Production CI assigns each of the six Linux pool sizes exactly once, assigns one 32-core Windows pool, and keeps only the final aggregator on a standard runner. The version and language jobs are environment contracts rather than slices of one gate inventory; the primary Node work has three coarse lanes instead of a gate-level shard matrix: -- `node 24 / core` uses the 96-core Linux pool. One checkout, setup, cache restore, and install feeds the unsharded 39-gate primary inventory other than the production site build. `run-gates` starts up to 32 independent gates, ESLint uses 32 workers, snapshots use up to 32 subprocesses, and coverage uses 16 forks. Build starts beside typecheck; snapshot and artifact consumers still wait for emitted output. Coverage stays below 32 forks because that setting twice caused Node 24's CJS lexer to terminate a Vitest worker and invalidate coverage. +- `node 24 / core` uses the 96-core Linux pool. One checkout, setup, cache restore, and install feeds 36 unsharded static, lint, documentation, hygiene, build, and artifact gates. `run-gates` starts up to 32 independent gates and ESLint uses 32 workers. Build starts eagerly; its artifact consumers still wait for emitted output. +- `node 24 / cpu` uses the 64-core Linux pool for six CPU- or dependency-critical gates: typecheck, coverage, build followed by snapshot replay, and two Node 24 compatibility smokes. Coverage and snapshot each use at most 16 workers. This lane builds separately so snapshot replay consumes same-lane output. Coverage stays below 32 forks because that setting twice caused Node 24's CJS lexer to terminate a Vitest worker and invalidate coverage. - `node 24 / production site` uses the 16-core Linux pool for the longest independent primary gate. This is one coarse split, not a shard matrix: the job performs one setup and one production VitePress build. +- Node 22.19 compatibility, Python 3.10, and Node 26 compatibility use the 4-, 8-, and 32-core Linux pools respectively. Distinct labels avoid both standard-runner setup outliers and the delayed second allocation observed when two jobs shared one pool. - `windows node 24 / complete` uses the 32-core Windows pool. One setup feeds the required package build, the required production site build, and the complete observational portability inventory. The outer scheduler has 32 slots. Required failures fail the job; observational failures are printed as non-blocking and preserve their former advisory status. ESLint itself stays single-threaded because 16 ESLint worker threads increased full-lint time to 174.54 seconds; outer gate concurrency uses the runner without multiplying Windows worker startup and TypeScript project loading. -The final shape followed two cold-path observations. A first candidate used two 16-core Windows jobs, and GitHub took 93 seconds to provision the second same-label runner despite the configured autoscaling ceiling. A later [documentation-head validation](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29900502413) took 64 seconds on the combined Linux job because its production site gate lasted 29.05 seconds, while a separate Windows blocking job took 266 seconds after spending 138 seconds restoring a 153 MB pnpm cache. Moving that independent Linux gate to one coarse 16-core lane and combining all Windows work on one 32-core box removed both duplicate setup waves. +The Windows shape followed two cold-path observations. A first candidate used two 16-core Windows jobs, and GitHub took 93 seconds to provision the second same-label runner despite the configured autoscaling ceiling. A later [documentation-head validation](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29900502413) took 266 seconds on a separate Windows blocking job after spending 138 seconds restoring a 153 MB pnpm cache. Combining all Windows work on one 32-core box removed the duplicate setup wave. + +Two later runs set the Linux boundaries. A [standard-runner validation](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29902209492) took 67 seconds for Node 26 even though repository work took five seconds, because GitHub spent 36 seconds in `Set up job`. Moving the environment contracts to distinct larger pools removed that lottery. The next [all-larger-runner validation](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29902541203) took 68 seconds on the 96-core primary job: repository work remained 26 seconds, but setup, cache, install, and finalization consumed 42 seconds. Moving typecheck, coverage, and the build-to-snapshot dependency chain to one coarse 64-core lane reduced the 96-core lane's repository critical path to 14.81 seconds without returning to per-gate shards. The workflow retains four manual diagnostics. `suite=larger-runner-benchmark` compares isolated critical lanes across every size, `suite=consolidated-runner-benchmark` compares whole aggregates, `suite=sharded-reference` preserves the former production shard topology, and `suite=serial-reference` remains the unsharded cross-platform completeness oracle. `suite=optimized-larger-runners` runs the exact production topology against a branch ref when a pull request cannot form a merge commit. @@ -40,16 +44,19 @@ Those isolated results showed that setup dominated but did not identify the prod The Linux 32-core failure was the first CJS-lexer worker crash. The 96-core aggregate was the only successful all-size result at the one-minute boundary. Although Windows repository work gained little above 16 cores, the 32-core pool can start the complete outer inventory together and, more importantly, removes an entire paid setup from production. -The exact [reduced-fanout validation run](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29901539360) passed every job at the tested branch head: +The exact [all-pool validation run](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29903067274) passed every job at the tested branch head: -| Production job | Active time | Repository work | Result | +| Production job (pool) | Active time | Repository work | Result | |---|---:|---:|---:| -| Linux core | 52 s | 39 gates in 25.84 s | passed | -| Linux production site | 44 s | VitePress in 22.42 s | passed | -| Slowest standard non-Windows job | 43 s | Node 26 compatibility | passed | -| Windows complete | 96 s | 37 gates in 31.13 s | passed | +| Node 22.19 compatibility (Linux 4) | 26 s | compatibility smokes | passed | +| Python 3.10 (Linux 8) | 22 s | complete keyless SDK suite | passed | +| Production site (Linux 16) | 39 s | VitePress in 22.57 s | passed | +| Node 26 compatibility (Linux 32) | 24 s | compatibility smokes | passed | +| Primary CPU (Linux 64) | 53 s | 6 gates in 23.70 s | passed | +| Primary core (Linux 96) | 47 s | 36 gates in 14.81 s | passed | +| Windows complete (Windows 32) | 109 s | 37 gates in 32.71 s | passed | -All three paid jobs began together. The Windows job spent 21 seconds restoring its pnpm cache and 25 seconds installing dependencies, so its margin measures hosted setup variance as well as repository work. Every non-Windows job stays below one minute and the sole Windows job stays below three minutes. +All seven paid jobs began in the same second. The slowest non-Windows job finished in 53 seconds. The Windows job spent 20 seconds restoring its pnpm cache and 15 seconds installing dependencies, so its 109-second active time measures hosted setup variance as well as repository work. Every non-Windows job stays below one minute and the sole Windows job stays below three minutes. ## Alternatives considered @@ -59,17 +66,19 @@ All three paid jobs began together. The Windows job spent 21 seconds restoring i **Run the production site inside the Linux core aggregate.** This reached 50 seconds with warm hosted setup, then crossed the threshold at 64 seconds when the site gate took 29.05 seconds. One coarse independent site job protects the target without returning to gate-level sharding. +**Keep every primary gate on the 96-core Linux runner.** Repository work completed in 26 seconds, but a 42-second cold path still pushed the job to 68 seconds. The 64-core CPU lane owns the three longest independent or dependency-critical paths; the remaining 36-gate core inventory completes its repository work in 14.81 seconds. + **Keep required and observational Windows checks in separate jobs.** The split preserved status semantics at the workflow level but paid setup twice, and a cold cache pushed the required job to 266 seconds. `run-gates` now preserves those semantics inside one process: build and production site are required, while the remaining inventory is explicitly non-blocking. **Prebuild before starting the Linux aggregate.** This moved build onto the setup path and produced a 66-second candidate. Starting build eagerly inside `run-gates` preserves artifact dependencies while overlapping it with unrelated checks. **Use native ESLint worker concurrency on Windows.** Sixteen workers made lint more than five times slower than the final single-threaded result. Outer gate parallelism uses the 32-core runner without multiplying ESLint's Windows worker startup and TypeScript project loading. -**Move compatibility, Python, and aggregation to larger runners.** These standard-runner jobs completed in 43 seconds or less. Paid capacity would not shorten the critical path. +**Keep compatibility and Python on standard runners.** Warm runs completed in 43 seconds or less, but one Node 26 job later spent 36 seconds in GitHub setup and crossed the target despite only five seconds of repository work. Distinct larger pools stabilize those environment contracts; the three-second final aggregator remains on a standard runner because it begins only after the paid jobs release capacity. ## Consequences -The reduced-fanout validation consumed one billed 96-core Linux minute, one billed 16-core Linux minute, and two billed 32-core Windows minutes. At the configured larger-runner rates, its larger-runner cost was $0.618. The all-size critical benchmark cost $2.936. GitHub rounds each larger-runner job up to a whole minute, so reducing paid job count from 49 to three matters as much as shortening repository work. +The all-pool validation consumed one billed minute at each Linux size and two billed 32-core Windows minutes. At the configured larger-runner rates, its larger-runner cost was $0.896. The all-size critical benchmark cost $2.936. GitHub rounds each larger-runner job up to a whole minute, so reducing paid job count from 49 to seven matters as much as shortening repository work. The existing zero-dollar Actions budget did not block larger-runner jobs. The repo-only runner group, bounded workflow topology, manual benchmark triggers, and job timeouts are the observed cost controls; the budget is not treated as an execution guard. diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md index f16db8eeb9..04a32f9ac1 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md @@ -14,13 +14,17 @@ Status: implemented 组织在仅限本仓库使用的 `dsh-larger-ci` 运行器组中保留 12 个 x64 大型运行器池：Ubuntu 24.04 和 Windows 2025 各设 4、8、16、32、64、96 核规格。公网 IP 已禁用。每个池的自动扩缩容上限为 256；该上限既不会分配闲置机器，也不能免除限制工作流需求的必要性。 -生产 CI 使用 3 个粗粒度大型运行器作业，并让 Node 兼容性、Python 和最终聚合作业继续使用标准运行器： +生产 CI 对 6 种 Linux 池规格各使用一次，另使用一个 32 核 Windows 池，只有最终聚合作业继续使用标准运行器。版本和语言作业属于环境契约，并非从同一门禁清单切出的分片；主 Node 工作采用 3 个粗粒度通道，而非门禁级分片矩阵： -- `node 24 / core` 使用 96 核 Linux 池。只需执行一次代码检出、设置、缓存恢复和安装，即可供生产网站构建以外、未分片的 39 项主门禁清单使用。`run-gates` 最多同时启动 32 项相互独立的门禁，ESLint 使用 32 个工作线程，快照最多使用 32 个子进程，覆盖率运行使用 16 个 fork。构建与类型检查同时启动；快照和产物消费方仍会等待生成的输出。覆盖率运行的 fork 数保持低于 32，因为将其设为 32 曾两次导致 Node 24 的 CJS 词法分析器终止一个 Vitest 工作进程，使覆盖率结果失效。 +- `node 24 / core` 使用 96 核 Linux 池。只需执行一次代码检出、设置、缓存恢复和安装，即可供 36 项未分片的静态、lint、文档、hygiene、构建和产物门禁使用。`run-gates` 最多同时启动 32 项相互独立的门禁，ESLint 使用 32 个工作线程。构建会尽早启动；其产物消费方仍会等待生成的输出。 +- `node 24 / cpu` 使用 64 核 Linux 池运行 6 项 CPU 或依赖关键门禁：类型检查、覆盖率、构建后回放快照，以及两项 Node 24 兼容性冒烟测试。覆盖率和快照各最多使用 16 个工作进程。该通道单独构建，使快照回放使用同一通道生成的输出。覆盖率运行的 fork 数保持低于 32，因为将其设为 32 曾两次导致 Node 24 的 CJS 词法分析器终止一个 Vitest 工作进程，使覆盖率结果失效。 - `node 24 / production site` 使用 16 核 Linux 池运行耗时最长的独立主门禁。这是一项粗粒度拆分，而非分片矩阵：该作业执行一次设置和一次生产版 VitePress 构建。 +- Node 22.19 兼容性、Python 3.10 和 Node 26 兼容性分别使用 4 核、8 核和 32 核 Linux 池。不同标签既避免了标准运行器设置耗时的异常值，也避免了两个作业共享一个池时曾出现的第二次分配延迟。 - `windows node 24 / complete` 使用 32 核 Windows 池。一次设置供必需的包构建、必需的生产网站构建以及完整的观测性可移植性清单共用。外层调度器设有 32 个槽位。任何必需项失败都会使作业失败；观测项失败会明确打印为非阻塞，并保留其原有的建议性状态。ESLint 本身仍采用单线程，因为启用 16 个 ESLint 工作线程会把完整 lint 的耗时增加至 174.54 秒；外层门禁并发能够利用该运行器，同时不会成倍增加 Windows 工作线程的启动和 TypeScript 项目加载开销。 -最终结构由两项冷路径观测结果确定。首个候选方案使用两个 16 核 Windows 作业；尽管已配置自动扩缩容上限，GitHub 仍花费 93 秒才预配好第二台同标签运行器。随后一次[文档分支头验证](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29900502413)中，合并后的 Linux 作业耗时 64 秒，因为其中的生产网站门禁耗时 29.05 秒；另一个独立的 Windows 阻塞性作业则耗时 266 秒，其中恢复 153 MB pnpm 缓存就用了 138 秒。把这项独立的 Linux 门禁移入一个粗粒度 16 核通道，并把所有 Windows 工作合并到一台 32 核机器上，消除了这两轮重复设置。 +Windows 结构由两项冷路径观测结果确定。首个候选方案使用两个 16 核 Windows 作业；尽管已配置自动扩缩容上限，GitHub 仍花费 93 秒才预配好第二台同标签运行器。随后一次[文档分支头验证](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29900502413)中，一个独立的 Windows 阻塞性作业耗时 266 秒，其中恢复 153 MB pnpm 缓存就用了 138 秒。把所有 Windows 工作合并到一台 32 核机器上，消除了这轮重复设置。 + +随后两次运行确定了 Linux 的边界。一次[标准运行器验证](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29902209492)中，Node 26 作业耗时 67 秒，而仓库工作仅耗时 5 秒，因为 GitHub 在 `Set up job` 上花费了 36 秒。把各项环境契约迁移到不同的大型运行器池后，这种随机波动随之消失。接下来的[全大型运行器验证](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29902541203)中，96 核主作业耗时 68 秒：仓库工作仍为 26 秒，但设置、缓存、安装和收尾共占用 42 秒。把类型检查、覆盖率以及构建至快照的依赖链迁移到一个粗粒度 64 核通道后，96 核通道的仓库关键路径缩短至 14.81 秒，且无需恢复按门禁分片。 工作流保留 4 项手动诊断。`suite=larger-runner-benchmark` 比较所有规格下相互独立的关键通道，`suite=consolidated-runner-benchmark` 比较完整聚合流程，`suite=sharded-reference` 保留原生产分片拓扑，`suite=serial-reference` 则继续作为未分片的跨平台完整性判定基准。当拉取请求无法生成合并提交时，`suite=optimized-larger-runners` 会直接针对分支引用运行与生产环境完全相同的拓扑。 @@ -40,16 +44,19 @@ Status: implemented Linux 32 核作业的失败是首次发生的 CJS 词法分析器工作进程崩溃。在所有规格的结果中，96 核聚合作业是唯一成功达到 1 分钟边界的结果。尽管 Windows 仓库工作在超过 16 核后收益很小，32 核池可以让完整的外层清单同时启动；更重要的是，它能从生产环境中消除整整一次付费设置。 -精确的[低扇出验证运行](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29901539360)在受测分支头通过了所有作业： +精确的[全运行器池验证运行](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29903067274)在受测分支头通过了所有作业： -| 生产作业 | 活动耗时 | 仓库工作 | 结果 | +| 生产作业（运行器池） | 活动耗时 | 仓库工作 | 结果 | |---|---:|---:|---:| -| Linux 核心 | 52 秒 | 39 项门禁耗时 25.84 秒 | 通过 | -| Linux 生产网站 | 44 秒 | VitePress 耗时 22.42 秒 | 通过 | -| 最慢的标准非 Windows 作业 | 43 秒 | Node 26 兼容性 | 通过 | -| Windows 完整流程 | 96 秒 | 37 项门禁耗时 31.13 秒 | 通过 | +| Node 22.19 兼容性（Linux 4） | 26 秒 | 兼容性冒烟测试 | 通过 | +| Python 3.10（Linux 8） | 22 秒 | 完整无密钥 SDK 套件 | 通过 | +| 生产网站（Linux 16） | 39 秒 | VitePress 耗时 22.57 秒 | 通过 | +| Node 26 兼容性（Linux 32） | 24 秒 | 兼容性冒烟测试 | 通过 | +| 主流程 CPU（Linux 64） | 53 秒 | 6 项门禁耗时 23.70 秒 | 通过 | +| 主流程核心（Linux 96） | 47 秒 | 36 项门禁耗时 14.81 秒 | 通过 | +| Windows 完整流程（Windows 32） | 109 秒 | 37 项门禁耗时 32.71 秒 | 通过 | -3 个付费作业同时开始运行。Windows 作业花费 21 秒恢复 pnpm 缓存、25 秒安装依赖，因此其余量既反映托管环境的设置波动，也反映仓库工作耗时。每个非 Windows 作业均低于 1 分钟，唯一的 Windows 作业也低于 3 分钟。 +7 个付费作业均在同一秒开始运行。最慢的非 Windows 作业耗时 53 秒。Windows 作业花费 20 秒恢复 pnpm 缓存、15 秒安装依赖，因此其 109 秒的活动耗时既反映托管环境的设置波动，也反映仓库工作耗时。每个非 Windows 作业均低于 1 分钟，唯一的 Windows 作业也低于 3 分钟。 ## 曾考虑的替代方案 @@ -59,17 +66,19 @@ Linux 32 核作业的失败是首次发生的 CJS 词法分析器工作进程崩 **在 Linux 核心聚合流程内运行生产网站。** 托管环境设置处于热状态时，此方案用时 50 秒；当网站门禁耗时 29.05 秒时，总耗时增至 64 秒，超过了目标。一项独立的粗粒度网站作业能够保障目标，而无需重新采用门禁级分片。 +**将所有主门禁都保留在 96 核 Linux 运行器上。** 仓库工作在 26 秒内完成，但 42 秒的冷路径仍把作业总耗时推高至 68 秒。64 核 CPU 通道负责 3 条耗时最长的独立或依赖关键路径；其余 36 项核心门禁清单的仓库工作耗时为 14.81 秒。 + **将必需的 Windows 检查和观测性 Windows 检查保留在不同作业中。** 这种拆分在工作流层保留了状态语义，却要支付两次设置开销，而且冷缓存让必需作业的耗时达到 266 秒。`run-gates` 目前在一个进程内保留这些语义：构建和生产网站属于必需项，其余清单则明确为非阻塞。 **在启动 Linux 聚合流程前预先构建。** 此方案让构建成为设置路径的一部分，并产生了一个耗时 66 秒的候选结果。在 `run-gates` 内尽早启动构建，既能保留产物依赖关系，又能让构建与无关检查重叠执行。 **在 Windows 上使用 ESLint 原生工作线程并发。** 16 个工作线程让 lint 比最终的单线程结果慢 5 倍以上。外层门禁并发能够利用 32 核运行器，同时不会成倍增加 ESLint 在 Windows 上启动工作线程和加载 TypeScript 项目的开销。 -**将兼容性、Python 和聚合作业迁移到大型运行器。** 这些标准运行器作业都在 43 秒以内完成。付费容量不会缩短关键路径。 +**让兼容性和 Python 继续使用标准运行器。** 热运行均在 43 秒以内完成，但后来一个 Node 26 作业在 GitHub 设置上花费 36 秒，即使仓库工作仅耗时 5 秒，仍超过了目标。不同的大型运行器池能够稳定这些环境契约；最终聚合作业耗时 3 秒，仍使用标准运行器，因为它只会在付费作业释放容量后启动。 ## 后果 -低扇出验证产生的计费时长为：96 核 Linux 1 分钟、16 核 Linux 1 分钟和 32 核 Windows 2 分钟。按已配置的大型运行器费率计算，其大型运行器成本为 $0.618。全规格关键通道基准测试的成本为 $2.936。GitHub 会把每个大型运行器作业向上取整到整分钟计费，因此把付费作业数从 49 个减少到 3 个，与缩短仓库工作耗时同样重要。 +全运行器池验证中，每种 Linux 规格各计费 1 分钟，32 核 Windows 计费 2 分钟。按已配置的大型运行器费率计算，其大型运行器成本为 $0.896。全规格关键通道基准测试的成本为 $2.936。GitHub 会把每个大型运行器作业向上取整到整分钟计费，因此把付费作业数从 49 个减少到 7 个，与缩短仓库工作耗时同样重要。 现有的零美元 Actions 预算并未阻止大型运行器作业。仅限本仓库的运行器组、有界的工作流拓扑、手动基准测试触发和作业超时限制才是经实测有效的成本控制机制；该预算不被视为执行防护措施。 From b63e6ee72d4cb0453285e0f5f5e124b2388e34ff Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 16:26:25 +0800 Subject: [PATCH 37/49] ci: bound coverage worker forks --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 17cd71ffde..80a158210e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -438,7 +438,7 @@ jobs: name: node 24 / cpu runner: dsh-ubuntu-24-04-64core command: pnpm run check:ci:primary-cpu - coverage_workers: '16' + coverage_workers: '12' gate_concurrency: '6' primary_cpu: true snapshot_workers: '16' From 5e4776500d4335c0de4ca7f71ca276c04af514a1 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 16:33:47 +0800 Subject: [PATCH 38/49] docs: record stable coverage concurrency --- ...ence-based-larger-hosted-runners.i18n.yaml | 4 ++-- ...22-evidence-based-larger-hosted-runners.md | 22 ++++++++++--------- ...evidence-based-larger-hosted-runners.zh.md | 22 ++++++++++--------- 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml index 5e7792d08d..509cff58e2 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -2026-07-22-evidence-based-larger-hosted-runners.md: 5390cce001f5d493fc4fb3eaee9203c34af1175d -2026-07-22-evidence-based-larger-hosted-runners.zh.md: 04a32f9ac1e0b131a695695b8f6cece3a8ffd3fa +2026-07-22-evidence-based-larger-hosted-runners.md: 81de62fa15623b96ac4913ee6c6fc15e886222b0 +2026-07-22-evidence-based-larger-hosted-runners.zh.md: 4e31937284b221d92552957ecc0434d9fc2e2914 diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md index 5390cce001..81de62fa15 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md @@ -17,7 +17,7 @@ The organization keeps twelve x64 larger-runner pools in the repo-restricted `ds Production CI assigns each of the six Linux pool sizes exactly once, assigns one 32-core Windows pool, and keeps only the final aggregator on a standard runner. The version and language jobs are environment contracts rather than slices of one gate inventory; the primary Node work has three coarse lanes instead of a gate-level shard matrix: - `node 24 / core` uses the 96-core Linux pool. One checkout, setup, cache restore, and install feeds 36 unsharded static, lint, documentation, hygiene, build, and artifact gates. `run-gates` starts up to 32 independent gates and ESLint uses 32 workers. Build starts eagerly; its artifact consumers still wait for emitted output. -- `node 24 / cpu` uses the 64-core Linux pool for six CPU- or dependency-critical gates: typecheck, coverage, build followed by snapshot replay, and two Node 24 compatibility smokes. Coverage and snapshot each use at most 16 workers. This lane builds separately so snapshot replay consumes same-lane output. Coverage stays below 32 forks because that setting twice caused Node 24's CJS lexer to terminate a Vitest worker and invalidate coverage. +- `node 24 / cpu` uses the 64-core Linux pool for six CPU- or dependency-critical gates: typecheck, coverage, build followed by snapshot replay, and two Node 24 compatibility smokes. Coverage uses at most 12 workers and snapshot uses at most 16. This lane builds separately so snapshot replay consumes same-lane output. Coverage stays at 12 forks because 32 forks crashed Node 24's CJS lexer twice and a later 16-fork run reproduced the same worker loss and invalid coverage. - `node 24 / production site` uses the 16-core Linux pool for the longest independent primary gate. This is one coarse split, not a shard matrix: the job performs one setup and one production VitePress build. - Node 22.19 compatibility, Python 3.10, and Node 26 compatibility use the 4-, 8-, and 32-core Linux pools respectively. Distinct labels avoid both standard-runner setup outliers and the delayed second allocation observed when two jobs shared one pool. - `windows node 24 / complete` uses the 32-core Windows pool. One setup feeds the required package build, the required production site build, and the complete observational portability inventory. The outer scheduler has 32 slots. Required failures fail the job; observational failures are printed as non-blocking and preserve their former advisory status. ESLint itself stays single-threaded because 16 ESLint worker threads increased full-lint time to 174.54 seconds; outer gate concurrency uses the runner without multiplying Windows worker startup and TypeScript project loading. @@ -26,6 +26,8 @@ The Windows shape followed two cold-path observations. A first candidate used tw Two later runs set the Linux boundaries. A [standard-runner validation](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29902209492) took 67 seconds for Node 26 even though repository work took five seconds, because GitHub spent 36 seconds in `Set up job`. Moving the environment contracts to distinct larger pools removed that lottery. The next [all-larger-runner validation](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29902541203) took 68 seconds on the 96-core primary job: repository work remained 26 seconds, but setup, cache, install, and finalization consumed 42 seconds. Moving typecheck, coverage, and the build-to-snapshot dependency chain to one coarse 64-core lane reduced the 96-core lane's repository critical path to 14.81 seconds without returning to per-gate shards. +A [documentation-head repeat](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29903735616) showed that 16 coverage forks still admitted the CJS-lexer crash. The coverage process completed its remaining tests in 23.73 seconds, but the dead worker left one file below threshold and correctly failed the lane. Twelve forks completed the same gate in 26.06 seconds in the final run, keeping the CPU lane below one minute while restoring process headroom. + The workflow retains four manual diagnostics. `suite=larger-runner-benchmark` compares isolated critical lanes across every size, `suite=consolidated-runner-benchmark` compares whole aggregates, `suite=sharded-reference` preserves the former production shard topology, and `suite=serial-reference` remains the unsharded cross-platform completeness oracle. `suite=optimized-larger-runners` runs the exact production topology against a branch ref when a pull request cannot form a merge commit. The first [twelve-size critical-lane benchmark](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29895295659) used a workflow-only commit on top of the standard-runner [baseline](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29850033610), so the code, lockfile, and commands were identical: @@ -44,19 +46,19 @@ Those isolated results showed that setup dominated but did not identify the prod The Linux 32-core failure was the first CJS-lexer worker crash. The 96-core aggregate was the only successful all-size result at the one-minute boundary. Although Windows repository work gained little above 16 cores, the 32-core pool can start the complete outer inventory together and, more importantly, removes an entire paid setup from production. -The exact [all-pool validation run](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29903067274) passed every job at the tested branch head: +The exact [all-pool validation run](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29904080103) passed every job at the tested branch head: | Production job (pool) | Active time | Repository work | Result | |---|---:|---:|---:| -| Node 22.19 compatibility (Linux 4) | 26 s | compatibility smokes | passed | -| Python 3.10 (Linux 8) | 22 s | complete keyless SDK suite | passed | -| Production site (Linux 16) | 39 s | VitePress in 22.57 s | passed | -| Node 26 compatibility (Linux 32) | 24 s | compatibility smokes | passed | -| Primary CPU (Linux 64) | 53 s | 6 gates in 23.70 s | passed | -| Primary core (Linux 96) | 47 s | 36 gates in 14.81 s | passed | -| Windows complete (Windows 32) | 109 s | 37 gates in 32.71 s | passed | +| Node 22.19 compatibility (Linux 4) | 30 s | compatibility smokes | passed | +| Python 3.10 (Linux 8) | 31 s | complete keyless SDK suite | passed | +| Production site (Linux 16) | 51 s | VitePress in 24.23 s | passed | +| Node 26 compatibility (Linux 32) | 27 s | compatibility smokes | passed | +| Primary CPU (Linux 64) | 56 s | 6 gates in 26.07 s | passed | +| Primary core (Linux 96) | 49 s | 36 gates in 15.58 s | passed | +| Windows complete (Windows 32) | 102 s | 37 gates in 29.98 s | passed | -All seven paid jobs began in the same second. The slowest non-Windows job finished in 53 seconds. The Windows job spent 20 seconds restoring its pnpm cache and 15 seconds installing dependencies, so its 109-second active time measures hosted setup variance as well as repository work. Every non-Windows job stays below one minute and the sole Windows job stays below three minutes. +All seven paid jobs began within one second. The slowest non-Windows job finished in 56 seconds. The Windows job spent 21 seconds restoring its pnpm cache and 14 seconds installing dependencies, so its 102-second active time measures hosted setup variance as well as repository work. Every non-Windows job stays below one minute and the sole Windows job stays below three minutes. ## Alternatives considered diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md index 04a32f9ac1..4e31937284 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md @@ -17,7 +17,7 @@ Status: implemented 生产 CI 对 6 种 Linux 池规格各使用一次，另使用一个 32 核 Windows 池，只有最终聚合作业继续使用标准运行器。版本和语言作业属于环境契约，并非从同一门禁清单切出的分片；主 Node 工作采用 3 个粗粒度通道，而非门禁级分片矩阵： - `node 24 / core` 使用 96 核 Linux 池。只需执行一次代码检出、设置、缓存恢复和安装，即可供 36 项未分片的静态、lint、文档、hygiene、构建和产物门禁使用。`run-gates` 最多同时启动 32 项相互独立的门禁，ESLint 使用 32 个工作线程。构建会尽早启动；其产物消费方仍会等待生成的输出。 -- `node 24 / cpu` 使用 64 核 Linux 池运行 6 项 CPU 或依赖关键门禁：类型检查、覆盖率、构建后回放快照，以及两项 Node 24 兼容性冒烟测试。覆盖率和快照各最多使用 16 个工作进程。该通道单独构建，使快照回放使用同一通道生成的输出。覆盖率运行的 fork 数保持低于 32，因为将其设为 32 曾两次导致 Node 24 的 CJS 词法分析器终止一个 Vitest 工作进程，使覆盖率结果失效。 +- `node 24 / cpu` 使用 64 核 Linux 池运行 6 项 CPU 或依赖关键门禁：类型检查、覆盖率、构建后回放快照，以及两项 Node 24 兼容性冒烟测试。覆盖率最多使用 12 个工作进程，快照最多使用 16 个。该通道单独构建，使快照回放使用同一通道生成的输出。覆盖率运行固定使用 12 个 fork，因为 32 个 fork 曾两次导致 Node 24 的 CJS 词法分析器崩溃，而后续一次使用 16 个 fork 的运行又复现了相同的工作进程丢失，并使覆盖率结果无效。 - `node 24 / production site` 使用 16 核 Linux 池运行耗时最长的独立主门禁。这是一项粗粒度拆分，而非分片矩阵：该作业执行一次设置和一次生产版 VitePress 构建。 - Node 22.19 兼容性、Python 3.10 和 Node 26 兼容性分别使用 4 核、8 核和 32 核 Linux 池。不同标签既避免了标准运行器设置耗时的异常值，也避免了两个作业共享一个池时曾出现的第二次分配延迟。 - `windows node 24 / complete` 使用 32 核 Windows 池。一次设置供必需的包构建、必需的生产网站构建以及完整的观测性可移植性清单共用。外层调度器设有 32 个槽位。任何必需项失败都会使作业失败；观测项失败会明确打印为非阻塞，并保留其原有的建议性状态。ESLint 本身仍采用单线程，因为启用 16 个 ESLint 工作线程会把完整 lint 的耗时增加至 174.54 秒；外层门禁并发能够利用该运行器，同时不会成倍增加 Windows 工作线程的启动和 TypeScript 项目加载开销。 @@ -26,6 +26,8 @@ Windows 结构由两项冷路径观测结果确定。首个候选方案使用两 随后两次运行确定了 Linux 的边界。一次[标准运行器验证](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29902209492)中，Node 26 作业耗时 67 秒，而仓库工作仅耗时 5 秒，因为 GitHub 在 `Set up job` 上花费了 36 秒。把各项环境契约迁移到不同的大型运行器池后，这种随机波动随之消失。接下来的[全大型运行器验证](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29902541203)中，96 核主作业耗时 68 秒：仓库工作仍为 26 秒，但设置、缓存、安装和收尾共占用 42 秒。把类型检查、覆盖率以及构建至快照的依赖链迁移到一个粗粒度 64 核通道后，96 核通道的仓库关键路径缩短至 14.81 秒，且无需恢复按门禁分片。 +一次[文档分支头重复验证](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29903735616)表明，覆盖率使用 16 个 fork 时仍未能阻止 CJS 词法分析器崩溃。覆盖率进程在 23.73 秒内完成了余下测试，但已终止的工作进程让一个文件的覆盖率低于阈值，因此该通道按预期失败。最终运行使用 12 个 fork，在 26.06 秒内完成同一门禁，既让 CPU 通道保持在 1 分钟以内，也恢复了进程余量。 + 工作流保留 4 项手动诊断。`suite=larger-runner-benchmark` 比较所有规格下相互独立的关键通道，`suite=consolidated-runner-benchmark` 比较完整聚合流程，`suite=sharded-reference` 保留原生产分片拓扑，`suite=serial-reference` 则继续作为未分片的跨平台完整性判定基准。当拉取请求无法生成合并提交时，`suite=optimized-larger-runners` 会直接针对分支引用运行与生产环境完全相同的拓扑。 首次[涵盖 12 种规格的关键通道基准测试](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29895295659)以标准运行器[基线](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29850033610)为基础，只叠加了一个仅修改工作流的提交，因此代码、锁文件和命令完全相同： @@ -44,19 +46,19 @@ Windows 结构由两项冷路径观测结果确定。首个候选方案使用两 Linux 32 核作业的失败是首次发生的 CJS 词法分析器工作进程崩溃。在所有规格的结果中，96 核聚合作业是唯一成功达到 1 分钟边界的结果。尽管 Windows 仓库工作在超过 16 核后收益很小，32 核池可以让完整的外层清单同时启动；更重要的是，它能从生产环境中消除整整一次付费设置。 -精确的[全运行器池验证运行](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29903067274)在受测分支头通过了所有作业： +精确的[全运行器池验证运行](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29904080103)在受测分支头通过了所有作业： | 生产作业（运行器池） | 活动耗时 | 仓库工作 | 结果 | |---|---:|---:|---:| -| Node 22.19 兼容性（Linux 4） | 26 秒 | 兼容性冒烟测试 | 通过 | -| Python 3.10（Linux 8） | 22 秒 | 完整无密钥 SDK 套件 | 通过 | -| 生产网站（Linux 16） | 39 秒 | VitePress 耗时 22.57 秒 | 通过 | -| Node 26 兼容性（Linux 32） | 24 秒 | 兼容性冒烟测试 | 通过 | -| 主流程 CPU（Linux 64） | 53 秒 | 6 项门禁耗时 23.70 秒 | 通过 | -| 主流程核心（Linux 96） | 47 秒 | 36 项门禁耗时 14.81 秒 | 通过 | -| Windows 完整流程（Windows 32） | 109 秒 | 37 项门禁耗时 32.71 秒 | 通过 | +| Node 22.19 兼容性（Linux 4） | 30 秒 | 兼容性冒烟测试 | 通过 | +| Python 3.10（Linux 8） | 31 秒 | 完整无密钥 SDK 套件 | 通过 | +| 生产网站（Linux 16） | 51 秒 | VitePress 耗时 24.23 秒 | 通过 | +| Node 26 兼容性（Linux 32） | 27 秒 | 兼容性冒烟测试 | 通过 | +| 主流程 CPU（Linux 64） | 56 秒 | 6 项门禁耗时 26.07 秒 | 通过 | +| 主流程核心（Linux 96） | 49 秒 | 36 项门禁耗时 15.58 秒 | 通过 | +| Windows 完整流程（Windows 32） | 102 秒 | 37 项门禁耗时 29.98 秒 | 通过 | -7 个付费作业均在同一秒开始运行。最慢的非 Windows 作业耗时 53 秒。Windows 作业花费 20 秒恢复 pnpm 缓存、15 秒安装依赖，因此其 109 秒的活动耗时既反映托管环境的设置波动，也反映仓库工作耗时。每个非 Windows 作业均低于 1 分钟，唯一的 Windows 作业也低于 3 分钟。 +7 个付费作业均在 1 秒内开始运行。最慢的非 Windows 作业耗时 56 秒。Windows 作业花费 21 秒恢复 pnpm 缓存、14 秒安装依赖，因此其 102 秒的活动耗时既反映托管环境的设置波动，也反映仓库工作耗时。每个非 Windows 作业均低于 1 分钟，唯一的 Windows 作业也低于 3 分钟。 ## 曾考虑的替代方案 From d7e0104a72e937757f1082a44bf240969c4e41eb Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 16:45:59 +0800 Subject: [PATCH 39/49] ci: unpack bubblewrap without package transaction --- .github/workflows/ci.yml | 61 +++++++------------------------- scripts/prepare-ci-bubblewrap.sh | 30 ++++++++++++++++ 2 files changed, 43 insertions(+), 48 deletions(-) create mode 100755 scripts/prepare-ci-bubblewrap.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 80a158210e..9aa20e929b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -277,23 +277,16 @@ jobs: # re-executing their bash calls under a real runner. ubuntu-latest has # no bubblewrap preinstalled and no built Landlock launcher, so without # this the confined executions fail closed (SANDBOX_UNAVAILABLE). The - # install retries after refreshing stale indexes and applies the Ubuntu - # 24.04 AppArmor userns knob. Bubblewrap preparation is independent of - # dependency installation and the build, so it runs beside both. - - name: Install and prepare built snapshot runtime and bubblewrap + # pinned Ubuntu payload is verified and extracted into the ephemeral + # runner instead of paying for a system package transaction. Bubblewrap + # preparation is independent of dependency installation and the build, + # so it runs beside both. + - name: Prepare built snapshot runtime and bubblewrap if: startsWith(matrix.lane, 'snapshot-') run: | pnpm install --frozen-lockfile & install_pid=$! - ( - if ! sudo apt-get install -yq --no-install-recommends bubblewrap; then - echo "initial bubblewrap install failed; refreshing APT indexes and retrying" - sudo apt-get update -q - sudo apt-get install -yq --no-install-recommends bubblewrap - fi - sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 \ - || echo "apparmor userns knob absent — the functional probe decides" - ) & + bash scripts/prepare-ci-bubblewrap.sh & sandbox_pid=$! install_status=0 wait "$install_pid" || install_status=$? @@ -356,19 +349,11 @@ jobs: restore-keys: | ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-full- - - name: Install and prepare bubblewrap + - name: Install dependencies and prepare bubblewrap run: | pnpm install --frozen-lockfile & install_pid=$! - ( - if ! sudo apt-get install -yq --no-install-recommends bubblewrap; then - echo "initial bubblewrap install failed; refreshing APT indexes and retrying" - sudo apt-get update -q - sudo apt-get install -yq --no-install-recommends bubblewrap - fi - sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 \ - || echo "apparmor userns knob absent — the functional probe decides" - ) & + bash scripts/prepare-ci-bubblewrap.sh & sandbox_pid=$! install_status=0 wait "$install_pid" || install_status=$? @@ -473,20 +458,12 @@ jobs: if: matrix.primary_cpu != true run: pnpm install --frozen-lockfile - - name: Install and prepare bubblewrap + - name: Install dependencies and prepare bubblewrap if: matrix.primary_cpu == true run: | pnpm install --frozen-lockfile & install_pid=$! - ( - if ! sudo apt-get install -yq --no-install-recommends bubblewrap; then - echo "initial bubblewrap install failed; refreshing APT indexes and retrying" - sudo apt-get update -q - sudo apt-get install -yq --no-install-recommends bubblewrap - fi - sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 \ - || echo "apparmor userns knob absent — the functional probe decides" - ) & + bash scripts/prepare-ci-bubblewrap.sh & sandbox_pid=$! install_status=0 wait "$install_pid" || install_status=$? @@ -731,12 +708,8 @@ jobs: - name: Install (immutable) run: pnpm install --frozen-lockfile - - name: Install bubblewrap (unrestrict userns) - run: | - sudo apt-get update -q - sudo apt-get install -yq --no-install-recommends bubblewrap - sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 \ - || echo "apparmor userns knob absent — the functional probe decides" + - name: Prepare bubblewrap (unrestrict userns) + run: bash scripts/prepare-ci-bubblewrap.sh - name: Run complete unsharded primary Node CI serially env: @@ -1016,15 +989,7 @@ jobs: run: | pnpm install --frozen-lockfile & install_pid=$! - ( - if ! sudo apt-get install -yq --no-install-recommends bubblewrap; then - echo "initial bubblewrap install failed; refreshing APT indexes and retrying" - sudo apt-get update -q - sudo apt-get install -yq --no-install-recommends bubblewrap - fi - sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 \ - || echo "apparmor userns knob absent — the functional probe decides" - ) & + bash scripts/prepare-ci-bubblewrap.sh & sandbox_pid=$! install_status=0 wait "$install_pid" || install_status=$? diff --git a/scripts/prepare-ci-bubblewrap.sh b/scripts/prepare-ci-bubblewrap.sh new file mode 100755 index 0000000000..e42ee8cd2c --- /dev/null +++ b/scripts/prepare-ci-bubblewrap.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Ubuntu's package transaction scans the hosted image's full dpkg database and +# runs post-install hooks. CI needs only the signed-archive payload, so pin and +# verify that payload before extracting it into the ephemeral runner directory. +readonly BUBBLEWRAP_VERSION='0.9.0-1ubuntu0.1' +readonly BUBBLEWRAP_SHA256='1b506492bd9c7fd0cdb4f02ac822f1d3e336b0aead5113c1239baf8db5db562a' +readonly BUBBLEWRAP_URL="https://archive.ubuntu.com/ubuntu/pool/main/b/bubblewrap/bubblewrap_${BUBBLEWRAP_VERSION}_amd64.deb" + +: "${RUNNER_TEMP:?prepare-ci-bubblewrap requires RUNNER_TEMP}" +: "${GITHUB_PATH:?prepare-ci-bubblewrap requires GITHUB_PATH}" + +if [[ "$(uname -s)" != 'Linux' || "$(uname -m)" != 'x86_64' ]]; then + echo 'prepare-ci-bubblewrap supports only Linux x86_64 hosted runners' >&2 + exit 1 +fi + +archive="${RUNNER_TEMP}/bubblewrap_${BUBBLEWRAP_VERSION}_amd64.deb" +root="${RUNNER_TEMP}/dsh-bubblewrap" + +curl --fail --silent --show-error --location --retry 3 --output "$archive" "$BUBBLEWRAP_URL" +printf '%s %s\n' "$BUBBLEWRAP_SHA256" "$archive" | sha256sum --check --status +mkdir -p "$root" +dpkg-deb --extract "$archive" "$root" +printf '%s\n' "$root/usr/bin" >> "$GITHUB_PATH" + +sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 \ + || echo 'apparmor userns knob absent — the functional probe decides' +"$root/usr/bin/bwrap" --version From 11236c46fbe9a1b93cdafd01ff4372a941dea349 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 16:55:39 +0800 Subject: [PATCH 40/49] docs: record final larger-runner evidence --- ...ence-based-larger-hosted-runners.i18n.yaml | 4 ++-- ...22-evidence-based-larger-hosted-runners.md | 22 ++++++++++--------- ...evidence-based-larger-hosted-runners.zh.md | 22 ++++++++++--------- scripts/prepare-ci-bubblewrap.sh | 2 ++ 4 files changed, 28 insertions(+), 22 deletions(-) diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml index 509cff58e2..c036f6996e 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -2026-07-22-evidence-based-larger-hosted-runners.md: 81de62fa15623b96ac4913ee6c6fc15e886222b0 -2026-07-22-evidence-based-larger-hosted-runners.zh.md: 4e31937284b221d92552957ecc0434d9fc2e2914 +2026-07-22-evidence-based-larger-hosted-runners.md: 30a8b0517690d70ad90f8042a3f1ba841cace76e +2026-07-22-evidence-based-larger-hosted-runners.zh.md: 1335fc66bcbf24cdda3508210dd92786b52b9fff diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md index 81de62fa15..30a8b05176 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md @@ -28,6 +28,8 @@ Two later runs set the Linux boundaries. A [standard-runner validation](https:// A [documentation-head repeat](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29903735616) showed that 16 coverage forks still admitted the CJS-lexer crash. The coverage process completed its remaining tests in 23.73 seconds, but the dead worker left one file below threshold and correctly failed the lane. Twelve forks completed the same gate in 26.06 seconds in the final run, keeping the CPU lane below one minute while restoring process headroom. +One later exact-head run exposed a host-image tax rather than a repository bottleneck: the CPU lane completed its six gates in 28 seconds but took 66 seconds overall because registering the 50 KB Bubblewrap package scanned the runner's 202,507-file package database and consumed 18 seconds. `scripts/prepare-ci-bubblewrap.sh` instead downloads the pinned Ubuntu 24.04 package payload, verifies its archive checksum, extracts it into the ephemeral runner directory, and runs the same functional confinement probe used by the provider. Dependency installation and this sub-second preparation still overlap. This preserves the real Bubblewrap coverage without mutating the hosted image or adding another shard. + The workflow retains four manual diagnostics. `suite=larger-runner-benchmark` compares isolated critical lanes across every size, `suite=consolidated-runner-benchmark` compares whole aggregates, `suite=sharded-reference` preserves the former production shard topology, and `suite=serial-reference` remains the unsharded cross-platform completeness oracle. `suite=optimized-larger-runners` runs the exact production topology against a branch ref when a pull request cannot form a merge commit. The first [twelve-size critical-lane benchmark](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29895295659) used a workflow-only commit on top of the standard-runner [baseline](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29850033610), so the code, lockfile, and commands were identical: @@ -46,19 +48,19 @@ Those isolated results showed that setup dominated but did not identify the prod The Linux 32-core failure was the first CJS-lexer worker crash. The 96-core aggregate was the only successful all-size result at the one-minute boundary. Although Windows repository work gained little above 16 cores, the 32-core pool can start the complete outer inventory together and, more importantly, removes an entire paid setup from production. -The exact [all-pool validation run](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29904080103) passed every job at the tested branch head: +The exact [all-pool validation run](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29905362252) passed every job at the tested branch head: | Production job (pool) | Active time | Repository work | Result | |---|---:|---:|---:| -| Node 22.19 compatibility (Linux 4) | 30 s | compatibility smokes | passed | -| Python 3.10 (Linux 8) | 31 s | complete keyless SDK suite | passed | -| Production site (Linux 16) | 51 s | VitePress in 24.23 s | passed | -| Node 26 compatibility (Linux 32) | 27 s | compatibility smokes | passed | -| Primary CPU (Linux 64) | 56 s | 6 gates in 26.07 s | passed | -| Primary core (Linux 96) | 49 s | 36 gates in 15.58 s | passed | -| Windows complete (Windows 32) | 102 s | 37 gates in 29.98 s | passed | +| Node 22.19 compatibility (Linux 4) | 26 s | compatibility smokes | passed | +| Python 3.10 (Linux 8) | 23 s | complete keyless SDK suite | passed | +| Production site (Linux 16) | 48 s | VitePress in 25 s | passed | +| Node 26 compatibility (Linux 32) | 28 s | compatibility smokes | passed | +| Primary CPU (Linux 64) | 46 s | 6 gates in 25.74 s | passed | +| Primary core (Linux 96) | 42 s | 36 gates in 15.13 s | passed | +| Windows complete (Windows 32) | 137 s | 37 gates in 37.74 s | passed | -All seven paid jobs began within one second. The slowest non-Windows job finished in 56 seconds. The Windows job spent 21 seconds restoring its pnpm cache and 14 seconds installing dependencies, so its 102-second active time measures hosted setup variance as well as repository work. Every non-Windows job stays below one minute and the sole Windows job stays below three minutes. +All seven paid jobs began within one second. The slowest non-Windows job finished in 48 seconds. The Windows job spent 25 seconds checking out, 22 seconds enabling Developer Mode, 19 seconds restoring its pnpm cache, and 16 seconds installing dependencies, so its 137-second active time measures hosted setup variance as well as repository work. Every non-Windows job stays below one minute and the sole Windows job stays below three minutes. ## Alternatives considered @@ -80,7 +82,7 @@ All seven paid jobs began within one second. The slowest non-Windows job finishe ## Consequences -The all-pool validation consumed one billed minute at each Linux size and two billed 32-core Windows minutes. At the configured larger-runner rates, its larger-runner cost was $0.896. The all-size critical benchmark cost $2.936. GitHub rounds each larger-runner job up to a whole minute, so reducing paid job count from 49 to seven matters as much as shortening repository work. +The all-pool validation consumed one billed minute at each Linux size and three billed 32-core Windows minutes. At the configured larger-runner rates, its larger-runner cost was $1.058. A Windows run below 120 seconds costs $0.896 instead; both shapes remain inside the three-minute target. The all-size critical benchmark cost $2.936. GitHub rounds each larger-runner job up to a whole minute, so reducing paid job count from 49 to seven matters as much as shortening repository work. The existing zero-dollar Actions budget did not block larger-runner jobs. The repo-only runner group, bounded workflow topology, manual benchmark triggers, and job timeouts are the observed cost controls; the budget is not treated as an execution guard. diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md index 4e31937284..1335fc66bc 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md @@ -28,6 +28,8 @@ Windows 结构由两项冷路径观测结果确定。首个候选方案使用两 一次[文档分支头重复验证](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29903735616)表明，覆盖率使用 16 个 fork 时仍未能阻止 CJS 词法分析器崩溃。覆盖率进程在 23.73 秒内完成了余下测试，但已终止的工作进程让一个文件的覆盖率低于阈值，因此该通道按预期失败。最终运行使用 12 个 fork，在 26.06 秒内完成同一门禁，既让 CPU 通道保持在 1 分钟以内，也恢复了进程余量。 +后来一次分支头精确运行暴露的是主机映像的额外开销，而非仓库瓶颈：CPU 通道在 28 秒内完成了 6 项门禁，总耗时却达到 66 秒，因为注册 50 KB 的 Bubblewrap 包时扫描了运行器中记录 202,507 个文件的包数据库，耗时 18 秒。`scripts/prepare-ci-bubblewrap.sh` 改为下载固定版本的 Ubuntu 24.04 包内容，验证归档文件的校验和，将其解压到临时运行器目录，并执行与提供方相同的功能性隔离探针。依赖安装仍与这项不足 1 秒的准备工作重叠执行。这样既保留了对真实 Bubblewrap 的覆盖率，又无需修改托管映像或增加分片。 + 工作流保留 4 项手动诊断。`suite=larger-runner-benchmark` 比较所有规格下相互独立的关键通道，`suite=consolidated-runner-benchmark` 比较完整聚合流程，`suite=sharded-reference` 保留原生产分片拓扑，`suite=serial-reference` 则继续作为未分片的跨平台完整性判定基准。当拉取请求无法生成合并提交时，`suite=optimized-larger-runners` 会直接针对分支引用运行与生产环境完全相同的拓扑。 首次[涵盖 12 种规格的关键通道基准测试](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29895295659)以标准运行器[基线](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29850033610)为基础，只叠加了一个仅修改工作流的提交，因此代码、锁文件和命令完全相同： @@ -46,19 +48,19 @@ Windows 结构由两项冷路径观测结果确定。首个候选方案使用两 Linux 32 核作业的失败是首次发生的 CJS 词法分析器工作进程崩溃。在所有规格的结果中，96 核聚合作业是唯一成功达到 1 分钟边界的结果。尽管 Windows 仓库工作在超过 16 核后收益很小，32 核池可以让完整的外层清单同时启动；更重要的是，它能从生产环境中消除整整一次付费设置。 -精确的[全运行器池验证运行](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29904080103)在受测分支头通过了所有作业： +精确的[全运行器池验证运行](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29905362252)在受测分支头通过了所有作业： | 生产作业（运行器池） | 活动耗时 | 仓库工作 | 结果 | |---|---:|---:|---:| -| Node 22.19 兼容性（Linux 4） | 30 秒 | 兼容性冒烟测试 | 通过 | -| Python 3.10（Linux 8） | 31 秒 | 完整无密钥 SDK 套件 | 通过 | -| 生产网站（Linux 16） | 51 秒 | VitePress 耗时 24.23 秒 | 通过 | -| Node 26 兼容性（Linux 32） | 27 秒 | 兼容性冒烟测试 | 通过 | -| 主流程 CPU（Linux 64） | 56 秒 | 6 项门禁耗时 26.07 秒 | 通过 | -| 主流程核心（Linux 96） | 49 秒 | 36 项门禁耗时 15.58 秒 | 通过 | -| Windows 完整流程（Windows 32） | 102 秒 | 37 项门禁耗时 29.98 秒 | 通过 | +| Node 22.19 兼容性（Linux 4） | 26 秒 | 兼容性冒烟测试 | 通过 | +| Python 3.10（Linux 8） | 23 秒 | 完整无密钥 SDK 套件 | 通过 | +| 生产网站（Linux 16） | 48 秒 | VitePress 耗时 25 秒 | 通过 | +| Node 26 兼容性（Linux 32） | 28 秒 | 兼容性冒烟测试 | 通过 | +| 主流程 CPU（Linux 64） | 46 秒 | 6 项门禁耗时 25.74 秒 | 通过 | +| 主流程核心（Linux 96） | 42 秒 | 36 项门禁耗时 15.13 秒 | 通过 | +| Windows 完整流程（Windows 32） | 137 秒 | 37 项门禁耗时 37.74 秒 | 通过 | -7 个付费作业均在 1 秒内开始运行。最慢的非 Windows 作业耗时 56 秒。Windows 作业花费 21 秒恢复 pnpm 缓存、14 秒安装依赖，因此其 102 秒的活动耗时既反映托管环境的设置波动，也反映仓库工作耗时。每个非 Windows 作业均低于 1 分钟，唯一的 Windows 作业也低于 3 分钟。 +7 个付费作业均在 1 秒内开始运行。最慢的非 Windows 作业耗时 48 秒。Windows 作业花费 25 秒检出代码、22 秒启用开发人员模式、19 秒恢复 pnpm 缓存、16 秒安装依赖，因此其 137 秒的活动耗时既反映托管环境的设置波动，也反映仓库工作耗时。每个非 Windows 作业均低于 1 分钟，唯一的 Windows 作业也低于 3 分钟。 ## 曾考虑的替代方案 @@ -80,7 +82,7 @@ Linux 32 核作业的失败是首次发生的 CJS 词法分析器工作进程崩 ## 后果 -全运行器池验证中，每种 Linux 规格各计费 1 分钟，32 核 Windows 计费 2 分钟。按已配置的大型运行器费率计算，其大型运行器成本为 $0.896。全规格关键通道基准测试的成本为 $2.936。GitHub 会把每个大型运行器作业向上取整到整分钟计费，因此把付费作业数从 49 个减少到 7 个，与缩短仓库工作耗时同样重要。 +全运行器池验证中，每种 Linux 规格各计费 1 分钟，32 核 Windows 计费 3 分钟。按已配置的大型运行器费率计算，其大型运行器成本为 $1.058；如果 Windows 运行低于 120 秒，成本则为 $0.896。两种运行形态都保持在 3 分钟目标以内。全规格关键通道基准测试的成本为 $2.936。GitHub 会把每个大型运行器作业向上取整到整分钟计费，因此把付费作业数从 49 个减少到 7 个，与缩短仓库工作耗时同样重要。 现有的零美元 Actions 预算并未阻止大型运行器作业。仅限本仓库的运行器组、有界的工作流拓扑、手动基准测试触发和作业超时限制才是经实测有效的成本控制机制；该预算不被视为执行防护措施。 diff --git a/scripts/prepare-ci-bubblewrap.sh b/scripts/prepare-ci-bubblewrap.sh index e42ee8cd2c..00a513db8f 100755 --- a/scripts/prepare-ci-bubblewrap.sh +++ b/scripts/prepare-ci-bubblewrap.sh @@ -28,3 +28,5 @@ printf '%s\n' "$root/usr/bin" >> "$GITHUB_PATH" sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 \ || echo 'apparmor userns knob absent — the functional probe decides' "$root/usr/bin/bwrap" --version +"$root/usr/bin/bwrap" --ro-bind / / --dev /dev --proc /proc --die-with-parent -- true +echo 'bubblewrap functional probe passed' From 41609c4ce42c90d932f3995521bc5ae2d20cfccd Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 17:53:01 +0800 Subject: [PATCH 41/49] ci: consolidate primary checks on one larger runner --- .../2026-07-06-parallel-github-ci-gates.md | 10 +- ...ence-based-larger-hosted-runners.i18n.yaml | 4 +- ...22-evidence-based-larger-hosted-runners.md | 82 ++- ...evidence-based-larger-hosted-runners.zh.md | 82 ++- .github/workflows/ci.yml | 504 +----------------- examples/acp-agent/tests/acp.snapshot.ts | 4 - .../tests/snapshot-scenario-shard.spec.ts | 20 - .../tests/snapshot-scenario-shard.ts | 18 - package.json | 2 - packages/support/acp-snapshot/README.md | 2 +- packages/support/acp-snapshot/src/index.ts | 1 - .../acp-snapshot/src/scenario-shard.ts | 36 -- packages/support/acp-snapshot/src/suite.ts | 14 +- .../acp-snapshot/tests/scenario-shard.spec.ts | 25 - .../support/acp-snapshot/tests/suite.spec.ts | 22 - scripts/coverage-shards.spec.ts | 46 -- scripts/coverage-shards.ts | 114 ---- scripts/lint-shards.spec.ts | 81 --- scripts/lint-shards.ts | 56 -- scripts/run-gates.ts | 94 +--- scripts/snapshot-shards.spec.ts | 54 -- scripts/snapshot-shards.ts | 49 -- scripts/static-shards.spec.ts | 26 - scripts/static-shards.ts | 86 --- 24 files changed, 102 insertions(+), 1330 deletions(-) delete mode 100644 examples/acp-agent/tests/snapshot-scenario-shard.spec.ts delete mode 100644 examples/acp-agent/tests/snapshot-scenario-shard.ts delete mode 100644 packages/support/acp-snapshot/src/scenario-shard.ts delete mode 100644 packages/support/acp-snapshot/tests/scenario-shard.spec.ts delete mode 100644 scripts/coverage-shards.spec.ts delete mode 100644 scripts/coverage-shards.ts delete mode 100644 scripts/lint-shards.spec.ts delete mode 100644 scripts/lint-shards.ts delete mode 100644 scripts/snapshot-shards.spec.ts delete mode 100644 scripts/snapshot-shards.ts delete mode 100644 scripts/static-shards.spec.ts delete mode 100644 scripts/static-shards.ts diff --git a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md index c7904ae378..fef5852153 100644 --- a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md +++ b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md @@ -12,13 +12,13 @@ The artifact boundary remains load-bearing. `publint`, `verify-node-next-types`, ## Decision -The production topology below was superseded by [Evidence-based larger hosted runners](2026-07-22-evidence-based-larger-hosted-runners.md). Its shard selectors and former job layout remain available only through manual `suite=sharded-reference`; this note preserves why that earlier topology was implemented. +The production topology below is historical and is superseded by [Evidence-based larger hosted runners](2026-07-22-evidence-based-larger-hosted-runners.md). The larger-runner decision removes its shard selectors and workflow jobs; this note preserves why that earlier topology was implemented. [CI](../../../../.github/workflows/ci.yml) treats one minute for non-Windows jobs and three minutes for Windows jobs as observed performance targets, not cancellation deadlines. Hosted-runner variance should leave complete timing evidence and useful failure logs instead of cancelling an otherwise-correct gate. The [serial cross-platform CI reference](2026-07-21-serial-cross-platform-ci-reference.md) independently runs the complete unsharded primary Node aggregate on Linux, macOS, and Windows so the optimized lane inventory is not its own completeness oracle. -[scripts/run-gates.ts](../../../../scripts/run-gates.ts) remains the common bounded scheduler, but GitHub supplies explicit shard names for the expensive gate families. [scripts/static-shards.ts](../../../../scripts/static-shards.ts) partitions static gates into foundation, documentation-type, API-contract, catalog, prose, documentation-projection, and documentation-build ownership and rejects a missing or duplicate gate assignment; low-cost ownership groups may share one hosted lane to stay below the runner-provisioning concurrency wave. Linux lint uses disjoint A-C, D-M, N-S, and T-Z package-source and package-test lanes, while the longer Windows target uses one complete package-source lane and one complete package-test lane; both topologies include a repository complement that still starts from `.` so a new top-level lint target cannot disappear between shards and owns the single cross-file duplication run. [scripts/coverage-shards.ts](../../../../scripts/coverage-shards.ts) assigns every workspace package to exactly one source-coverage lane; its test expands the live package tree, so a new package makes CI red until it has an owner. Directory filters retain a trailing separator because Vitest positional filters match substrings and would otherwise admit prefix-named sibling packages. Each coverage lane includes only its owned source files, repeats the exhaustive companion topology test, and runs without a preceding build because the complete coverage suite passes from a tree with every generated `lib/` removed. +In that topology, [scripts/run-gates.ts](../../../../scripts/run-gates.ts) was the common bounded scheduler and GitHub supplied explicit shard names for the expensive gate families. `scripts/static-shards.ts` partitioned static gates into foundation, documentation-type, API-contract, catalog, prose, documentation-projection, and documentation-build ownership and rejected a missing or duplicate gate assignment. Linux lint used disjoint A-C, D-M, N-S, and T-Z package-source and package-test lanes, while Windows used complete package-source and package-test lanes; both included a repository complement starting from `.` so new top-level targets could not disappear between shards and owned the single cross-file duplication run. `scripts/coverage-shards.ts` assigned every workspace package to exactly one source-coverage lane. Directory filters retained a trailing separator because Vitest positional filters match substrings and would otherwise admit prefix-named siblings. Each coverage lane included only its owned source files, repeated the exhaustive companion topology test, and ran without a preceding build because the complete coverage suite passes from a tree with every generated `lib/` removed. -Snapshot replay uses two explicit multi-file lanes and eight scenario partitions of the large ACP file. [scripts/snapshot-shards.ts](../../../../scripts/snapshot-shards.ts) owns that inventory, and its test discovers every file admitted by the snapshot config so a new file cannot land outside CI. Each snapshot job installs dependencies while its Linux runner installs bubblewrap, refreshing APT indexes and retrying if the hosted image's initial package install fails, then builds the shipped runtime and runs only its assigned replay surface. CI explicitly retains the suite's bounded concurrency of five subprocesses instead of clamping it to the runner's two logical CPUs, because replay spends most of its time waiting on child protocol I/O. Fixture guards still inspect the complete ACP scenario table in every partition. Static, lint, coverage, and snapshot sharding changes only GitHub scheduling: the ordinary local package scripts still run their complete suites. +Snapshot replay used two explicit multi-file lanes and eight scenario partitions of the large ACP file. `scripts/snapshot-shards.ts` owned that inventory, and its test discovered every file admitted by the snapshot config. Each snapshot job installed dependencies while its Linux runner prepared Bubblewrap, built the shipped runtime, and ran only its assigned replay surface. The suite retained bounded concurrency of five subprocesses because replay spent most of its time waiting on child protocol I/O. Fixture guards still inspected the complete ACP scenario table in every partition. Cold standalone documentation typechecking rebuilds the complete project-reference graph, so a dedicated documentation-type lane builds once and checks Markdown blocks against those declarations. The Linux documentation lane uses VitePress's MPA build to retain page rendering and dead-link validation within the observed non-Windows target; separate blocking Windows build and production-site lanes preserve the emitted-package and shipped-site checks without putting both critical paths in one job. @@ -41,9 +41,7 @@ The workflow caches the pnpm store, keys each immutable ESLint cache to its owni ## Consequences -The Actions UI contains more matrix checks and total runner time can exceed a serial workflow, but PR wall time is the slowest bounded lane instead of the sum of unrelated work. Repeated setup and builds are the deliberate price of sub-minute non-Windows feedback and sub-three-minute Windows feedback. - -Shard inventories are repository contracts. Static selection validates the complete live gate list at runtime, coverage tests validate exhaustive package ownership, and Vitest owns deterministic file sharding for snapshots and built-bin smokes. Adding a gate or package therefore requires an explicit scheduling decision instead of silently lengthening an existing lane. +The shard inventories and matrix jobs described above are not part of the current repository contract. The superseding larger-runner decision keeps the complete primary inventory in one process and uses the serial suite as its independent completeness oracle. The optimized publication validators rely on the manifest `files` contract enforced by `verify-package-invariants`. If publication rules grow beyond that contract, the structural gate and both staged views must change together. diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml index c036f6996e..d2114f1bd0 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -2026-07-22-evidence-based-larger-hosted-runners.md: 30a8b0517690d70ad90f8042a3f1ba841cace76e -2026-07-22-evidence-based-larger-hosted-runners.zh.md: 1335fc66bcbf24cdda3508210dd92786b52b9fff +2026-07-22-evidence-based-larger-hosted-runners.md: bb10a53058a7ae1a46d493933868887bb2f55265 +2026-07-22-evidence-based-larger-hosted-runners.zh.md: ffaf958560dae6e044da58f910c0f53e21789344 diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md index 30a8b05176..bb10a53058 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md @@ -14,76 +14,58 @@ Larger runners make it possible to pay setup once and parallelize inside the rep The organization keeps twelve x64 larger-runner pools in the repo-restricted `dsh-larger-ci` group: Ubuntu 24.04 and Windows 2025 at 4, 8, 16, 32, 64, and 96 cores. Public IPs are disabled. Each pool has an autoscaling ceiling of 256; the ceiling does not allocate idle machines or remove the need to bound workflow demand. -Production CI assigns each of the six Linux pool sizes exactly once, assigns one 32-core Windows pool, and keeps only the final aggregator on a standard runner. The version and language jobs are environment contracts rather than slices of one gate inventory; the primary Node work has three coarse lanes instead of a gate-level shard matrix: +Production CI uses five larger-runner executions and one standard-runner aggregator. The primary Node inventory is not sharded: -- `node 24 / core` uses the 96-core Linux pool. One checkout, setup, cache restore, and install feeds 36 unsharded static, lint, documentation, hygiene, build, and artifact gates. `run-gates` starts up to 32 independent gates and ESLint uses 32 workers. Build starts eagerly; its artifact consumers still wait for emitted output. -- `node 24 / cpu` uses the 64-core Linux pool for six CPU- or dependency-critical gates: typecheck, coverage, build followed by snapshot replay, and two Node 24 compatibility smokes. Coverage uses at most 12 workers and snapshot uses at most 16. This lane builds separately so snapshot replay consumes same-lane output. Coverage stays at 12 forks because 32 forks crashed Node 24's CJS lexer twice and a later 16-fork run reproduced the same worker loss and invalid coverage. -- `node 24 / production site` uses the 16-core Linux pool for the longest independent primary gate. This is one coarse split, not a shard matrix: the job performs one setup and one production VitePress build. -- Node 22.19 compatibility, Python 3.10, and Node 26 compatibility use the 4-, 8-, and 32-core Linux pools respectively. Distinct labels avoid both standard-runner setup outliers and the delayed second allocation observed when two jobs shared one pool. -- `windows node 24 / complete` uses the 32-core Windows pool. One setup feeds the required package build, the required production site build, and the complete observational portability inventory. The outer scheduler has 32 slots. Required failures fail the job; observational failures are printed as non-blocking and preserve their former advisory status. ESLint itself stays single-threaded because 16 ESLint worker threads increased full-lint time to 174.54 seconds; outer gate concurrency uses the runner without multiplying Windows worker startup and TypeScript project loading. +- `node 24 / complete` uses one 96-core Linux runner. One checkout, setup, cache restore, and install feeds all 42 primary gates. `run-gates` starts up to 32 independent gates; ESLint uses 32 workers, coverage uses at most 12, and snapshot replay uses at most 16. Build starts immediately beside typecheck, coverage, lint, and documentation work, while snapshot replay and publication consumers retain explicit dependencies on emitted `lib/` output. +- Node 22.19 and Node 26 use the 4- and 32-core Linux pools for their runtime compatibility smokes. Python 3.10 uses the 8-core Linux pool for the complete keyless SDK suite. These are environment contracts, not slices of the primary Node gate inventory. +- `windows node 24 / complete` uses one 32-core Windows runner. One setup feeds the required package build, required production site build, and complete observational portability inventory. Required failures fail the job; observational failures are reported as non-blocking. ESLint stays single-threaded because 16 ESLint workers took 174.54 seconds, while the outer scheduler retains 32 slots. -The Windows shape followed two cold-path observations. A first candidate used two 16-core Windows jobs, and GitHub took 93 seconds to provision the second same-label runner despite the configured autoscaling ceiling. A later [documentation-head validation](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29900502413) took 266 seconds on a separate Windows blocking job after spending 138 seconds restoring a 153 MB pnpm cache. Combining all Windows work on one 32-core box removed the duplicate setup wave. +The former gate-level and coarse primary shard jobs are absent from the workflow. Their static, lint, coverage, snapshot, and scenario shard selectors are also absent from the repository, so an unused diagnostic path cannot preserve a second CI architecture. -Two later runs set the Linux boundaries. A [standard-runner validation](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29902209492) took 67 seconds for Node 26 even though repository work took five seconds, because GitHub spent 36 seconds in `Set up job`. Moving the environment contracts to distinct larger pools removed that lottery. The next [all-larger-runner validation](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29902541203) took 68 seconds on the 96-core primary job: repository work remained 26 seconds, but setup, cache, install, and finalization consumed 42 seconds. Moving typecheck, coverage, and the build-to-snapshot dependency chain to one coarse 64-core lane reduced the 96-core lane's repository critical path to 14.81 seconds without returning to per-gate shards. +An [exact-head all-size benchmark](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29908491351) ran the complete unsharded primary Node aggregate on every Linux pool before the eager-build correction: -A [documentation-head repeat](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29903735616) showed that 16 coverage forks still admitted the CJS-lexer crash. The coverage process completed its remaining tests in 23.73 seconds, but the dead worker left one file below threshold and correctly failed the lane. Twelve forks completed the same gate in 26.06 seconds in the final run, keeping the CPU lane below one minute while restoring process headroom. - -One later exact-head run exposed a host-image tax rather than a repository bottleneck: the CPU lane completed its six gates in 28 seconds but took 66 seconds overall because registering the 50 KB Bubblewrap package scanned the runner's 202,507-file package database and consumed 18 seconds. `scripts/prepare-ci-bubblewrap.sh` instead downloads the pinned Ubuntu 24.04 package payload, verifies its archive checksum, extracts it into the ephemeral runner directory, and runs the same functional confinement probe used by the provider. Dependency installation and this sub-second preparation still overlap. This preserves the real Bubblewrap coverage without mutating the hosted image or adding another shard. - -The workflow retains four manual diagnostics. `suite=larger-runner-benchmark` compares isolated critical lanes across every size, `suite=consolidated-runner-benchmark` compares whole aggregates, `suite=sharded-reference` preserves the former production shard topology, and `suite=serial-reference` remains the unsharded cross-platform completeness oracle. `suite=optimized-larger-runners` runs the exact production topology against a branch ref when a pull request cannot form a merge commit. - -The first [twelve-size critical-lane benchmark](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29895295659) used a workflow-only commit on top of the standard-runner [baseline](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29850033610), so the code, lockfile, and commands were identical: - -| Critical job | Standard | 4 cores | 8 cores | 16 cores | 32 cores | 64 cores | 96 cores | -|---|---:|---:|---:|---:|---:|---:|---:| -| Linux typecheck | 56 s | 38 s | 35 s | 40 s | 35 s | 44 s | 40 s | -| Windows production site | 160 s | 117 s | 103 s | 113 s | 75 s | 105 s | 108 s | - -Those isolated results showed that setup dominated but did not identify the production size. A [whole-aggregate benchmark without native ESLint concurrency](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29897826082) found a 69-second single-threaded Linux lint gate. After enabling native Linux ESLint concurrency, the [second whole-aggregate benchmark](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29898331705) produced these active job times: - -| Job | 4 cores | 8 cores | 16 cores | 32 cores | 64 cores | 96 cores | +| Complete Linux primary | 4 cores | 8 cores | 16 cores | 32 cores | 64 cores | 96 cores | |---|---:|---:|---:|---:|---:|---:| -| Linux complete primary | 147 s | 104 s | 95 s | failed at 57 s | 66 s | 60 s | -| Windows blocking builds | 137 s | 127 s | 113 s | 107 s | 105 s | 131 s | +| Active time | 243 s | 144 s | 103 s | 87 s | 62 s | 65 s | -The Linux 32-core failure was the first CJS-lexer worker crash. The 96-core aggregate was the only successful all-size result at the one-minute boundary. Although Windows repository work gained little above 16 cores, the 32-core pool can start the complete outer inventory together and, more importantly, removes an entire paid setup from production. +The 96-core trace spent 39.14 seconds in repository gates. Typecheck occupied 25.71 seconds, then a scheduler dependency delayed the 2.13-second build and 11.29-second snapshot replay until it finished. The same run already proved build and typecheck independently, and the former CPU lane ran them concurrently. Removing that dependency makes lint at 33.30 seconds the measured critical gate while preserving dependencies only for consumers of build output. The 64-core trace exposed the same idle chain: typecheck, build, and snapshot consumed 44.85 seconds in sequence while its independent lint and documentation builds finished in 36.83 and 36.15 seconds. More cores therefore become useful only after the repository scheduler can feed them. -The exact [all-pool validation run](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29905362252) passed every job at the tested branch head: +The same benchmark measured the required Windows build surfaces across every provisioned size: -| Production job (pool) | Active time | Repository work | Result | -|---|---:|---:|---:| -| Node 22.19 compatibility (Linux 4) | 26 s | compatibility smokes | passed | -| Python 3.10 (Linux 8) | 23 s | complete keyless SDK suite | passed | -| Production site (Linux 16) | 48 s | VitePress in 25 s | passed | -| Node 26 compatibility (Linux 32) | 28 s | compatibility smokes | passed | -| Primary CPU (Linux 64) | 46 s | 6 gates in 25.74 s | passed | -| Primary core (Linux 96) | 42 s | 36 gates in 15.13 s | passed | -| Windows complete (Windows 32) | 137 s | 37 gates in 37.74 s | passed | +| Windows blocking builds | 4 cores | 8 cores | 16 cores | 32 cores | 64 cores | 96 cores | +|---|---:|---:|---:|---:|---:|---:| +| Active time | 152 s | 104 s | 104 s | 92 s | 103 s | 110 s | -All seven paid jobs began within one second. The slowest non-Windows job finished in 48 seconds. The Windows job spent 25 seconds checking out, 22 seconds enabling Developer Mode, 19 seconds restoring its pnpm cache, and 16 seconds installing dependencies, so its 137-second active time measures hosted setup variance as well as repository work. Every non-Windows job stays below one minute and the sole Windows job stays below three minutes. +Repository work gains little above 16 Windows cores, but the 32-core pool can start the complete outer inventory together. A [retargeted production validation](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29907581119/attempts/2) completed the full one-box Windows inventory in 173 seconds, including coverage and snapshot replay, so Windows remains consolidated. + +Two host effects remain part of the decision. A standard Node 26 job once spent 36 of its 67 seconds in `Set up job`, which is why environment contracts use distinct larger-runner pools instead of standard capacity. A Linux candidate spent 18 seconds registering a 50 KB Bubblewrap package because the hosted image scanned 202,507 package-database files. [`scripts/prepare-ci-bubblewrap.sh`](../../../../scripts/prepare-ci-bubblewrap.sh) instead verifies and extracts the pinned payload into the ephemeral runner directory, runs a functional confinement probe, and overlaps that preparation with dependency installation. + +Coverage remains capped at 12 forks. Thirty-two forks crashed Node 24's CJS lexer twice, and a later 16-fork run reproduced the worker loss and invalid coverage result. Twelve forks preserve process headroom without becoming the single-host critical path. + +The workflow retains three manual measurement suites. `suite=larger-runner-benchmark` compares isolated critical lanes across every size, `suite=consolidated-runner-benchmark` compares whole aggregates, and `suite=serial-reference` remains the unsharded cross-platform completeness oracle. `suite=optimized-larger-runners` dispatches the exact production topology against a branch ref when a pull request cannot form a merge commit. ## Alternatives considered -**Keep the former shard topology in production.** The shards can be fast when provisioned together, but 49 larger-runner jobs repeat setup and create more chances for a cold outlier. The 231-second Windows control demonstrated that a short lint shard does not protect the end-to-end job target. +**Keep the three coarse primary Linux lanes.** The core, CPU, and production-site jobs met the latency targets, but they paid three setup waves and left primary Node work sharded after larger runners were available. The all-size trace showed that one unnecessary dependency, not a lack of host capacity, kept the single-box aggregate above one minute. -**Select a production size from the critical-lane benchmark.** Four cores looked cost-effective for isolated typecheck and site builds, but the full aggregate found repository-wide lint and dependent artifact work that those commands did not represent. +**Keep the former gate-level shard topology as a manual reference.** A dormant second topology kept hundreds of workflow lines, selector modules, and scenario-partition behavior alive. The all-size and serial suites provide timing and completeness controls without preserving production code that no required job exercises. -**Run the production site inside the Linux core aggregate.** This reached 50 seconds with warm hosted setup, then crossed the threshold at 64 seconds when the site gate took 29.05 seconds. One coarse independent site job protects the target without returning to gate-level sharding. +**Use the 64-core pool for the complete primary aggregate.** Its sampled active time was three seconds lower than the 96-core result because hosted setup was nine seconds faster, but its repository gates were 5.72 seconds slower. Production uses 96 cores for the shorter controllable critical path; the benchmark suite retains both pools so a sustained image or pricing change can reverse that choice with evidence. -**Keep every primary gate on the 96-core Linux runner.** Repository work completed in 26 seconds, but a 42-second cold path still pushed the job to 68 seconds. The 64-core CPU lane owns the three longest independent or dependency-critical paths; the remaining 36-gate core inventory completes its repository work in 14.81 seconds. +**Keep build behind typecheck.** This orders independent compiler invocations and turns snapshot replay into a three-stage critical chain. Build output has its own success dependency, so only snapshot and publication consumers wait for it. -**Keep required and observational Windows checks in separate jobs.** The split preserved status semantics at the workflow level but paid setup twice, and a cold cache pushed the required job to 266 seconds. `run-gates` now preserves those semantics inside one process: build and production site are required, while the remaining inventory is explicitly non-blocking. +**Keep compatibility and Python on standard runners.** Warm standard runs can fit, but runner setup alone has crossed the non-Windows target. Distinct larger pools isolate these environment contracts from that allocation lottery. -**Prebuild before starting the Linux aggregate.** This moved build onto the setup path and produced a 66-second candidate. Starting build eagerly inside `run-gates` preserves artifact dependencies while overlapping it with unrelated checks. +**Keep required and observational Windows checks in separate jobs.** The split preserves status semantics at the workflow level but pays setup twice. `run-gates` preserves the same required versus non-blocking distinction inside one process. -**Use native ESLint worker concurrency on Windows.** Sixteen workers made lint more than five times slower than the final single-threaded result. Outer gate parallelism uses the 32-core runner without multiplying ESLint's Windows worker startup and TypeScript project loading. - -**Keep compatibility and Python on standard runners.** Warm runs completed in 43 seconds or less, but one Node 26 job later spent 36 seconds in GitHub setup and crossed the target despite only five seconds of repository work. Distinct larger pools stabilize those environment contracts; the three-second final aggregator remains on a standard runner because it begins only after the paid jobs release capacity. +**Install Bubblewrap through the system package manager.** This uses the host's package database and can dominate the job even when the payload is tiny. Pinned extraction plus a confinement probe preserves the runtime contract without mutating the hosted image. ## Consequences -The all-pool validation consumed one billed minute at each Linux size and three billed 32-core Windows minutes. At the configured larger-runner rates, its larger-runner cost was $1.058. A Windows run below 120 seconds costs $0.896 instead; both shapes remain inside the three-minute target. The all-size critical benchmark cost $2.936. GitHub rounds each larger-runner job up to a whole minute, so reducing paid job count from 49 to seven matters as much as shortening repository work. +Primary Node CI has one job, one setup wave, one complete gate inventory, and no shard selectors. Together with two Node compatibility executions, Python, and Windows, production has five paid larger-runner executions instead of seven coarse-lane executions or 49 gate-level executions. -The existing zero-dollar Actions budget did not block larger-runner jobs. The repo-only runner group, bounded workflow topology, manual benchmark triggers, and job timeouts are the observed cost controls; the budget is not treated as an execution guard. +GitHub rounds each larger-runner execution up to a whole minute, so eliminating setup waves reduces billed time as well as workflow complexity. The final aggregator remains on a standard runner because it begins only after the paid jobs release capacity. -Production CI depends on the organization-owned runner names in this note and in `.github/workflows/ci.yml`. Missing or renamed pools leave jobs queued instead of falling back to standard capacity. Manual all-size, consolidated, former-shard, and serial suites remain available so image, dependency, scheduler, or pricing changes can be remeasured before changing production labels. +The current targets are observed performance contracts, not cancellation deadlines. Exact-head production runs must show every non-Windows job below one minute and the consolidated Windows job below three minutes; manual all-size and serial suites remain available when image, dependency, scheduler, or pricing changes need remeasurement. + +Production CI depends on the organization-owned runner labels in [`.github/workflows/ci.yml`](../../../../.github/workflows/ci.yml). Missing or renamed pools leave jobs queued instead of falling back to standard capacity. All twelve pools remain provisioned so the manual benchmarks can re-evaluate the production size without an administrative setup cycle. diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md index 1335fc66bc..ffaf958560 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md @@ -14,76 +14,58 @@ Status: implemented 组织在仅限本仓库使用的 `dsh-larger-ci` 运行器组中保留 12 个 x64 大型运行器池：Ubuntu 24.04 和 Windows 2025 各设 4、8、16、32、64、96 核规格。公网 IP 已禁用。每个池的自动扩缩容上限为 256；该上限既不会分配闲置机器，也不能免除限制工作流需求的必要性。 -生产 CI 对 6 种 Linux 池规格各使用一次，另使用一个 32 核 Windows 池，只有最终聚合作业继续使用标准运行器。版本和语言作业属于环境契约，并非从同一门禁清单切出的分片；主 Node 工作采用 3 个粗粒度通道，而非门禁级分片矩阵： +生产 CI 包含 5 次大型运行器执行和 1 个标准运行器聚合作业。主 Node 门禁清单不再分片： -- `node 24 / core` 使用 96 核 Linux 池。只需执行一次代码检出、设置、缓存恢复和安装，即可供 36 项未分片的静态、lint、文档、hygiene、构建和产物门禁使用。`run-gates` 最多同时启动 32 项相互独立的门禁，ESLint 使用 32 个工作线程。构建会尽早启动；其产物消费方仍会等待生成的输出。 -- `node 24 / cpu` 使用 64 核 Linux 池运行 6 项 CPU 或依赖关键门禁：类型检查、覆盖率、构建后回放快照，以及两项 Node 24 兼容性冒烟测试。覆盖率最多使用 12 个工作进程，快照最多使用 16 个。该通道单独构建，使快照回放使用同一通道生成的输出。覆盖率运行固定使用 12 个 fork，因为 32 个 fork 曾两次导致 Node 24 的 CJS 词法分析器崩溃，而后续一次使用 16 个 fork 的运行又复现了相同的工作进程丢失，并使覆盖率结果无效。 -- `node 24 / production site` 使用 16 核 Linux 池运行耗时最长的独立主门禁。这是一项粗粒度拆分，而非分片矩阵：该作业执行一次设置和一次生产版 VitePress 构建。 -- Node 22.19 兼容性、Python 3.10 和 Node 26 兼容性分别使用 4 核、8 核和 32 核 Linux 池。不同标签既避免了标准运行器设置耗时的异常值，也避免了两个作业共享一个池时曾出现的第二次分配延迟。 -- `windows node 24 / complete` 使用 32 核 Windows 池。一次设置供必需的包构建、必需的生产网站构建以及完整的观测性可移植性清单共用。外层调度器设有 32 个槽位。任何必需项失败都会使作业失败；观测项失败会明确打印为非阻塞，并保留其原有的建议性状态。ESLint 本身仍采用单线程，因为启用 16 个 ESLint 工作线程会把完整 lint 的耗时增加至 174.54 秒；外层门禁并发能够利用该运行器，同时不会成倍增加 Windows 工作线程的启动和 TypeScript 项目加载开销。 +- `node 24 / complete` 使用一台 96 核 Linux 运行器。只需执行一次代码检出、设置、缓存恢复和安装，即可供全部 42 项主门禁使用。`run-gates` 最多同时启动 32 项相互独立的门禁；ESLint 使用 32 个工作线程，覆盖率最多使用 12 个，快照回放最多使用 16 个。构建会立即与类型检查、覆盖率、lint 和文档工作同时启动，而快照回放和发布消费方仍显式依赖生成的 `lib/` 输出。 +- Node 22.19 和 Node 26 分别使用 4 核和 32 核 Linux 池运行各自的运行时兼容性冒烟测试。Python 3.10 使用 8 核 Linux 池运行完整的无密钥 SDK 套件。这些作业属于环境契约，并非主 Node 门禁清单的分片。 +- `windows node 24 / complete` 使用一台 32 核 Windows 运行器。一次设置供必需的包构建、必需的生产网站构建以及完整的观测性可移植性清单共用。任何必需项失败都会使作业失败；观测项失败则报告为非阻塞。ESLint 保持单线程，因为 16 个 ESLint 工作线程耗时 174.54 秒，外层调度器则保留 32 个槽位。 -Windows 结构由两项冷路径观测结果确定。首个候选方案使用两个 16 核 Windows 作业；尽管已配置自动扩缩容上限，GitHub 仍花费 93 秒才预配好第二台同标签运行器。随后一次[文档分支头验证](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29900502413)中，一个独立的 Windows 阻塞性作业耗时 266 秒，其中恢复 153 MB pnpm 缓存就用了 138 秒。把所有 Windows 工作合并到一台 32 核机器上，消除了这轮重复设置。 +原有的门禁级和粗粒度主流程分片作业已从工作流中移除。相应的静态、lint、覆盖率、快照和场景分片选择器也已从仓库中移除，因此未使用的诊断路径无法继续维系第二套 CI 架构。 -随后两次运行确定了 Linux 的边界。一次[标准运行器验证](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29902209492)中，Node 26 作业耗时 67 秒，而仓库工作仅耗时 5 秒，因为 GitHub 在 `Set up job` 上花费了 36 秒。把各项环境契约迁移到不同的大型运行器池后，这种随机波动随之消失。接下来的[全大型运行器验证](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29902541203)中，96 核主作业耗时 68 秒：仓库工作仍为 26 秒，但设置、缓存、安装和收尾共占用 42 秒。把类型检查、覆盖率以及构建至快照的依赖链迁移到一个粗粒度 64 核通道后，96 核通道的仓库关键路径缩短至 14.81 秒，且无需恢复按门禁分片。 +一次[分支头精确的全规格基准测试](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29908491351)在修正构建尽早启动逻辑前，对每种 Linux 池都运行了完整且未分片的主 Node 聚合流程： -一次[文档分支头重复验证](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29903735616)表明，覆盖率使用 16 个 fork 时仍未能阻止 CJS 词法分析器崩溃。覆盖率进程在 23.73 秒内完成了余下测试，但已终止的工作进程让一个文件的覆盖率低于阈值，因此该通道按预期失败。最终运行使用 12 个 fork，在 26.06 秒内完成同一门禁，既让 CPU 通道保持在 1 分钟以内，也恢复了进程余量。 - -后来一次分支头精确运行暴露的是主机映像的额外开销，而非仓库瓶颈：CPU 通道在 28 秒内完成了 6 项门禁，总耗时却达到 66 秒，因为注册 50 KB 的 Bubblewrap 包时扫描了运行器中记录 202,507 个文件的包数据库，耗时 18 秒。`scripts/prepare-ci-bubblewrap.sh` 改为下载固定版本的 Ubuntu 24.04 包内容，验证归档文件的校验和，将其解压到临时运行器目录，并执行与提供方相同的功能性隔离探针。依赖安装仍与这项不足 1 秒的准备工作重叠执行。这样既保留了对真实 Bubblewrap 的覆盖率，又无需修改托管映像或增加分片。 - -工作流保留 4 项手动诊断。`suite=larger-runner-benchmark` 比较所有规格下相互独立的关键通道，`suite=consolidated-runner-benchmark` 比较完整聚合流程，`suite=sharded-reference` 保留原生产分片拓扑，`suite=serial-reference` 则继续作为未分片的跨平台完整性判定基准。当拉取请求无法生成合并提交时，`suite=optimized-larger-runners` 会直接针对分支引用运行与生产环境完全相同的拓扑。 - -首次[涵盖 12 种规格的关键通道基准测试](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29895295659)以标准运行器[基线](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29850033610)为基础，只叠加了一个仅修改工作流的提交，因此代码、锁文件和命令完全相同： - -| 关键作业 | 标准 | 4 核 | 8 核 | 16 核 | 32 核 | 64 核 | 96 核 | -|---|---:|---:|---:|---:|---:|---:|---:| -| Linux 类型检查 | 56 秒 | 38 秒 | 35 秒 | 40 秒 | 35 秒 | 44 秒 | 40 秒 | -| Windows 生产网站 | 160 秒 | 117 秒 | 103 秒 | 113 秒 | 75 秒 | 105 秒 | 108 秒 | - -这些单项结果表明设置开销占主导地位，却无法确定生产环境应选用的规格。一项[未启用 ESLint 原生并发的完整聚合基准测试](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29897826082)发现，Linux 单线程 lint 门禁耗时 69 秒。启用 Linux ESLint 原生并发后，[第二次完整聚合基准测试](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29898331705)得到了以下作业活动耗时： - -| 作业 | 4 核 | 8 核 | 16 核 | 32 核 | 64 核 | 96 核 | +| Linux 完整主流程 | 4 核 | 8 核 | 16 核 | 32 核 | 64 核 | 96 核 | |---|---:|---:|---:|---:|---:|---:| -| Linux 完整主流程 | 147 秒 | 104 秒 | 95 秒 | 57 秒时失败 | 66 秒 | 60 秒 | -| Windows 阻塞性构建 | 137 秒 | 127 秒 | 113 秒 | 107 秒 | 105 秒 | 131 秒 | +| 活动耗时 | 243 秒 | 144 秒 | 103 秒 | 87 秒 | 62 秒 | 65 秒 | -Linux 32 核作业的失败是首次发生的 CJS 词法分析器工作进程崩溃。在所有规格的结果中，96 核聚合作业是唯一成功达到 1 分钟边界的结果。尽管 Windows 仓库工作在超过 16 核后收益很小，32 核池可以让完整的外层清单同时启动；更重要的是，它能从生产环境中消除整整一次付费设置。 +96 核运行轨迹中的仓库门禁耗时 39.14 秒。类型检查占用 25.71 秒，随后一项调度器依赖使耗时 2.13 秒的构建和耗时 11.29 秒的快照回放都要等到类型检查结束后才启动。同一次运行已经分别证明构建和类型检查可以独立执行，原 CPU 通道也曾让二者并发运行。移除这项依赖后，耗时 33.30 秒的 lint 成为实测关键门禁，而只有构建输出的消费方仍保留依赖关系。64 核运行轨迹暴露了相同的空闲链：类型检查、构建和快照依次执行，共耗时 44.85 秒，而相互独立的 lint 和文档构建分别在 36.83 秒和 36.15 秒内完成。因此，只有仓库调度器能够为更多核心持续提供工作时，增加核心数才有价值。 -精确的[全运行器池验证运行](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29905362252)在受测分支头通过了所有作业： +同一项基准测试还测量了每种已预配规格上的 Windows 必需构建项： -| 生产作业（运行器池） | 活动耗时 | 仓库工作 | 结果 | -|---|---:|---:|---:| -| Node 22.19 兼容性（Linux 4） | 26 秒 | 兼容性冒烟测试 | 通过 | -| Python 3.10（Linux 8） | 23 秒 | 完整无密钥 SDK 套件 | 通过 | -| 生产网站（Linux 16） | 48 秒 | VitePress 耗时 25 秒 | 通过 | -| Node 26 兼容性（Linux 32） | 28 秒 | 兼容性冒烟测试 | 通过 | -| 主流程 CPU（Linux 64） | 46 秒 | 6 项门禁耗时 25.74 秒 | 通过 | -| 主流程核心（Linux 96） | 42 秒 | 36 项门禁耗时 15.13 秒 | 通过 | -| Windows 完整流程（Windows 32） | 137 秒 | 37 项门禁耗时 37.74 秒 | 通过 | +| Windows 阻塞性构建 | 4 核 | 8 核 | 16 核 | 32 核 | 64 核 | 96 核 | +|---|---:|---:|---:|---:|---:|---:| +| 活动耗时 | 152 秒 | 104 秒 | 104 秒 | 92 秒 | 103 秒 | 110 秒 | -7 个付费作业均在 1 秒内开始运行。最慢的非 Windows 作业耗时 48 秒。Windows 作业花费 25 秒检出代码、22 秒启用开发人员模式、19 秒恢复 pnpm 缓存、16 秒安装依赖，因此其 137 秒的活动耗时既反映托管环境的设置波动，也反映仓库工作耗时。每个非 Windows 作业均低于 1 分钟，唯一的 Windows 作业也低于 3 分钟。 +Windows 仓库工作在超过 16 核后收益很小，但 32 核池可以让完整的外层清单同时启动。一次[重新定向的生产验证](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29907581119/attempts/2)在 173 秒内完成了单机 Windows 完整清单，其中包括覆盖率和快照回放，因此 Windows 继续采用合并执行方式。 + +两项主机效应仍构成这项决策的依据。一个标准 Node 26 作业曾在总共 67 秒的耗时中，把 36 秒用在 `Set up job` 上，因此各项环境契约使用不同的大型运行器池，而非标准容量。一个 Linux 候选作业在注册 50 KB 的 Bubblewrap 包时耗时 18 秒，因为托管映像扫描了 202,507 个包数据库文件。[`scripts/prepare-ci-bubblewrap.sh`](../../../../scripts/prepare-ci-bubblewrap.sh) 改为验证固定包内容并将其解压到临时运行器目录，执行功能性隔离探针，并让这项准备工作与依赖安装重叠执行。 + +覆盖率仍限制为 12 个 fork。32 个 fork 曾两次导致 Node 24 的 CJS 词法分析器崩溃，后来一次使用 16 个 fork 的运行又复现了工作进程丢失和无效的覆盖率结果。12 个 fork 能保留进程余量，同时不会成为单机关键路径。 + +工作流保留 3 项手动测量套件。`suite=larger-runner-benchmark` 比较所有规格下相互独立的关键通道，`suite=consolidated-runner-benchmark` 比较完整聚合流程，`suite=serial-reference` 则继续作为未分片的跨平台完整性判定基准。当拉取请求无法生成合并提交时，`suite=optimized-larger-runners` 会直接针对分支引用触发与生产环境完全相同的拓扑。 ## 曾考虑的替代方案 -**在生产环境中保留原分片拓扑。** 各分片在一同完成预配时可以很快，但 49 个大型运行器作业会重复设置，也增加了出现冷启动异常值的机会。耗时 231 秒的 Windows 对照结果表明，短小的 lint 分片并不能保障端到端作业达到时长目标。 +**保留 3 个粗粒度 Linux 主流程通道。** 核心、CPU 和生产网站作业均达到延迟目标，但它们需要 3 轮设置，而且在大型运行器已经可用后仍对主 Node 工作进行分片。全规格运行轨迹表明，让单机聚合流程超过 1 分钟的是一项不必要的依赖，而非主机容量不足。 -**根据关键通道基准测试选择生产规格。** 对单独的类型检查和网站构建而言，4 核看起来具备成本效益，但完整聚合流程发现了这些命令未覆盖的全仓库 lint 和存在依赖关系的产物工作。 +**将原有的门禁级分片拓扑保留为手动参考。** 一套闲置的第二拓扑会让数百行工作流、选择器模块和场景分区行为继续存活。全规格和串行套件无需保留任何必需作业都不执行的生产代码，也能提供计时与完整性对照。 -**在 Linux 核心聚合流程内运行生产网站。** 托管环境设置处于热状态时，此方案用时 50 秒；当网站门禁耗时 29.05 秒时，总耗时增至 64 秒，超过了目标。一项独立的粗粒度网站作业能够保障目标，而无需重新采用门禁级分片。 +**使用 64 核池运行完整主聚合流程。** 由于托管设置快了 9 秒，其采样活动耗时比 96 核结果少 3 秒，但仓库门禁慢了 5.72 秒。生产环境使用 96 核来缩短可控的关键路径；基准测试套件保留两种规格，因此如果映像或定价发生持续性变化，仍可根据证据反转这项选择。 -**将所有主门禁都保留在 96 核 Linux 运行器上。** 仓库工作在 26 秒内完成，但 42 秒的冷路径仍把作业总耗时推高至 68 秒。64 核 CPU 通道负责 3 条耗时最长的独立或依赖关键路径；其余 36 项核心门禁清单的仓库工作耗时为 14.81 秒。 +**让构建继续等待类型检查。** 此方案会给相互独立的编译器调用排定先后顺序，并把快照回放变成 3 阶段关键链。构建输出本身有独立的成功依赖关系，因此只有快照和发布消费方需要等待它。 -**将必需的 Windows 检查和观测性 Windows 检查保留在不同作业中。** 这种拆分在工作流层保留了状态语义，却要支付两次设置开销，而且冷缓存让必需作业的耗时达到 266 秒。`run-gates` 目前在一个进程内保留这些语义：构建和生产网站属于必需项，其余清单则明确为非阻塞。 +**让兼容性和 Python 继续使用标准运行器。** 标准运行器热运行可以达到目标，但仅运行器设置一项就曾超过非 Windows 目标。不同的大型运行器池可以让这些环境契约免受这种分配波动影响。 -**在启动 Linux 聚合流程前预先构建。** 此方案让构建成为设置路径的一部分，并产生了一个耗时 66 秒的候选结果。在 `run-gates` 内尽早启动构建，既能保留产物依赖关系，又能让构建与无关检查重叠执行。 +**将必需的 Windows 检查和观测性 Windows 检查保留在不同作业中。** 这种拆分在工作流层保留状态语义，却需要支付两次设置开销。`run-gates` 在一个进程内保留了相同的必需与非阻塞区别。 -**在 Windows 上使用 ESLint 原生工作线程并发。** 16 个工作线程让 lint 比最终的单线程结果慢 5 倍以上。外层门禁并发能够利用 32 核运行器，同时不会成倍增加 ESLint 在 Windows 上启动工作线程和加载 TypeScript 项目的开销。 - -**让兼容性和 Python 继续使用标准运行器。** 热运行均在 43 秒以内完成，但后来一个 Node 26 作业在 GitHub 设置上花费 36 秒，即使仓库工作仅耗时 5 秒，仍超过了目标。不同的大型运行器池能够稳定这些环境契约；最终聚合作业耗时 3 秒，仍使用标准运行器，因为它只会在付费作业释放容量后启动。 +**通过系统包管理器安装 Bubblewrap。** 此方案会使用主机的包数据库，即使包内容很小，也可能主导整个作业耗时。固定版本的解压方式配合隔离探针，无需修改托管映像即可保留运行时契约。 ## 后果 -全运行器池验证中，每种 Linux 规格各计费 1 分钟，32 核 Windows 计费 3 分钟。按已配置的大型运行器费率计算，其大型运行器成本为 $1.058；如果 Windows 运行低于 120 秒，成本则为 $0.896。两种运行形态都保持在 3 分钟目标以内。全规格关键通道基准测试的成本为 $2.936。GitHub 会把每个大型运行器作业向上取整到整分钟计费，因此把付费作业数从 49 个减少到 7 个，与缩短仓库工作耗时同样重要。 +主 Node CI 只有 1 个作业、1 轮设置、1 份完整门禁清单，而且没有分片选择器。加上 2 次 Node 兼容性执行、Python 和 Windows，生产环境共有 5 次付费大型运行器执行，而非 7 次粗粒度通道执行或 49 次门禁级执行。 -现有的零美元 Actions 预算并未阻止大型运行器作业。仅限本仓库的运行器组、有界的工作流拓扑、手动基准测试触发和作业超时限制才是经实测有效的成本控制机制；该预算不被视为执行防护措施。 +GitHub 会把每次大型运行器执行向上取整到整分钟计费，因此消除设置轮次既能减少计费时长，也能降低工作流复杂度。最终聚合作业仍使用标准运行器，因为它只会在付费作业释放容量后启动。 -生产 CI 依赖本 Agent Note 和 `.github/workflows/ci.yml` 中由组织持有的运行器名称。池缺失或改名会让作业一直排队，不会回退到标准容量。手动全规格、完整聚合、原分片和串行套件均予以保留，以便在映像、依赖、调度器或定价发生变化后重新测量，再调整生产标签。 +当前目标是基于观测得到的性能契约，而非取消截止时间。分支头精确的生产运行必须表明每个非 Windows 作业都低于 1 分钟，合并后的 Windows 作业低于 3 分钟；当映像、依赖、调度器或定价发生变化而需要重新测量时，仍可使用手动全规格和串行套件。 + +生产 CI 依赖 [`.github/workflows/ci.yml`](../../../../.github/workflows/ci.yml) 中由组织持有的运行器标签。池缺失或改名会让作业一直排队，不会回退到标准容量。全部 12 个池均保持已预配状态，因此手动基准测试无需再次经过管理配置周期，就能重新评估生产规格。 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 68a2d902ba..c6bb9fe627 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,6 @@ on: - serial-reference - larger-runner-benchmark - consolidated-runner-benchmark - - sharded-reference - optimized-larger-runners concurrency: @@ -29,299 +28,21 @@ env: PRIMARY_NODE_VERSION: '24' jobs: - node-24-sharded: - if: github.event_name == 'workflow_dispatch' && inputs.suite == 'sharded-reference' - runs-on: ${{ startsWith(matrix.lane, 'snapshot-') && 'dsh-ubuntu-24-04-8core' || 'dsh-ubuntu-24-04-4core' }} - name: node 24 / ${{ matrix.lane }} - env: - DSH_GATE_CONCURRENCY: ${{ matrix.gate_concurrency }} - DSH_PUBLINT_CONCURRENCY: ${{ matrix.publint_concurrency }} - DSH_ARTIFACT_SHARD: ${{ matrix.artifact_shard }} - DSH_COVERAGE_MAX_WORKERS: ${{ matrix.coverage_max_workers }} - DSH_COVERAGE_SHARD: ${{ matrix.coverage_shard }} - DSH_LINT_SHARD: ${{ matrix.lint_shard }} - DSH_SNAPSHOT_LANE: ${{ matrix.snapshot_lane }} - DSH_STATIC_SHARD: ${{ matrix.static_shard }} - DSH_SNAPSHOT_MAX_CONCURRENCY: ${{ matrix.snapshot_max_concurrency }} - DSH_SNAPSHOT_PREBUILT: ${{ matrix.snapshot_prebuilt }} - DSH_ESLINT_CACHE: ${{ matrix.eslint_cache }} - strategy: - fail-fast: false - matrix: - include: - - lane: static-foundation-projection - command: pnpm run check:ci:static - gate_concurrency: '4' - static_shard: 'foundation,site-projection' - - lane: static-doc-types - command: pnpm run check:ci:static - gate_concurrency: '1' - static_shard: doc-types - - lane: static-api-contracts - command: pnpm run check:ci:static - gate_concurrency: '4' - static_shard: api-contracts - - lane: static-catalogs - command: pnpm run check:ci:static - gate_concurrency: '4' - static_shard: catalogs - - lane: static-prose - command: pnpm run check:ci:static - gate_concurrency: '4' - static_shard: prose - - lane: static-site-mpa - command: pnpm run check:ci:static - gate_concurrency: '1' - static_shard: site-build - - lane: typecheck - command: pnpm run typecheck - - lane: lint-package-sources-a-c - command: pnpm run check:ci:lint - gate_concurrency: '1' - eslint_cache: '1' - lint_shard: package-sources-a-c - - lane: lint-package-sources-d-m - command: pnpm run check:ci:lint - gate_concurrency: '1' - eslint_cache: '1' - lint_shard: package-sources-d-m - - lane: lint-package-sources-n-s - command: pnpm run check:ci:lint - gate_concurrency: '1' - eslint_cache: '1' - lint_shard: package-sources-n-s - - lane: lint-package-sources-t-z - command: pnpm run check:ci:lint - gate_concurrency: '1' - eslint_cache: '1' - lint_shard: package-sources-t-z - - lane: lint-package-tests-a-c - command: pnpm run check:ci:lint - gate_concurrency: '1' - eslint_cache: '1' - lint_shard: package-tests-a-c - - lane: lint-package-tests-d-m - command: pnpm run check:ci:lint - gate_concurrency: '1' - eslint_cache: '1' - lint_shard: package-tests-d-m - - lane: lint-package-tests-n-s - command: pnpm run check:ci:lint - gate_concurrency: '1' - eslint_cache: '1' - lint_shard: package-tests-n-s - - lane: lint-package-tests-t-z - command: pnpm run check:ci:lint - gate_concurrency: '1' - eslint_cache: '1' - lint_shard: package-tests-t-z - - lane: lint-repository - command: pnpm run check:ci:lint - gate_concurrency: '1' - eslint_cache: '1' - lint_shard: repository - - lane: coverage-core-loop - command: pnpm run check:ci:coverage - gate_concurrency: '1' - coverage_max_workers: '4' - coverage_shard: core-loop - - lane: coverage-state-session - command: pnpm run check:ci:coverage - gate_concurrency: '1' - coverage_max_workers: '4' - coverage_shard: state-session - - lane: coverage-session-title - command: pnpm run check:ci:coverage - gate_concurrency: '1' - coverage_max_workers: '4' - coverage_shard: session-title - - lane: coverage-models - command: pnpm run check:ci:coverage - gate_concurrency: '1' - coverage_max_workers: '4' - coverage_shard: models - - lane: coverage-integrations - command: pnpm run check:ci:coverage - gate_concurrency: '1' - coverage_max_workers: '4' - coverage_shard: integrations - - lane: coverage-sdk-capabilities - command: pnpm run check:ci:coverage - gate_concurrency: '1' - coverage_max_workers: '4' - coverage_shard: sdk-capabilities - - lane: coverage-interfaces - command: pnpm run check:ci:coverage - gate_concurrency: '1' - coverage_max_workers: '4' - coverage_shard: interfaces - - lane: coverage-execution - command: pnpm run check:ci:coverage - gate_concurrency: '1' - coverage_max_workers: '4' - coverage_shard: execution - - lane: coverage-workflow - command: pnpm run check:ci:coverage - gate_concurrency: '1' - coverage_max_workers: '4' - coverage_shard: workflow - - lane: coverage-workflow-worker - command: pnpm run check:ci:coverage - gate_concurrency: '1' - coverage_max_workers: '4' - coverage_shard: workflow-worker - - lane: coverage-delegation - command: pnpm run check:ci:coverage - gate_concurrency: '1' - coverage_max_workers: '4' - coverage_shard: delegation - - lane: coverage-repository - command: pnpm run check:ci:coverage - gate_concurrency: '1' - coverage_max_workers: '4' - coverage_shard: repository - - lane: snapshot-support - command: pnpm run check:ci:snapshot - gate_concurrency: '1' - snapshot_lane: support - snapshot_max_concurrency: '5' - snapshot_prebuilt: '1' - - lane: snapshot-agents - command: pnpm run check:ci:snapshot - gate_concurrency: '1' - snapshot_lane: agents - snapshot_max_concurrency: '5' - snapshot_prebuilt: '1' - - lane: snapshot-acp-1 - command: pnpm run check:ci:snapshot - gate_concurrency: '1' - snapshot_lane: acp-1 - snapshot_max_concurrency: '5' - snapshot_prebuilt: '1' - - lane: snapshot-acp-2 - command: pnpm run check:ci:snapshot - gate_concurrency: '1' - snapshot_lane: acp-2 - snapshot_max_concurrency: '5' - snapshot_prebuilt: '1' - - lane: snapshot-acp-3 - command: pnpm run check:ci:snapshot - gate_concurrency: '1' - snapshot_lane: acp-3 - snapshot_max_concurrency: '5' - snapshot_prebuilt: '1' - - lane: snapshot-acp-4 - command: pnpm run check:ci:snapshot - gate_concurrency: '1' - snapshot_lane: acp-4 - snapshot_max_concurrency: '5' - snapshot_prebuilt: '1' - - lane: snapshot-acp-5 - command: pnpm run check:ci:snapshot - gate_concurrency: '1' - snapshot_lane: acp-5 - snapshot_max_concurrency: '5' - snapshot_prebuilt: '1' - - lane: snapshot-acp-6 - command: pnpm run check:ci:snapshot - gate_concurrency: '1' - snapshot_lane: acp-6 - snapshot_max_concurrency: '5' - snapshot_prebuilt: '1' - - lane: snapshot-acp-7 - command: pnpm run check:ci:snapshot - gate_concurrency: '1' - snapshot_lane: acp-7 - snapshot_max_concurrency: '5' - snapshot_prebuilt: '1' - - lane: snapshot-acp-8 - command: pnpm run check:ci:snapshot - gate_concurrency: '1' - snapshot_lane: acp-8 - snapshot_max_concurrency: '5' - snapshot_prebuilt: '1' - - lane: artifacts-metadata - command: pnpm run check:ci:artifacts - gate_concurrency: '3' - publint_concurrency: '8' - artifact_shard: metadata - - lane: artifacts-smoke - command: pnpm run check:ci:artifacts - gate_concurrency: '1' - artifact_shard: smoke - steps: - - uses: actions/checkout@v6 - - - uses: actions/setup-node@v6 - with: - node-version: ${{ env.PRIMARY_NODE_VERSION }} - - - name: Enable corepack and resolve pnpm store path - id: pnpm-store - run: | - corepack enable - echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" - - - uses: actions/cache@v4 - with: - path: ${{ steps.pnpm-store.outputs.path }} - key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm- - - - name: Install (immutable) - if: ${{ ! startsWith(matrix.lane, 'snapshot-') }} - run: pnpm install --frozen-lockfile - - # The snapshot lanes REPLAY the sandbox example's recorded scenarios, - # re-executing their bash calls under a real runner. ubuntu-latest has - # no bubblewrap preinstalled and no built Landlock launcher, so without - # this the confined executions fail closed (SANDBOX_UNAVAILABLE). The - # pinned Ubuntu payload is verified and extracted into the ephemeral - # runner instead of paying for a system package transaction. Bubblewrap - # preparation is independent of dependency installation and the build, - # so it runs beside both. - - name: Prepare built snapshot runtime and bubblewrap - if: startsWith(matrix.lane, 'snapshot-') - run: | - pnpm install --frozen-lockfile & - install_pid=$! - bash scripts/prepare-ci-bubblewrap.sh & - sandbox_pid=$! - install_status=0 - wait "$install_pid" || install_status=$? - build_status=0 - if (( install_status == 0 )); then - pnpm run build || build_status=$? - fi - sandbox_status=0 - wait "$sandbox_pid" || sandbox_status=$? - if (( install_status != 0 )); then exit "$install_status"; fi - if (( build_status != 0 )); then exit "$build_status"; fi - exit "$sandbox_status" - - - uses: actions/cache@v4 - if: startsWith(matrix.lane, 'lint-') - with: - path: .cache/eslint - key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-${{ matrix.lint_shard }}-${{ hashFiles('pnpm-lock.yaml', 'eslint.config.mjs', 'tsconfig.json', 'packages/*/*/tsconfig.json', 'examples/*/tsconfig.json') }} - restore-keys: | - ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-${{ matrix.lint_shard }}- - - - name: Run gates - run: ${{ matrix.command }} # One large runner pays hosted setup once, then the repository scheduler - # overlaps the unsharded core inventory. CPU and site chains use coarse lanes. + # overlaps the complete unsharded primary Node inventory. Build starts eagerly; + # only consumers of emitted output wait for it. node-24: if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' runs-on: dsh-ubuntu-24-04-96core - name: node 24 / core + name: node 24 / complete env: + DSH_COVERAGE_MAX_WORKERS: '12' DSH_ESLINT_CACHE: '1' DSH_ESLINT_CONCURRENCY: '32' DSH_GATE_CONCURRENCY: '32' DSH_PUBLINT_CONCURRENCY: '32' + DSH_SNAPSHOT_MAX_CONCURRENCY: '16' steps: - uses: actions/checkout@v6 @@ -362,53 +83,19 @@ jobs: if (( install_status != 0 )); then exit "$install_status"; fi exit "$sandbox_status" - - name: Run unsharded primary Node core CI concurrently - run: pnpm run check:ci:large-runner + - name: Run complete unsharded primary Node CI concurrently + run: pnpm run check:ci - # Keep the longest independent Linux gate on its own coarse-grained runner so - # cold install variance does not push either primary aggregate over a minute. - node-24-site: - if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' - runs-on: dsh-ubuntu-24-04-16core - name: node 24 / production site - steps: - - uses: actions/checkout@v6 - - - uses: actions/setup-node@v6 - with: - node-version: ${{ env.PRIMARY_NODE_VERSION }} - - - name: Enable corepack and resolve pnpm store path - id: pnpm-store - run: | - corepack enable - echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" - - - uses: actions/cache@v4 - with: - path: ${{ steps.pnpm-store.outputs.path }} - key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm- - - - name: Install (immutable) - run: pnpm install --frozen-lockfile - - - name: Build documentation site (production SPA) - run: pnpm run docs:build node-compat: if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' # Distinct larger-runner pools avoid both standard-runner setup outliers and - # the delayed second allocation observed when two jobs shared one pool. The - # primary Node row also owns the CPU/dependency-critical primary gate chain. + # delayed allocation when independent environment contracts share one pool. runs-on: ${{ matrix.runner }} name: ${{ matrix.name }} env: - DSH_COVERAGE_MAX_WORKERS: ${{ matrix.coverage_workers }} DSH_GATE_CONCURRENCY: ${{ matrix.gate_concurrency }} - DSH_NODE_COMPAT_SKIP_TYPECHECK: ${{ matrix.skip_typecheck }} - DSH_SNAPSHOT_MAX_CONCURRENCY: ${{ matrix.snapshot_workers }} + DSH_NODE_COMPAT_SKIP_TYPECHECK: '1' strategy: fail-fast: false matrix: @@ -416,24 +103,11 @@ jobs: - node: '22.19' name: node 22.19 runner: dsh-ubuntu-24-04-4core - command: pnpm run check:node-compat gate_concurrency: '2' - skip_typecheck: '1' - - node: 24 - name: node 24 / cpu - runner: dsh-ubuntu-24-04-64core - command: pnpm run check:ci:primary-cpu - coverage_workers: '12' - gate_concurrency: '6' - primary_cpu: true - snapshot_workers: '16' - skip_typecheck: '1' - node: 26 name: node 26 runner: dsh-ubuntu-24-04-32core - command: pnpm run check:node-compat gate_concurrency: '2' - skip_typecheck: '1' steps: - uses: actions/checkout@v6 @@ -455,25 +129,10 @@ jobs: ${{ runner.os }}-node-${{ matrix.node }}-pnpm- - name: Install (immutable) - if: matrix.primary_cpu != true run: pnpm install --frozen-lockfile - - name: Install dependencies and prepare bubblewrap - if: matrix.primary_cpu == true - run: | - pnpm install --frozen-lockfile & - install_pid=$! - bash scripts/prepare-ci-bubblewrap.sh & - sandbox_pid=$! - install_status=0 - wait "$install_pid" || install_status=$? - sandbox_status=0 - wait "$sandbox_pid" || sandbox_status=$? - if (( install_status != 0 )); then exit "$install_status"; fi - exit "$sandbox_status" - - - name: Run assigned gates - run: ${{ matrix.command }} + - name: Run compatibility smokes + run: pnpm run check:node-compat python-sdk: if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' @@ -493,145 +152,6 @@ jobs: - name: Run complete keyless Python suite run: uv run --python 3.10 --group test --project python/sdk pytest - # Manual sharded Windows reference matching the former production topology. - # Blocking build and production-site lanes keep the already-green - # native outputs protected without putting both critical paths in one job. - # The broader observational gate matrix below exposes the remaining - # portability work without blocking mainline merges. - windows-build-sharded: - if: github.event_name == 'workflow_dispatch' && inputs.suite == 'sharded-reference' - runs-on: dsh-windows-2025-4core - name: windows / build - steps: - - uses: actions/checkout@v6 - - - uses: actions/setup-node@v6 - with: - node-version: ${{ env.PRIMARY_NODE_VERSION }} - - - name: Install (immutable) - run: | - corepack enable - pnpm install --frozen-lockfile - - - name: Build (tsc -b + tsdown) - run: pnpm run build - - windows-site-sharded: - if: github.event_name == 'workflow_dispatch' && inputs.suite == 'sharded-reference' - runs-on: dsh-windows-2025-4core - name: windows / production site - steps: - - uses: actions/checkout@v6 - - - uses: actions/setup-node@v6 - with: - node-version: ${{ env.PRIMARY_NODE_VERSION }} - - - name: Install (immutable) - run: | - corepack enable - pnpm install --frozen-lockfile - - - name: Build documentation site (production SPA) - run: pnpm run docs:build - - # Observational, non-blocking Windows static, lint, and artifact lanes. Coverage - # and snapshot stay Linux-only until their platform-specific runtime failures - # have dedicated support. Run the gates from native PowerShell: an MSYS parent - # would change the environment being measured. This job intentionally stays - # out of all-checks-passed.needs. - windows-gates-sharded: - if: github.event_name == 'workflow_dispatch' && inputs.suite == 'sharded-reference' - continue-on-error: true - runs-on: dsh-windows-2025-4core - name: windows node 24 / ${{ matrix.lane }} - env: - DSH_GATE_CONCURRENCY: ${{ matrix.gate_concurrency }} - DSH_PUBLINT_CONCURRENCY: ${{ matrix.publint_concurrency }} - DSH_ARTIFACT_SHARD: ${{ matrix.artifact_shard }} - DSH_LINT_SHARD: ${{ matrix.lint_shard }} - DSH_STATIC_SHARD: ${{ matrix.static_shard }} - DSH_ESLINT_CACHE: ${{ matrix.eslint_cache }} - strategy: - fail-fast: false - matrix: - include: - - lane: static-general - command: pnpm run check:ci:static - gate_concurrency: '4' - static_shard: 'foundation,catalogs,prose' - - lane: static-contracts - command: pnpm run check:ci:static - gate_concurrency: '4' - static_shard: 'doc-types,api-contracts' - - lane: lint-package-sources - command: pnpm run check:ci:lint - gate_concurrency: '1' - eslint_cache: '1' - lint_shard: package-sources - - lane: lint-package-tests - command: pnpm run check:ci:lint - gate_concurrency: '1' - eslint_cache: '1' - lint_shard: package-tests - - lane: lint-repository - command: pnpm run check:ci:lint - gate_concurrency: '1' - eslint_cache: '1' - lint_shard: repository - - lane: artifacts-metadata - command: pnpm run check:ci:artifacts - gate_concurrency: '3' - publint_concurrency: '8' - artifact_shard: metadata - - lane: artifacts-smoke - command: pnpm run check:ci:artifacts - gate_concurrency: '1' - artifact_shard: smoke - steps: - - uses: actions/checkout@v6 - - - name: Enable Developer Mode (symlink support) - shell: pwsh - run: >- - reg add "HKLM\SOFTWARE\Microsoft\Windows\CurrentVersion\AppModelUnlock" - /t REG_DWORD /f /v "AllowDevelopmentWithoutDevLicense" /d "1" - - - uses: actions/setup-node@v6 - with: - node-version: ${{ env.PRIMARY_NODE_VERSION }} - - - name: Enable corepack and resolve pnpm store path - id: pnpm-store - shell: pwsh - run: | - corepack enable - "path=$(pnpm store path --silent)" >> $env:GITHUB_OUTPUT - - - uses: actions/cache@v4 - with: - path: ${{ steps.pnpm-store.outputs.path }} - key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm- - - - name: Install (immutable) - shell: pwsh - run: pnpm install --frozen-lockfile - - - uses: actions/cache@v4 - if: startsWith(matrix.lane, 'lint-') - with: - path: .cache/eslint - key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-${{ matrix.lint_shard }}-${{ hashFiles('pnpm-lock.yaml', 'eslint.config.mjs', 'tsconfig.json', 'packages/*/*/tsconfig.json', 'examples/*/tsconfig.json') }} - restore-keys: | - ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-${{ matrix.lint_shard }}- - - - name: Run gates - shell: pwsh - run: ${{ matrix.command }} - # One Windows box shares setup across the required build/site checks and the # complete observational portability inventory. run-gates reports failures # from observational gates without allowing them to fail the required job. @@ -1035,7 +555,7 @@ jobs: all-checks-passed: name: all checks passed runs-on: ubuntu-latest - needs: [node-24, node-24-site, node-compat, python-sdk, windows] + needs: [node-24, node-compat, python-sdk, windows] if: always() && (github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners') steps: - name: Fail if any needed job did not succeed diff --git a/examples/acp-agent/tests/acp.snapshot.ts b/examples/acp-agent/tests/acp.snapshot.ts index 41edde1aae..65df474f97 100644 --- a/examples/acp-agent/tests/acp.snapshot.ts +++ b/examples/acp-agent/tests/acp.snapshot.ts @@ -1,7 +1,6 @@ import { fileURLToPath } from 'node:url' import { dirname, join } from 'node:path' import { defineAcpSnapshotSuite, type Scenario, type SnapshotSuiteOptions } from '@deepseek-ai/dsh-acp-snapshot' -import { snapshotScenarioShardFromEnv } from './snapshot-scenario-shard.ts' /** * The acp-agent example's snapshot suite: the scenario table for @@ -199,12 +198,9 @@ const SCENARIOS: Scenario[] = [ { name: 'fs-escalation-approved', hasModelTurn: true, recorded: true, headerClass: 'sandbox' }, ] -const scenarioShard = snapshotScenarioShardFromEnv(process.env.DSH_SNAPSHOT_SCENARIO_SHARD) - defineAcpSnapshotSuite({ agent: AGENT, snapshotsDir: join(dirname(fileURLToPath(import.meta.url)), 'snapshots'), scenarios: SCENARIOS, mode: snapshotModeFromEnv(process.env.DSH_SNAPSHOT), - ...scenarioShard === undefined ? {} : { scenarioShard }, }) diff --git a/examples/acp-agent/tests/snapshot-scenario-shard.spec.ts b/examples/acp-agent/tests/snapshot-scenario-shard.spec.ts deleted file mode 100644 index 5a70cfe3b1..0000000000 --- a/examples/acp-agent/tests/snapshot-scenario-shard.spec.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { snapshotScenarioShardFromEnv } from './snapshot-scenario-shard.ts' - -describe('ACP snapshot scenario shard environment', () => { - it('keeps ordinary snapshot runs complete', () => { - expect(snapshotScenarioShardFromEnv()).toBeUndefined() - expect(snapshotScenarioShardFromEnv('')).toBeUndefined() - }) - - it('parses a valid one-based shard', () => { - expect(snapshotScenarioShardFromEnv('2/4')).toEqual({ index: 2, total: 4 }) - }) - - it.each(['0/1', '1/0', '2/1', '1.5/2', 'missing', '999999999999999999999/999999999999999999999'])( - 'rejects %s', - (value) => { - expect(() => snapshotScenarioShardFromEnv(value)).toThrow('DSH_SNAPSHOT_SCENARIO_SHARD') - }, - ) -}) diff --git a/examples/acp-agent/tests/snapshot-scenario-shard.ts b/examples/acp-agent/tests/snapshot-scenario-shard.ts deleted file mode 100644 index 4e2763bdb6..0000000000 --- a/examples/acp-agent/tests/snapshot-scenario-shard.ts +++ /dev/null @@ -1,18 +0,0 @@ -import type { SnapshotScenarioShard } from '@deepseek-ai/dsh-acp-snapshot' - -/** - * Parse the optional CI scenario shard passed to this snapshot suite. - * - * @param value An `INDEX/TOTAL` string or an unset value. - * @returns A validated one-based shard, or undefined for the complete suite. - */ -export function snapshotScenarioShardFromEnv(value?: string): SnapshotScenarioShard | undefined { - if (value === undefined || value === '') return undefined - const match = /^([1-9]\d*)\/([1-9]\d*)$/.exec(value) - if (match === null) throw new Error(`DSH_SNAPSHOT_SCENARIO_SHARD must be INDEX/TOTAL, got ${JSON.stringify(value)}`) - const shard = { index: Number(match[1]), total: Number(match[2]) } - if (!Number.isSafeInteger(shard.index) || !Number.isSafeInteger(shard.total) || shard.index > shard.total) { - throw new Error(`DSH_SNAPSHOT_SCENARIO_SHARD is out of range: ${JSON.stringify(value)}`) - } - return shard -} diff --git a/package.json b/package.json index 69d22d7ef1..e6daf9b139 100644 --- a/package.json +++ b/package.json @@ -26,8 +26,6 @@ "test:snapshot:record": "DSH_SNAPSHOT=record vitest run --config vitest.snapshot.config.ts --update", "test:snapshot:refresh": "DSH_SNAPSHOT=refresh vitest run --config vitest.snapshot.config.ts", "check:ci": "tsx scripts/run-gates.ts ci-primary", - "check:ci:primary-cpu": "tsx scripts/run-gates.ts ci-primary-cpu", - "check:ci:large-runner": "tsx scripts/run-gates.ts ci-primary-large-runner", "check:ci:static": "tsx scripts/run-gates.ts ci-static", "check:ci:lint": "tsx scripts/run-gates.ts ci-lint", "check:ci:coverage": "tsx scripts/run-gates.ts ci-coverage", diff --git a/packages/support/acp-snapshot/README.md b/packages/support/acp-snapshot/README.md index 0875e0fb86..6e37534e73 100644 --- a/packages/support/acp-snapshot/README.md +++ b/packages/support/acp-snapshot/README.md @@ -7,7 +7,7 @@ Four layers, importable separately: - **`launchAcpTestAgent` (launcher)** — boots a source agent under tsx or a built `lib` agent under plain Node from a temp cwd, connects the SDK client over a raw-byte stdout tee, collects session updates and stderr, surfaces asynchronous spawn failures through startup, fails closed on unhandled permission requests, and owns graceful or signalled shutdown. Shutdown waits for process exit, inherited stdio closure, and ACP parser exhaustion before resolving or propagating a child error, so captures are complete and callers can remove owned paths after either outcome. When Windows accepts forced termination but publishes its exit marker asynchronously, shutdown gives that marker a bounded grace before treating fallback refusal as a second failure. Snapshot and ordinary e2e suites share this process boundary; a test supplies only agent paths, cwd, environment overrides, and any permission policy. - **`runScenario` (harness)** — drives ACP JSON-RPC stdio from a deterministic `input.json` script through the launcher, tees raw stdout for the expected-output and purity checks, and harvests every persisted raw JSONL session log (parent and subagent children, primary-first) after graceful stdin EOF. `AgentUnderTest` supplies absolute `binScript`, optional `libBinScript`, `configPath`, and `tsconfigPath` paths because the subprocess cwd is outside the repo. Startup failures preserve captured agent stderr in the rejected diagnostic. - **Normalizers** — pure functions turning the two captured surfaces into stable text: `normalizeStdout` (JSON-RPC ids → first-seen sequence; UUIDs/cwd → tokens; cwd-rooted separators selected as canonical `/` or host-native; `session_info_update.updatedAt` → `{{updatedAt}}`; doubles as the stdout-purity check), `normalizeSessionLog` (times zeroed, `seq` kept, the same cwd-path policy), `scrubSystemPrompts` (prompt text → `{{system}}`), `scrubToolSchemas` (schema bulk → `{{tools}}`), and `scrubRequestHeaders` (all header bulk → `{{system}}`/`{{tools}}`/`{{messagePrefix}}` outside each pin, structure kept — [pinned-header Agent Note](../../../.agents/notes/implemented/testing/2026-07-06-pin-request-header-content-in-one-scenario.md)). -- **`defineAcpSnapshotSuite` (factory)** — registers the whole describe/it tree for a scenario table: per-scenario expected-output and re-persisted-log comparisons, record/refresh fixture write-back, rejection of structured `UNKNOWN_TOOL` results, the per-header-class pin (`system-prompt.expected.md` plus `tool-schemas.expected.json`) with its live uniformity guard, and the fixture guard block (no orphan scenario dirs, required files present, exactly one pin per class, every JSONL prompt/schema-scrubbed, non-pinning fixtures fully header-scrubbed). Replay may partition subprocess-backed scenarios with `scenarioShard`; every lane still runs fixture guards against the complete table, while record and refresh reject sharding because they write fixtures. Refresh preserves existing volatile fields by event position and gives a newly inserted `session/title` its preceding event's time, so feature-driven insertions do not churn the remainder of a fixture. Each scenario directory's `session.jsonl` plus contiguous `session..jsonl` siblings are the ordered primary/child inventory; the scenario table does not duplicate their count. Must be called at vitest collection time. +- **`defineAcpSnapshotSuite` (factory)** — registers the whole describe/it tree for a scenario table: per-scenario expected-output and re-persisted-log comparisons, record/refresh fixture write-back, rejection of structured `UNKNOWN_TOOL` results, the per-header-class pin (`system-prompt.expected.md` plus `tool-schemas.expected.json`) with its live uniformity guard, and the fixture guard block (no orphan scenario dirs, required files present, exactly one pin per class, every JSONL prompt/schema-scrubbed, non-pinning fixtures fully header-scrubbed). Refresh preserves existing volatile fields by event position and gives a newly inserted `session/title` its preceding event's time, so feature-driven insertions do not churn the remainder of a fixture. Each scenario directory's `session.jsonl` plus contiguous `session..jsonl` siblings are the ordered primary/child inventory; the scenario table does not duplicate their count. Must be called at vitest collection time. A consuming `*.snapshot.ts` is the scenario table plus one factory call: diff --git a/packages/support/acp-snapshot/src/index.ts b/packages/support/acp-snapshot/src/index.ts index c67721ecdd..2a03947fef 100644 --- a/packages/support/acp-snapshot/src/index.ts +++ b/packages/support/acp-snapshot/src/index.ts @@ -48,4 +48,3 @@ export { type Scenario, type SnapshotSuiteOptions, } from './suite.ts' -export type { SnapshotScenarioShard } from './scenario-shard.ts' diff --git a/packages/support/acp-snapshot/src/scenario-shard.ts b/packages/support/acp-snapshot/src/scenario-shard.ts deleted file mode 100644 index 67b2492b80..0000000000 --- a/packages/support/acp-snapshot/src/scenario-shard.ts +++ /dev/null @@ -1,36 +0,0 @@ -/** Scenario-level sharding for one ACP snapshot suite. */ - -/** A one-based, exhaustive partition of a scenario table. */ -export interface SnapshotScenarioShard { - /** One-based lane index. */ - index: number - /** Total number of lanes. */ - total: number -} - -/** - * Select one stable modulo partition while rejecting empty or malformed lanes. - * - * @param scenarios Complete ordered scenario table. - * @param shard Optional one-based shard description. - * @returns The complete table or the selected non-empty partition. - */ -export function selectSnapshotScenarios( - scenarios: readonly T[], - shard?: SnapshotScenarioShard, -): T[] { - if (shard === undefined) return [...scenarios] - if (!Number.isSafeInteger(shard.index) || shard.index < 1) { - throw new Error(`acp-snapshot: shard index must be a positive integer, got ${shard.index}`) - } - if (!Number.isSafeInteger(shard.total) || shard.total < 1) { - throw new Error(`acp-snapshot: shard total must be a positive integer, got ${shard.total}`) - } - if (shard.index > shard.total) { - throw new Error(`acp-snapshot: shard index ${shard.index} exceeds total ${shard.total}`) - } - if (shard.total > scenarios.length) { - throw new Error(`acp-snapshot: ${shard.total} shards exceed ${scenarios.length} scenarios`) - } - return scenarios.filter((_, offset) => offset % shard.total === shard.index - 1) -} diff --git a/packages/support/acp-snapshot/src/suite.ts b/packages/support/acp-snapshot/src/suite.ts index 708fc15134..ab6008913c 100644 --- a/packages/support/acp-snapshot/src/suite.ts +++ b/packages/support/acp-snapshot/src/suite.ts @@ -29,7 +29,6 @@ import { scrubSystemPrompts, scrubToolSchemas, } from './normalize.ts' -import { selectSnapshotScenarios, type SnapshotScenarioShard } from './scenario-shard.ts' /** The readable system-prompt snapshot beside each header-pinning fixture. */ const SYSTEM_PROMPT_SNAPSHOT = 'system-prompt.expected.md' @@ -170,11 +169,6 @@ export interface SnapshotSuiteOptions { snapshotsDir: string /** The scenario table; exactly one entry per header class must set `pinsHeader`. */ scenarios: Scenario[] - /** - * Optional replay-only scenario partition. Fixture guards still validate the - * complete table in every lane; only subprocess-backed scenario tests split. - */ - scenarioShard?: SnapshotScenarioShard /** * `replay` (keyless, the default tier), `record` (live API; re-records the * `recorded` scenarios' fixtures and refreshes the Vitest expected outputs under @@ -517,11 +511,7 @@ export function stabilizeRefreshLog(fresh: string, existing: string, replacement * @param options The agent, snapshots directory, scenario table, and mode. */ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void { - const { agent, snapshotsDir, scenarios, mode, scenarioShard } = options - if (scenarioShard !== undefined && mode !== 'replay') { - throw new Error('acp-snapshot: scenario sharding is supported only in replay mode') - } - const selectedScenarios = selectSnapshotScenarios(scenarios, scenarioShard) + const { agent, snapshotsDir, scenarios, mode } = options const RECORDING = mode === 'record' const REFRESHING = mode === 'refresh' const childMode: 'replay' | 'record' = RECORDING ? 'record' : 'replay' @@ -546,7 +536,7 @@ export function defineAcpSnapshotSuite(options: SnapshotSuiteOptions): void { } scenarioSuite('snapshot scenarios', () => { - for (const scenario of selectedScenarios) { + for (const scenario of scenarios) { // In RECORD mode, only re-run the `recorded` (live-API) scenarios; the `authored` ones // (sidecar-driven errors/cancel) are never re-recorded. `posixOnly` scenarios skip on // Windows, where their process semantics cannot be driven. diff --git a/packages/support/acp-snapshot/tests/scenario-shard.spec.ts b/packages/support/acp-snapshot/tests/scenario-shard.spec.ts deleted file mode 100644 index eb5c117153..0000000000 --- a/packages/support/acp-snapshot/tests/scenario-shard.spec.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { selectSnapshotScenarios } from '../src/scenario-shard.ts' - -describe('ACP snapshot scenario shards', () => { - it('keeps the ordinary suite complete', () => { - expect(selectSnapshotScenarios(['a', 'b', 'c'])).toEqual(['a', 'b', 'c']) - }) - - it('partitions the ordered table without gaps or overlap', () => { - const scenarios = ['a', 'b', 'c', 'd', 'e'] - expect(selectSnapshotScenarios(scenarios, { index: 1, total: 2 })).toEqual(['a', 'c', 'e']) - expect(selectSnapshotScenarios(scenarios, { index: 2, total: 2 })).toEqual(['b', 'd']) - }) - - it.each([ - [{ index: 0, total: 1 }, 'index must be a positive integer'], - [{ index: 1.5, total: 2 }, 'index must be a positive integer'], - [{ index: 1, total: 0 }, 'total must be a positive integer'], - [{ index: 1, total: Number.NaN }, 'total must be a positive integer'], - [{ index: 3, total: 2 }, 'exceeds total'], - [{ index: 1, total: 4 }, 'exceed 3 scenarios'], - ] as const)('rejects malformed shard %#', (shard, message) => { - expect(() => selectSnapshotScenarios(['a', 'b', 'c'], shard)).toThrow(message) - }) -}) diff --git a/packages/support/acp-snapshot/tests/suite.spec.ts b/packages/support/acp-snapshot/tests/suite.spec.ts index 3fdcc85423..b80b5a50d2 100644 --- a/packages/support/acp-snapshot/tests/suite.spec.ts +++ b/packages/support/acp-snapshot/tests/suite.spec.ts @@ -107,16 +107,6 @@ describe('defineAcpSnapshotSuite: replay mode', () => { defineAcpSnapshotSuite({ agent: AGENT, snapshotsDir: REPLAY_DIR, scenarios: REPLAY_SCENARIOS, mode: 'replay' }) }) -describe('defineAcpSnapshotSuite: sharded replay mode', () => { - defineAcpSnapshotSuite({ - agent: AGENT, - snapshotsDir: REPLAY_DIR, - scenarios: REPLAY_SCENARIOS, - mode: 'replay', - scenarioShard: { index: 2, total: 2 }, - }) -}) - // The record suite's tests run in registration order: rec-pin re-records the // pinned fixture FIRST, so rec-child's uniformity guard reads the fresh pin. describe('defineAcpSnapshotSuite: record mode', () => { @@ -166,18 +156,6 @@ describe('defineAcpSnapshotSuite: record inventory write-back', () => { }) describe('defineAcpSnapshotSuite: registration contract', () => { - it('rejects scenario sharding in a fixture-writing mode', () => { - expect(() => { - defineAcpSnapshotSuite({ - agent: AGENT, - snapshotsDir: REPLAY_DIR, - scenarios: REPLAY_SCENARIOS, - mode: 'refresh', - scenarioShard: { index: 1, total: 2 }, - }) - }).toThrow('supported only in replay mode') - }) - it("throws when a scenario's header class has no pinning scenario", () => { expect(() => { defineAcpSnapshotSuite({ diff --git a/scripts/coverage-shards.spec.ts b/scripts/coverage-shards.spec.ts deleted file mode 100644 index 678011618e..0000000000 --- a/scripts/coverage-shards.spec.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { readdirSync } from 'node:fs' -import { resolve } from 'node:path' -import { describe, expect, it } from 'vitest' -import { coverageArgs, coverageShards } from './coverage-shards.ts' - -const repositoryRoot = resolve(import.meta.dirname, '..') - -describe('coverage shards', () => { - it('assigns every workspace package to exactly one lane', () => { - const packagesRoot = resolve(repositoryRoot, 'packages') - const workspacePackages = readdirSync(packagesRoot, { withFileTypes: true }) - .filter(group => group.isDirectory()) - .flatMap(group => readdirSync(resolve(packagesRoot, group.name), { withFileTypes: true }) - .filter(entry => entry.isDirectory()) - .map(entry => `${group.name}/${entry.name}`)) - .sort() - const assignedPackages = coverageShards.flatMap(shard => shard.packageRoots.flatMap((packageRoot) => { - if (packageRoot.includes('/')) return [packageRoot] - return readdirSync(resolve(packagesRoot, packageRoot), { withFileTypes: true }) - .filter(entry => entry.isDirectory()) - .map(entry => `${packageRoot}/${entry.name}`) - })) - - expect([...assignedPackages].sort()).toEqual(workspacePackages) - expect(new Set(assignedPackages).size).toBe(assignedPackages.length) - }) - - it.each(coverageShards)('selects tests and source includes for $name', (shard) => { - const args = coverageArgs(shard.name) - for (const packageRoot of shard.packageRoots) { - expect(args).toContain(`packages/${packageRoot}/`) - expect(args).toContain(packageRoot.includes('/') - ? `--coverage.include=packages/${packageRoot}/src/**/*.ts` - : `--coverage.include=packages/${packageRoot}/*/src/**/*.ts`) - } - if ('extraTestRoots' in shard) { - for (const testRoot of shard.extraTestRoots) expect(args).toContain(`${testRoot}/`) - } - expect(args).toContain('scripts/test-invariants.spec.ts') - expect(new Set(args).size).toBe(args.length) - }) - - it('rejects an unknown lane', () => { - expect(() => coverageArgs('missing')).toThrow('unknown DSH_COVERAGE_SHARD') - }) -}) diff --git a/scripts/coverage-shards.ts b/scripts/coverage-shards.ts deleted file mode 100644 index 84f4d1a63e..0000000000 --- a/scripts/coverage-shards.ts +++ /dev/null @@ -1,114 +0,0 @@ -/** Coverage shard definitions for the GitHub Actions source-test lanes. */ - -/** A coverage lane that owns complete package roots and optional cross-package tests. */ -export interface CoverageShard { - /** Stable lane identifier passed through `DSH_COVERAGE_SHARD`. */ - name: string - /** Group or package paths below `packages/` whose tests and source coverage belong to the lane. */ - packageRoots: readonly string[] - /** Additional test roots needed for cross-package behavior or repository scripts. */ - extraTestRoots?: readonly string[] -} - -/** Exhaustive, non-overlapping ownership of workspace packages in coverage CI. */ -export const coverageShards = [ - { - name: 'core-loop', - packageRoots: ['core/agent', 'core/agent-loop', 'core/tools'], - }, - { - name: 'state-session', - packageRoots: [ - 'core/session', - 'core/scope', - 'core/system-prompt', - 'context', - 'session-persistence', - 'session-query', - 'support/invariants', - ], - extraTestRoots: [ - 'packages/examples/cli-demo/tests', - 'packages/llm/token-meter/tests', - 'scripts', - ], - }, - { - name: 'models', - packageRoots: ['llm', 'compact'], - }, - { - name: 'session-title', - packageRoots: ['session-title'], - }, - { - name: 'integrations', - packageRoots: ['hooks/hook-protocol', 'lsp', 'mcp', 'hooks/hooks-claude'], - }, - { - name: 'sdk-capabilities', - packageRoots: [ - 'sdk', - 'hooks/hooks-codex', - 'web', - 'skill', - 'spill', - 'util', - 'guard', - 'todo', - 'timeout', - ], - }, - { - name: 'interfaces', - packageRoots: ['ui', 'examples', 'goal'], - extraTestRoots: ['examples'], - }, - { name: 'execution', packageRoots: ['fs', 'bash', 'sandbox', 'code-runtime'] }, - { - name: 'workflow', - packageRoots: ['workflow/workflow', 'workflow/tool-workflow', 'workflow/tool-ralph'], - }, - { - name: 'workflow-worker', - packageRoots: ['workflow/workflow-workerthread'], - }, - { name: 'delegation', packageRoots: ['subagent', 'tasks'] }, - { - name: 'repository', - packageRoots: [ - 'cordis', - 'support/acp-snapshot', - 'support/agent-loop-testkit', - 'support/llm-replay', - 'support/loader-smoke', - ], - }, -] as const satisfies readonly CoverageShard[] - -/** - * Build Vitest filters and coverage include globs for one source-test lane. - * - * @param name Stable shard name from {@link coverageShards}. - * @returns Positional test roots followed by per-group coverage include flags. - */ -export function coverageArgs(name: string): string[] { - const shard = coverageShards.find(candidate => candidate.name === name) - if (shard === undefined) { - throw new Error(`run-gates: unknown DSH_COVERAGE_SHARD ${JSON.stringify(name)}.`) - } - - // Vitest positional filters are substrings; the trailing separator keeps - // prefix-named sibling packages out of each lane. - const testRoots = new Set([ - ...shard.packageRoots.map(packageRoot => `packages/${packageRoot}/`), - ...('extraTestRoots' in shard ? shard.extraTestRoots.map(testRoot => `${testRoot}/`) : []), - 'scripts/test-invariants.spec.ts', - ]) - return [ - ...testRoots, - ...shard.packageRoots.map(packageRoot => packageRoot.includes('/') - ? `--coverage.include=packages/${packageRoot}/src/**/*.ts` - : `--coverage.include=packages/${packageRoot}/*/src/**/*.ts`), - ] -} diff --git a/scripts/lint-shards.spec.ts b/scripts/lint-shards.spec.ts deleted file mode 100644 index d3020fa6b2..0000000000 --- a/scripts/lint-shards.spec.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { readdirSync } from 'node:fs' -import { resolve } from 'node:path' -import { describe, expect, it } from 'vitest' -import { selectLintShard } from './lint-shards.ts' - -const packagesRoot = resolve(import.meta.dirname, '..', 'packages') - -describe('lint gate shards', () => { - it('keeps the unsharded local command complete', () => { - expect(selectLintShard()).toEqual({ eslintTargets: ['.'], includeDuplication: true }) - expect(selectLintShard('')).toEqual({ eslintTargets: ['.'], includeDuplication: true }) - }) - - it('partitions package sources and tests into alphabetic ranges plus their repository complement', () => { - expect(selectLintShard('package-sources-a-c')).toEqual({ - eslintTargets: ['packages/[a-c]*/*/src/**/*.ts'], - includeDuplication: false, - }) - expect(selectLintShard('package-sources-d-m')).toEqual({ - eslintTargets: ['packages/[d-m]*/*/src/**/*.ts'], - includeDuplication: false, - }) - expect(selectLintShard('package-sources-n-s')).toEqual({ - eslintTargets: ['packages/[n-s]*/*/src/**/*.ts'], - includeDuplication: false, - }) - expect(selectLintShard('package-sources-t-z')).toEqual({ - eslintTargets: ['packages/[t-z]*/*/src/**/*.ts'], - includeDuplication: false, - }) - expect(selectLintShard('package-tests-a-c')).toEqual({ - eslintTargets: ['packages/[a-c]*/*/tests/**/*.ts'], - includeDuplication: false, - }) - expect(selectLintShard('package-tests-d-m')).toEqual({ - eslintTargets: ['packages/[d-m]*/*/tests/**/*.ts'], - includeDuplication: false, - }) - expect(selectLintShard('package-tests-n-s')).toEqual({ - eslintTargets: ['packages/[n-s]*/*/tests/**/*.ts'], - includeDuplication: false, - }) - expect(selectLintShard('package-tests-t-z')).toEqual({ - eslintTargets: ['packages/[t-z]*/*/tests/**/*.ts'], - includeDuplication: false, - }) - expect(selectLintShard('package-sources')).toEqual({ - eslintTargets: ['packages/*/*/src/**/*.ts'], - includeDuplication: false, - }) - expect(selectLintShard('package-tests')).toEqual({ - eslintTargets: ['packages/*/*/tests/**/*.ts'], - includeDuplication: false, - }) - expect(selectLintShard('repository')).toEqual({ - eslintTargets: [ - '.', - '--ignore-pattern', - 'packages/*/*/src/**', - '--ignore-pattern', - 'packages/*/*/tests/**', - ], - includeDuplication: true, - }) - }) - - it('assigns every package group once in the Linux topology', () => { - const groups = readdirSync(packagesRoot, { withFileTypes: true }) - .filter(entry => entry.isDirectory()) - .map(entry => entry.name) - .sort() - const ranges = [/^[a-c]/u, /^[d-m]/u, /^[n-s]/u, /^[t-z]/u] - const assignments = ranges.flatMap(range => groups.filter(group => range.test(group))).sort() - - expect(assignments).toEqual(groups) - }) - - it('rejects an unknown lane', () => { - expect(() => selectLintShard('missing')).toThrow('unknown DSH_LINT_SHARD') - }) -}) diff --git a/scripts/lint-shards.ts b/scripts/lint-shards.ts deleted file mode 100644 index 05c806c49f..0000000000 --- a/scripts/lint-shards.ts +++ /dev/null @@ -1,56 +0,0 @@ -/** Lint-lane selection for GitHub Actions. */ - -/** One ESLint target set and whether it owns the cross-file duplication gate. */ -export interface LintSelection { - /** Shell-free arguments passed to ESLint before its cache options. */ - eslintTargets: readonly string[] - /** Whether this lane also runs the repository-wide duplication check. */ - includeDuplication: boolean -} - -/** - * Select an exhaustive lint partition without changing the ordinary local lint command. - * - * @param name Optional stable shard name from `DSH_LINT_SHARD`. - * @returns ESLint targets and ownership of the duplication gate. - */ -export function selectLintShard(name?: string): LintSelection { - switch (name) { - case undefined: - case '': - return { eslintTargets: ['.'], includeDuplication: true } - case 'package-sources-a-c': - return { eslintTargets: ['packages/[a-c]*/*/src/**/*.ts'], includeDuplication: false } - case 'package-sources-d-m': - return { eslintTargets: ['packages/[d-m]*/*/src/**/*.ts'], includeDuplication: false } - case 'package-sources-n-s': - return { eslintTargets: ['packages/[n-s]*/*/src/**/*.ts'], includeDuplication: false } - case 'package-sources-t-z': - return { eslintTargets: ['packages/[t-z]*/*/src/**/*.ts'], includeDuplication: false } - case 'package-sources': - return { eslintTargets: ['packages/*/*/src/**/*.ts'], includeDuplication: false } - case 'package-tests-a-c': - return { eslintTargets: ['packages/[a-c]*/*/tests/**/*.ts'], includeDuplication: false } - case 'package-tests-d-m': - return { eslintTargets: ['packages/[d-m]*/*/tests/**/*.ts'], includeDuplication: false } - case 'package-tests-n-s': - return { eslintTargets: ['packages/[n-s]*/*/tests/**/*.ts'], includeDuplication: false } - case 'package-tests-t-z': - return { eslintTargets: ['packages/[t-z]*/*/tests/**/*.ts'], includeDuplication: false } - case 'package-tests': - return { eslintTargets: ['packages/*/*/tests/**/*.ts'], includeDuplication: false } - case 'repository': - return { - eslintTargets: [ - '.', - '--ignore-pattern', - 'packages/*/*/src/**', - '--ignore-pattern', - 'packages/*/*/tests/**', - ], - includeDuplication: true, - } - default: - throw new Error(`run-gates: unknown DSH_LINT_SHARD ${JSON.stringify(name)}.`) - } -} diff --git a/scripts/run-gates.ts b/scripts/run-gates.ts index 3e31384d6b..bc1ade1458 100644 --- a/scripts/run-gates.ts +++ b/scripts/run-gates.ts @@ -8,15 +8,9 @@ import { spawn } from 'node:child_process' import { availableParallelism } from 'node:os' import { resolve } from 'node:path' import { performance } from 'node:perf_hooks' -import { coverageArgs } from './coverage-shards.ts' -import { selectLintShard } from './lint-shards.ts' -import { selectSnapshotLane } from './snapshot-shards.ts' -import { selectStaticGates } from './static-shards.ts' type Mode = | 'ci-primary' - | 'ci-primary-cpu' - | 'ci-primary-large-runner' | 'ci-static' | 'ci-lint' | 'ci-coverage' @@ -92,8 +86,6 @@ if (results.some(result => result.gate.allowFailure !== true && (result.status = function parseMode(raw: string | undefined): Mode { switch (raw) { case 'ci-primary': - case 'ci-primary-cpu': - case 'ci-primary-large-runner': case 'ci-static': case 'ci-lint': case 'ci-coverage': @@ -107,7 +99,7 @@ function parseMode(raw: string | undefined): Mode { return raw default: throw new Error( - `run-gates: expected mode ci-primary | ci-primary-cpu | ci-primary-large-runner | ci-static | ci-lint | ci-coverage | ci-snapshot | ci-artifacts | ci-windows-blocking | ci-windows-complete | ci-windows-observational | node-compat | doc-sync, got ${JSON.stringify(raw)}.`, + `run-gates: expected mode ci-primary | ci-static | ci-lint | ci-coverage | ci-snapshot | ci-artifacts | ci-windows-blocking | ci-windows-complete | ci-windows-observational | node-compat | doc-sync, got ${JSON.stringify(raw)}.`, ) } } @@ -173,25 +165,17 @@ function gatesForMode(selected: Mode): Gate[] { switch (selected) { case 'ci-primary': return ciPrimaryGates() - case 'ci-primary-cpu': - return ciPrimaryCpuGates() - case 'ci-primary-large-runner': - return ciPrimaryLargeRunnerGates() case 'ci-static': return ciStaticGates() - case 'ci-lint': { - const selection = selectLintShard(process.env.DSH_LINT_SHARD) + case 'ci-lint': return [ - lintGate(selection.eslintTargets), - ...selection.includeDuplication ? [pnpmScript('duplication', 'duplication')] : [], + lintGate(), + pnpmScript('duplication', 'duplication'), ] - } case 'ci-coverage': return [coverageGate()] case 'ci-snapshot': - return flagEnabled('DSH_SNAPSHOT_PREBUILT') - ? [snapshotGate([])] - : [pnpmScript('build', 'build'), snapshotGate()] + return [pnpmScript('build', 'build'), snapshotGate()] case 'ci-artifacts': return ciArtifactGates() case 'ci-windows-blocking': @@ -217,11 +201,12 @@ function ciPrimaryGates(): Gate[] { lintGate(), pnpmScript('duplication', 'duplication'), coverageGate(), + ...nodeCompatSmokeGates(), snapshotGate(), ...docSyncLeafGates(), pnpmScript('module-graph', 'verify-module-graph', { label: 'module graph' }), pnpmScript('knip', 'knip'), - pnpmScript('build', 'build', { needs: ['typecheck'] }), + pnpmScript('build', 'build'), pnpmScript('publint', 'publint', { needs: ['build'] }), pnpmScript('node-next-types', 'verify-node-next-types', { label: 'node-next types', @@ -232,30 +217,6 @@ function ciPrimaryGates(): Gate[] { ] } -function ciPrimaryLargeRunnerGates(): Gate[] { - // The CPU lane owns typecheck, coverage, and the build-to-snapshot chain. - // This core lane starts its own build eagerly for the remaining artifact consumers. - return ciPrimaryGates() - .filter(gate => !['coverage', 'docs-site-build', 'snapshot', 'typecheck'].includes(gate.id)) - .map((gate) => { - if (gate.id !== 'build') return gate - const eagerBuild = { ...gate } - delete eagerBuild.needs - return eagerBuild - }) -} - -function ciPrimaryCpuGates(): Gate[] { - // Build and snapshot stay together so the dependent replay consumes this lane's output. - return [ - pnpmScript('typecheck', 'typecheck'), - coverageGate(), - pnpmScript('build', 'build'), - snapshotGate(), - ...nodeCompatSmokeGates(), - ] -} - function nodeCompatGates(): Gate[] { return [ ...flagEnabled('DSH_NODE_COMPAT_SKIP_TYPECHECK') ? [] : [pnpmScript('typecheck', 'typecheck')], @@ -279,7 +240,7 @@ function nodeCompatSmokeGates(): Gate[] { } function ciStaticGates(): Gate[] { - const gates = [ + return [ pnpmScript('runtime-closure', 'verify-runtime-closure', { label: 'runtime closure' }), pnpmScript('constraints', 'constraints'), pnpmScript('package-invariants', 'verify-package-invariants', { label: 'package invariants' }), @@ -293,12 +254,10 @@ function ciStaticGates(): Gate[] { pnpmScript('module-graph', 'verify-module-graph', { label: 'module graph' }), pnpmScript('knip', 'knip'), ] - return selectStaticGates(gates, process.env.DSH_STATIC_SHARD) } function ciArtifactGates(): Gate[] { - const shard = process.env.DSH_ARTIFACT_SHARD - const metadataGates = [ + return [ pnpmScript('build', 'build'), pnpmScript('publint', 'publint', { needs: ['build'] }), pnpmScript('node-next-types', 'verify-node-next-types', { @@ -306,13 +265,8 @@ function ciArtifactGates(): Gate[] { needs: ['build'], }), builtPackageInvariantsGate(['build']), + builtBinSmokeGate(), ] - if (shard === 'metadata') return metadataGates - if (shard === 'smoke') return [pnpmScript('build', 'build'), builtBinSmokeGate()] - if (shard !== undefined && shard !== '') { - throw new Error(`run-gates: unknown DSH_ARTIFACT_SHARD ${JSON.stringify(shard)}.`) - } - return [...metadataGates, builtBinSmokeGate()] } function ciWindowsBlockingGates(): Gate[] { @@ -396,12 +350,10 @@ function eslintConcurrencyArgs(): string[] { } function coverageGate(): Gate { - const shard = process.env.DSH_COVERAGE_SHARD return pnpmExec('coverage', [ 'vitest', 'run', '--coverage', - ...(shard === undefined || shard === '' ? [] : coverageArgs(shard)), ...positiveIntArg('DSH_COVERAGE_MAX_WORKERS', '--maxWorkers'), ], { label: 'test:coverage', @@ -409,23 +361,12 @@ function coverageGate(): Gate { } // The snapshot suite boots the example bins in `lib` mode (built artifact under plain Node, -// plugins via real exports). CI normally pairs it with `build`, so it exercises what ships rather -// than the tsx/source path dev uses; callers with prebuilt output may omit that dependency. -function snapshotGate(needs: string[] = ['build']): Gate { - const lane = selectSnapshotLane(process.env.DSH_SNAPSHOT_LANE) - return pnpmExec('snapshot', [ - 'vitest', - 'run', - '--config', - 'vitest.snapshot.config.ts', - ...lane.files, - ], { - label: 'test:snapshot', - env: { - DSH_EXAMPLE_MODE: 'lib', - ...lane.scenarioShard === undefined ? {} : { DSH_SNAPSHOT_SCENARIO_SHARD: lane.scenarioShard }, - }, - ...needs.length === 0 ? {} : { needs }, +// plugins via real exports). CI pairs it with `build`, so it exercises what ships rather than +// the tsx/source path dev uses and therefore waits on `build`. +function snapshotGate(): Gate { + return pnpmScript('snapshot', 'test:snapshot', { + env: { DSH_EXAMPLE_MODE: 'lib' }, + needs: ['build'], }) } @@ -492,7 +433,7 @@ function docSyncLeafGates(options: { ] } -function builtBinSmokeGate(shard?: string): Gate { +function builtBinSmokeGate(): Gate { return pnpmExec('built-bin-smoke', [ 'vitest', 'run', @@ -508,7 +449,6 @@ function builtBinSmokeGate(shard?: string): Gate { // (the e2e lane runs unbuilt, so these files self-skip there). 'packages/workflow/workflow-workerthread/tests/built-worker.e2e.ts', 'packages/code-runtime/code-runtime-worker/tests/built-lib.e2e.ts', - ...(shard === undefined ? [] : [`--shard=${shard}`]), ], { label: 'built-bin smoke', needs: ['build'], diff --git a/scripts/snapshot-shards.spec.ts b/scripts/snapshot-shards.spec.ts deleted file mode 100644 index 1b8db7dc39..0000000000 --- a/scripts/snapshot-shards.spec.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { existsSync, readdirSync } from 'node:fs' -import { join, relative, sep } from 'node:path' -import { describe, expect, it } from 'vitest' -import { selectSnapshotLane, snapshotLanes } from './snapshot-shards.ts' - -const root = join(import.meta.dirname, '..') - -function snapshotFiles(dir: string): string[] { - if (!existsSync(dir)) return [] - return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { - const path = join(dir, entry.name) - if (entry.isDirectory()) return snapshotFiles(path) - return entry.name.endsWith('.snapshot.ts') ? [relative(root, path).split(sep).join('/')] : [] - }) -} - -describe('snapshot lanes', () => { - it('assigns every configured snapshot file and every ACP scenario shard', () => { - const discovered = [ - ...snapshotFiles(join(root, 'examples')), - ...snapshotFiles(join(root, 'packages/sdk')), - ...snapshotFiles(join(root, 'packages/ui/tui')), - ].filter(path => !path.includes('/node_modules/') && !path.includes('/lib/')).sort() - const ordinary = snapshotLanes.filter(lane => lane.scenarioShard === undefined).flatMap(lane => lane.files) - const acp = snapshotLanes.filter(lane => lane.scenarioShard !== undefined) - - expect(new Set(ordinary).size).toBe(ordinary.length) - expect(acp.map(lane => lane.files)).toEqual(Array.from( - { length: 8 }, - () => ['examples/acp-agent/tests/acp.snapshot.ts'], - )) - expect(acp.map(lane => lane.scenarioShard)).toEqual([ - '1/8', - '2/8', - '3/8', - '4/8', - '5/8', - '6/8', - '7/8', - '8/8', - ]) - expect([...ordinary, 'examples/acp-agent/tests/acp.snapshot.ts'].sort()).toEqual(discovered) - }) - - it('keeps ordinary runs complete and selects known lanes', () => { - expect(selectSnapshotLane()).toEqual({ name: 'complete', files: [] }) - expect(selectSnapshotLane('')).toEqual({ name: 'complete', files: [] }) - for (const lane of snapshotLanes) expect(selectSnapshotLane(lane.name)).toBe(lane) - }) - - it('rejects an unknown lane', () => { - expect(() => selectSnapshotLane('missing')).toThrow('unknown DSH_SNAPSHOT_LANE') - }) -}) diff --git a/scripts/snapshot-shards.ts b/scripts/snapshot-shards.ts deleted file mode 100644 index 5439be94fb..0000000000 --- a/scripts/snapshot-shards.ts +++ /dev/null @@ -1,49 +0,0 @@ -/** Snapshot-lane definitions for GitHub Actions. */ - -/** One explicit snapshot file lane, optionally split again by ACP scenarios. */ -export interface SnapshotLane { - /** Stable lane name passed through `DSH_SNAPSHOT_LANE`. */ - name: string - /** Snapshot test files owned by the lane. */ - files: readonly string[] - /** Optional one-based ACP scenario partition. */ - scenarioShard?: string -} - -/** Exhaustive file ownership plus scenario partitions for the large ACP suite. */ -export const snapshotLanes: readonly SnapshotLane[] = [ - { - name: 'support', - files: [ - 'packages/sdk/scripts/tests/config.snapshot.ts', - 'packages/sdk/create-sdk/tests/create.snapshot.ts', - 'packages/ui/tui/tests/tui.snapshot.ts', - ], - }, - { - name: 'agents', - files: [ - 'examples/tui-agent/tests/tui.snapshot.ts', - 'examples/acp-agent/tests/goal.snapshot.ts', - 'examples/headless-agent/tests/headless.snapshot.ts', - ], - }, - ...Array.from({ length: 8 }, (_, offset) => ({ - name: `acp-${offset + 1}`, - files: ['examples/acp-agent/tests/acp.snapshot.ts'], - scenarioShard: `${offset + 1}/8`, - })), -] - -/** - * Resolve one CI lane while preserving a complete ordinary snapshot run. - * - * @param name Optional stable lane name. - * @returns An empty file list for the full suite, or one explicit CI lane. - */ -export function selectSnapshotLane(name?: string): SnapshotLane { - if (name === undefined || name === '') return { name: 'complete', files: [] } - const lane = snapshotLanes.find(candidate => candidate.name === name) - if (lane === undefined) throw new Error(`run-gates: unknown DSH_SNAPSHOT_LANE ${JSON.stringify(name)}.`) - return lane -} diff --git a/scripts/static-shards.spec.ts b/scripts/static-shards.spec.ts deleted file mode 100644 index df9e2922f1..0000000000 --- a/scripts/static-shards.spec.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { selectStaticGates, staticShards } from './static-shards.ts' - -const completeInventory = staticShards.flatMap(shard => shard.gateIds).map(id => ({ id })) - -describe('static gate shards', () => { - it.each(staticShards)('selects only the gates owned by $name', (shard) => { - expect(selectStaticGates(completeInventory, shard.name).map(gate => gate.id)).toEqual(shard.gateIds) - }) - - it('selects multiple lanes in gate inventory order', () => { - const selectedNames = new Set(['foundation', 'catalogs', 'prose']) - const expected = staticShards - .filter(shard => selectedNames.has(shard.name)) - .flatMap(shard => shard.gateIds) - expect(selectStaticGates(completeInventory, 'foundation,catalogs,prose').map(gate => gate.id)).toEqual(expected) - }) - - it('rejects missing, duplicate, and unknown assignments', () => { - expect(() => selectStaticGates(completeInventory.slice(1))).toThrow('assign every static gate exactly once') - expect(() => selectStaticGates([...completeInventory, completeInventory[0]!])).toThrow('static gate IDs must be unique') - expect(() => selectStaticGates(completeInventory, 'missing')).toThrow('unknown DSH_STATIC_SHARD') - expect(() => selectStaticGates(completeInventory, 'foundation,foundation')).toThrow('nonempty and unique') - expect(() => selectStaticGates(completeInventory, 'foundation,,prose')).toThrow('nonempty and unique') - }) -}) diff --git a/scripts/static-shards.ts b/scripts/static-shards.ts deleted file mode 100644 index 3476099993..0000000000 --- a/scripts/static-shards.ts +++ /dev/null @@ -1,86 +0,0 @@ -/** Static-gate shard definitions for GitHub Actions. */ - -/** A static CI lane identified by the gate IDs it owns. */ -export interface StaticShard { - /** Stable lane identifier passed through `DSH_STATIC_SHARD`. */ - name: string - /** Gate IDs selected from the static gate inventory. */ - gateIds: readonly string[] -} - -/** Exhaustive, non-overlapping ownership of static CI gates. */ -export const staticShards = [ - { - name: 'foundation', - gateIds: [ - 'runtime-closure', - 'constraints', - 'package-invariants', - 'cordis-config', - 'module-graph', - 'knip', - ], - }, - { name: 'doc-types', gateIds: ['build', 'doc-typecheck'] }, - { - name: 'api-contracts', - gateIds: ['cordis-api', 'export-jsdoc', 'scoped-events', 'type-equivalence'], - }, - { - name: 'catalogs', - gateIds: ['cordis-catalog', 'tool-catalog', 'config-catalog', 'persistence-catalog', 'doc-graphs'], - }, - { - name: 'prose', - gateIds: [ - 'markdown-wrap', - 'markdown-links', - 'doc-refs', - 'package-paths', - 'package-readme-model-experience', - 'mermaid', - 'agent-note-classification', - 'agent-note-format', - 'translation-prompt', - 'translation-pairing', - 'doc-budgets', - 'package-readme-limitations', - ], - }, - { name: 'site-projection', gateIds: ['docs-site-projection'] }, - { name: 'site-build', gateIds: ['docs-site-build'] }, -] as const satisfies readonly StaticShard[] - -/** - * Validate the complete gate partition and optionally select one lane. - * - * @param gates Complete static gate inventory. - * @param name Optional comma-separated stable shard names. - * @returns All gates when no shard is requested, otherwise the selected lanes in inventory order. - */ -export function selectStaticGates(gates: readonly T[], name?: string): T[] { - const gateIds = gates.map(gate => gate.id) - const assignedIds = staticShards.flatMap(shard => shard.gateIds) - const uniqueGateIds = new Set(gateIds) - const uniqueAssignedIds = new Set(assignedIds) - if (uniqueGateIds.size !== gateIds.length) throw new Error('run-gates: static gate IDs must be unique.') - if (uniqueAssignedIds.size !== assignedIds.length) throw new Error('run-gates: static shard gate IDs must be unique.') - if (gateIds.length !== assignedIds.length - || gateIds.some(id => !uniqueAssignedIds.has(id)) - || assignedIds.some(id => !uniqueGateIds.has(id))) { - throw new Error('run-gates: static shards must assign every static gate exactly once.') - } - if (name === undefined || name === '') return [...gates] - - const shardNames = name.split(',') - if (shardNames.some(shardName => shardName === '') || new Set(shardNames).size !== shardNames.length) { - throw new Error(`run-gates: DSH_STATIC_SHARD names must be nonempty and unique, got ${JSON.stringify(name)}.`) - } - const selectedShards = shardNames.map((shardName) => { - const shard = staticShards.find(candidate => candidate.name === shardName) - if (shard === undefined) throw new Error(`run-gates: unknown DSH_STATIC_SHARD ${JSON.stringify(shardName)}.`) - return shard - }) - const selectedIds = new Set(selectedShards.flatMap(shard => shard.gateIds)) - return gates.filter(gate => selectedIds.has(gate.id)) -} From da5a1fc811e6043430b3f72a27bfc4e828cd0402 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 18:17:06 +0800 Subject: [PATCH 42/49] ci: separate master serial and PR jobs Skipped matrix jobs never expand matrix display values, so let GitHub use stable job identifiers. Bind optimized jobs to pull requests and the three serial references to master pushes. --- ...rial-cross-platform-ci-reference.i18n.yaml | 4 +-- ...7-21-serial-cross-platform-ci-reference.md | 12 ++++----- ...1-serial-cross-platform-ci-reference.zh.md | 12 ++++----- .github/workflows/ci.yml | 26 ++++++++----------- 4 files changed, 25 insertions(+), 29 deletions(-) diff --git a/.agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.i18n.yaml b/.agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.i18n.yaml index 5871fa1b07..9922018569 100644 --- a/.agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.i18n.yaml +++ b/.agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -2026-07-21-serial-cross-platform-ci-reference.md: e2cb9dbc5f8778b5a28236cb135f6dcbb979d38b -2026-07-21-serial-cross-platform-ci-reference.zh.md: 85c3403f1acb78e60e3a811503b5b264f84242c7 +2026-07-21-serial-cross-platform-ci-reference.md: ffc1fd5b37bc6c9e3427ee55a55300f93a1292f3 +2026-07-21-serial-cross-platform-ci-reference.zh.md: d7f87916865b83973abe6b0708203618cf536c8e diff --git a/.agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.md b/.agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.md index e2cb9dbc5f..ffc1fd5b37 100644 --- a/.agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.md +++ b/.agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.md @@ -6,7 +6,7 @@ English | [中文](2026-07-21-serial-cross-platform-ci-reference.zh.md) ## Problem -The pull-request workflow reaches its latency targets by partitioning static checks, lint, coverage, snapshot replay, and artifact validation across explicit GitHub jobs. Those partitions are exhaustively checked in code, but the optimized workflow still should not be its own only completeness oracle: a defect shared by shard selection and its inventory test could omit work while every optimized lane stays green. +The pull-request workflow reaches its latency targets by scheduling the complete primary Node inventory concurrently inside one larger runner. The optimized scheduler still should not be its own only completeness oracle: a defect in its gate inventory or dependency graph could omit work while the optimized job stays green. Encoding the one-minute non-Windows target and three-minute Windows target as job timeouts creates a separate failure mode. Hosted-runner startup and performance vary, so a correct gate can be cancelled at the target boundary before it emits useful diagnostics. The performance objective needs measurement against GitHub timestamps, while correctness needs enough time to finish. @@ -14,25 +14,25 @@ Reviewers also need a direct answer to a simpler question: what happens when the ## Decision -[CI](../../../../.github/workflows/ci.yml) accepts `workflow_dispatch` in addition to its normal push and pull-request events. A manual dispatch skips the optimized and compatibility jobs and exposes three explicit jobs named `serial / linux`, `serial / macos`, and `serial / windows`. They intentionally duplicate their short checkout, runtime setup, and immutable install sequences instead of hiding the operating systems behind a matrix or reusable workflow. +[CI](../../../../.github/workflows/ci.yml) gives pull-request and master-push events complementary responsibilities. Pull requests run only the optimized larger-runner and compatibility jobs. A push to `master` skips those jobs and runs three explicit references named `serial / linux`, `serial / macos`, and `serial / windows`. They intentionally duplicate their short checkout, runtime setup, and immutable install sequences instead of hiding the operating systems behind a matrix or reusable workflow. `workflow_dispatch` is reserved for runner benchmarks. Each reference job runs `pnpm run check:ci` without any shard selector. `DSH_GATE_CONCURRENCY=1` makes the top-level aggregate execute one ready gate at a time; coverage, snapshot replay, built-bin smoke, and publication validation also receive worker counts of one. The three operating-system jobs may run beside one another, but each host's repository gates are serial and complete. Linux installs bubblewrap before replaying snapshots, and Windows enables Developer Mode before installing the symlinked workspace. -Manual reference jobs are diagnostic and do not participate in the required `all checks passed` result. Pull-request and push events continue to run only the optimized lanes. The one-minute non-Windows and three-minute Windows objectives are evaluated from completed hosted-job timestamps and reported as measurements; they are not `timeout-minutes` values. +Master reference jobs are diagnostic and do not participate in the pull request's required `all checks passed` result. A pull request runs only the optimized jobs; a master push runs only the three serial references. The one-minute non-Windows and three-minute Windows objectives are evaluated from completed hosted-job timestamps and reported as measurements; they are not `timeout-minutes` values. The portable reference uses GitHub's standard `ubuntu-latest`, `macos-latest`, and `windows-2025` labels. A higher-core hosted runner remains a possible future benchmark, but it is not the default: larger runners require organization-owned labels and provisioning, while a reference oracle should remain runnable without repository-external runner configuration. Provisioning one later can change the performance experiment without changing this correctness baseline. ## Alternatives considered - **Set each timeout equal to its latency target** - rejected because scheduling variance would cancel correct work and suppress the evidence needed to diagnose a regression. -- **Trust only the optimized shard inventory** - rejected because selection and validation share implementation assumptions; an unsharded aggregate is an independent completeness check. +- **Trust only the concurrent primary inventory** - rejected because scheduling and validation share implementation assumptions; a serial aggregate is an independent completeness check. - **Run the serial references on every pull request** - rejected because they deliberately trade wall time and runner consumption for simplicity and are not needed in the fast feedback loop. - **Use one operating-system matrix** - rejected because three named jobs make the reference surface visible without another selection mechanism. -- **Move the fast workflow to larger runners now** - rejected as the portable default because it would couple ordinary CI to organization-specific runner capacity. It remains an opt-in experiment after such capacity has an owned label and budget. +- **Run the serial reference on larger runners** - rejected because the reference is the portable fallback for the organization-specific pull-request topology. The fast pull-request path uses provisioned larger runners; the serial master path keeps standard labels. ## Consequences -The workflow contains duplicated setup steps and a manual reference run can take much longer than the optimized pull-request path. That duplication is deliberate: reviewers can inspect each operating system's complete command without resolving a matrix or shard inventory. +The workflow contains duplicated setup steps and a master reference run can take much longer than the optimized pull-request path. That duplication is deliberate: reviewers can inspect each operating system's complete command without resolving a matrix or concurrent scheduler. The reference may expose platform failures that the optimized blocking set does not yet claim to support, especially on Windows. Such a failure is evidence about current cross-platform behavior rather than a reason to weaken or silently skip the aggregate. diff --git a/.agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.zh.md b/.agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.zh.md index 85c3403f1a..d7f8791686 100644 --- a/.agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.zh.md +++ b/.agents/notes/implemented/process/2026-07-21-serial-cross-platform-ci-reference.zh.md @@ -6,7 +6,7 @@ Status: implemented ## 问题 -拉取请求工作流通过把静态检查、lint、覆盖率、快照回放和产物验证拆分到显式的 GitHub 作业中来达到延迟目标。这些分区由代码穷举校验，但优化工作流仍不应成为自身唯一的完整性判定基准：如果分片选择逻辑及其清单测试存在同一缺陷，即使所有优化通道都保持绿灯，也可能漏掉部分工作。 +拉取请求工作流通过在一台更大型运行器内并发调度完整的主 Node 门禁清单来达到延迟目标。优化调度器仍不应成为自身唯一的完整性判定基准：如果其门禁清单或依赖图存在缺陷，即使优化作业保持绿灯，也可能漏掉部分工作。 将非 Windows 作业的 1 分钟目标和 Windows 作业的 3 分钟目标写成作业超时，会引入另一种失败模式。托管运行器的启动时间和性能会波动，因此即使门禁本身正确，也可能在到达目标时间边界时被取消，来不及输出有用的诊断信息。性能目标需要根据 GitHub 时间戳衡量，而正确性验证需要给门禁留足完成时间。 @@ -14,25 +14,25 @@ Status: implemented ## 决策 -[CI](../../../../.github/workflows/ci.yml) 除正常的 push 和拉取请求事件外，也接受 `workflow_dispatch`。手动触发时会跳过优化作业和兼容性作业，只提供三个名为 `serial / linux`、`serial / macos` 和 `serial / windows` 的显式作业。这些作业有意分别重复简短的代码检出、运行时设置和依赖锁定的安装步骤，不用矩阵或可复用工作流把操作系统差异隐藏起来。 +[CI](../../../../.github/workflows/ci.yml) 为拉取请求事件与 master 推送事件赋予互补的职责。拉取请求只运行使用更大型运行器的优化作业和兼容性作业。向 `master` 推送时会跳过这些作业，改为运行三个显式参考作业，名称分别为 `serial / linux`、`serial / macos` 和 `serial / windows`。这些作业有意分别重复简短的代码检出、运行时设置和依赖锁定的安装步骤，不用矩阵或可复用工作流把操作系统差异隐藏起来。`workflow_dispatch` 仅用于运行器基准测试。 每个参考作业均在不设置任何分片选择器的情况下运行 `pnpm run check:ci`。`DSH_GATE_CONCURRENCY=1` 使顶层聚合每次只执行一个已经就绪的门禁；覆盖率、快照回放、built-bin 冒烟测试和发布验证的并发数也设为 1。三种操作系统的作业可以彼此并行，但每台主机上的仓库门禁都串行运行且完整执行。Linux 在回放快照前安装 bubblewrap，Windows 则在安装采用符号链接的工作区前启用开发人员模式。 -手动参考作业仅用于诊断，不参与必需的 `all checks passed` 结果。拉取请求和 push 事件仍只运行优化通道。系统根据已完成托管作业的时间戳评估非 Windows 作业的 1 分钟目标和 Windows 作业的 3 分钟目标，并将其报告为测量结果，而不是写成 `timeout-minutes` 值。 +master 分支的参考作业仅用于诊断，不参与拉取请求所要求的 `all checks passed` 结果。拉取请求只运行优化作业；向 master 推送时只运行三个串行参考作业。系统根据已完成托管作业的时间戳评估非 Windows 作业的 1 分钟目标和 Windows 作业的 3 分钟目标，并将其报告为测量结果，而不是写成 `timeout-minutes` 值。 可移植的参考流程使用 GitHub 标准的 `ubuntu-latest`、`macos-latest` 和 `windows-2025` 标签。仍可将更高核心数的托管运行器作为未来的基准测试，但不将其设为默认选择：更大型运行器需要组织自有的标签和预配，而参考判定基准应无需仓库外部的运行器配置即可运行。日后完成这类预配，可以改变性能实验而无需改变该正确性基线。 ## 曾考虑的替代方案 - **将每个超时值设为相应延迟目标**：不予采纳，因为调度波动会中止原本正确的执行，并使诊断回归所需的证据无法产生。 -- **仅信任优化分片清单**：不予采纳，因为选择逻辑与校验逻辑共享实现假设；无分片的聚合流程是一项独立的完整性检查。 +- **仅信任并发执行的主门禁清单**：不予采纳，因为调度逻辑与校验逻辑共享实现假设；串行聚合流程是一项独立的完整性检查。 - **在每个拉取请求上运行串行参考作业**：不予采纳，因为这些作业有意以更长的总耗时和更多运行器用量换取简单性，快速反馈循环不需要它们。 - **使用一个操作系统矩阵**：不予采纳，因为三个具名作业无需另一套选择机制，就能让参考流程的构成清晰可见。 -- **立即把快速工作流迁移到更大型运行器**：不作为可移植的默认方案，因为这会让常规 CI 与特定组织的运行器容量耦合。等到这类容量拥有明确归属的标签和预算后，仍可将其作为一项可选实验。 +- **在更大型运行器上运行串行参考流程**：不予采纳，因为该参考流程是特定组织拉取请求拓扑的可移植后备方案。快速拉取请求路径使用已预配的更大型运行器；串行 master 路径保留标准标签。 ## 后果 -工作流包含重复的设置步骤，手动参考运行也可能比优化后的拉取请求路径耗时长得多。这些重复是有意保留的：评审人无需解析矩阵或分片清单，就能直接检查每种操作系统执行的完整命令。 +工作流包含重复的设置步骤，master 参考运行也可能比优化后的拉取请求路径耗时长得多。这些重复是有意保留的：评审人无需解析矩阵或并发调度器，就能直接检查每种操作系统执行的完整命令。 参考流程可能暴露某些平台上的故障，而优化后的阻塞门禁集合尚未声明支持这些平台，Windows 尤其如此。这类失败反映了当前的跨平台行为，不应成为削弱或静默跳过该聚合流程的理由。 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c6bb9fe627..a7ea422056 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,20 +2,18 @@ name: CI on: push: - branches: [main, master] + branches: [master] pull_request: workflow_dispatch: inputs: suite: description: Manual CI suite to run required: true - default: serial-reference + default: larger-runner-benchmark type: choice options: - - serial-reference - larger-runner-benchmark - consolidated-runner-benchmark - - optimized-larger-runners concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -33,7 +31,7 @@ jobs: # overlaps the complete unsharded primary Node inventory. Build starts eagerly; # only consumers of emitted output wait for it. node-24: - if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' + if: github.event_name == 'pull_request' runs-on: dsh-ubuntu-24-04-96core name: node 24 / complete env: @@ -88,7 +86,7 @@ jobs: node-compat: - if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' + if: github.event_name == 'pull_request' # Distinct larger-runner pools avoid both standard-runner setup outliers and # delayed allocation when independent environment contracts share one pool. runs-on: ${{ matrix.runner }} @@ -135,7 +133,7 @@ jobs: run: pnpm run check:node-compat python-sdk: - if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' + if: github.event_name == 'pull_request' runs-on: dsh-ubuntu-24-04-8core name: python 3.10 / keyless SDK steps: @@ -156,7 +154,7 @@ jobs: # complete observational portability inventory. run-gates reports failures # from observational gates without allowing them to fail the required job. windows: - if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' + if: github.event_name == 'pull_request' runs-on: dsh-windows-2025-32core name: windows node 24 / complete env: @@ -208,12 +206,12 @@ jobs: shell: pwsh run: pnpm run check:ci:windows-complete - # Manual reference runs deliberately avoid the optimized jobs above. + # Master pushes run only the serial reference jobs below. # Each host executes the complete, unsharded primary Node aggregate with one # gate worker, giving reviewers a simple cross-platform oracle for completeness # and timing. serial-linux: - if: github.event_name == 'workflow_dispatch' && inputs.suite == 'serial-reference' + if: github.event_name == 'push' && github.ref == 'refs/heads/master' name: serial / linux runs-on: ubuntu-latest steps: @@ -242,7 +240,7 @@ jobs: run: pnpm run check:ci serial-macos: - if: github.event_name == 'workflow_dispatch' && inputs.suite == 'serial-reference' + if: github.event_name == 'push' && github.ref == 'refs/heads/master' name: serial / macos runs-on: macos-latest steps: @@ -268,7 +266,7 @@ jobs: run: pnpm run check:ci serial-windows: - if: github.event_name == 'workflow_dispatch' && inputs.suite == 'serial-reference' + if: github.event_name == 'push' && github.ref == 'refs/heads/master' name: serial / windows runs-on: windows-2025 steps: @@ -306,7 +304,6 @@ jobs: # The named pools are restricted at the organization level to this repository. larger-runner-benchmark: if: github.event_name == 'workflow_dispatch' && inputs.suite == 'larger-runner-benchmark' - name: larger runner / ${{ matrix.platform }} / ${{ matrix.cores }} cores / ${{ matrix.workload }} runs-on: ${{ matrix.runner }} timeout-minutes: 15 strategy: @@ -407,7 +404,6 @@ jobs: # Windows runs both blocking build surfaces concurrently through run-gates. consolidated-runner-benchmark: if: github.event_name == 'workflow_dispatch' && inputs.suite == 'consolidated-runner-benchmark' - name: consolidated / ${{ matrix.platform }} / ${{ matrix.cores }} cores runs-on: ${{ matrix.runner }} timeout-minutes: 15 strategy: @@ -556,7 +552,7 @@ jobs: name: all checks passed runs-on: ubuntu-latest needs: [node-24, node-compat, python-sdk, windows] - if: always() && (github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners') + if: always() && github.event_name == 'pull_request' steps: - name: Fail if any needed job did not succeed if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'skipped') From cbbef79fa3374386e9f3639ca3d4dd947cef111a Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 19:00:55 +0800 Subject: [PATCH 43/49] ci: retune consolidated jobs for stacked workload --- ...ence-based-larger-hosted-runners.i18n.yaml | 4 +- ...22-evidence-based-larger-hosted-runners.md | 8 +- ...evidence-based-larger-hosted-runners.zh.md | 8 +- .github/workflows/ci.yml | 77 ++++++++++++------- 4 files changed, 61 insertions(+), 36 deletions(-) diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml index d2114f1bd0..d151088a0a 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -2026-07-22-evidence-based-larger-hosted-runners.md: bb10a53058a7ae1a46d493933868887bb2f55265 -2026-07-22-evidence-based-larger-hosted-runners.zh.md: ffaf958560dae6e044da58f910c0f53e21789344 +2026-07-22-evidence-based-larger-hosted-runners.md: 6170aca3c2675069a74d951b11273ed5d2731de8 +2026-07-22-evidence-based-larger-hosted-runners.zh.md: bd2095b17bdb954382b455bd160405a10be56ddb diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md index bb10a53058..6170aca3c2 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md @@ -16,9 +16,9 @@ The organization keeps twelve x64 larger-runner pools in the repo-restricted `ds Production CI uses five larger-runner executions and one standard-runner aggregator. The primary Node inventory is not sharded: -- `node 24 / complete` uses one 96-core Linux runner. One checkout, setup, cache restore, and install feeds all 42 primary gates. `run-gates` starts up to 32 independent gates; ESLint uses 32 workers, coverage uses at most 12, and snapshot replay uses at most 16. Build starts immediately beside typecheck, coverage, lint, and documentation work, while snapshot replay and publication consumers retain explicit dependencies on emitted `lib/` output. +- `node 24 / complete` uses one 96-core Linux runner. One checkout, setup, cache restore, and install feeds all 42 primary gates. `run-gates` starts up to 16 independent gates; ESLint uses 16 workers, coverage uses at most 12, and snapshot replay uses at most 8. Build starts as soon as the first short gates release scheduler slots, while snapshot replay and publication consumers retain explicit dependencies on emitted `lib/` output. Pull requests restore pnpm and ESLint caches without saving them, so cache compression and upload do not extend the required job; the master serial references refresh those caches outside the pull-request critical path. - Node 22.19 and Node 26 use the 4- and 32-core Linux pools for their runtime compatibility smokes. Python 3.10 uses the 8-core Linux pool for the complete keyless SDK suite. These are environment contracts, not slices of the primary Node gate inventory. -- `windows node 24 / complete` uses one 32-core Windows runner. One setup feeds the required package build, required production site build, and complete observational portability inventory. Required failures fail the job; observational failures are reported as non-blocking. ESLint stays single-threaded because 16 ESLint workers took 174.54 seconds, while the outer scheduler retains 32 slots. +- `windows node 24 / complete` uses one 32-core Windows runner. One setup feeds the required package build, required production site build, and complete observational portability inventory. Required failures fail the job; observational failures are reported as non-blocking. ESLint stays single-threaded because 16 ESLint workers took 174.54 seconds, coverage uses at most 12 workers, and the outer scheduler retains 16 slots. The job restores only the small master-refreshed ESLint cache and performs a clean pnpm install instead of restoring or saving the many-file package store. The former gate-level and coarse primary shard jobs are absent from the workflow. Their static, lint, coverage, snapshot, and scenario shard selectors are also absent from the repository, so an unused diagnostic path cannot preserve a second CI architecture. @@ -38,11 +38,13 @@ The same benchmark measured the required Windows build surfaces across every pro Repository work gains little above 16 Windows cores, but the 32-core pool can start the complete outer inventory together. A [retargeted production validation](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29907581119/attempts/2) completed the full one-box Windows inventory in 173 seconds, including coverage and snapshot replay, so Windows remains consolidated. +The larger client package graph makes cache mechanics and scheduler pressure part of the measured workload. In [one exact-head production run](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29912577681), Linux spent 39 seconds in repository gates but 69 seconds in the complete job, while Windows spent 117 seconds in repository gates and 228 seconds in the complete job. The Windows pnpm cache downloaded its 154 MB archive in about two seconds but spent 27 seconds extracting it, followed by a 23-second install and a 14-second post-job save. A [cacheless all-size trace](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29913033155) completed the same 32-core Windows install in 27 seconds. Production therefore avoids the Windows package-store cache, uses restore-only caches on latency-critical pull-request jobs, and bounds outer concurrency so typecheck, lint, coverage, and build do not oversubscribe one host. + Two host effects remain part of the decision. A standard Node 26 job once spent 36 of its 67 seconds in `Set up job`, which is why environment contracts use distinct larger-runner pools instead of standard capacity. A Linux candidate spent 18 seconds registering a 50 KB Bubblewrap package because the hosted image scanned 202,507 package-database files. [`scripts/prepare-ci-bubblewrap.sh`](../../../../scripts/prepare-ci-bubblewrap.sh) instead verifies and extracts the pinned payload into the ephemeral runner directory, runs a functional confinement probe, and overlaps that preparation with dependency installation. Coverage remains capped at 12 forks. Thirty-two forks crashed Node 24's CJS lexer twice, and a later 16-fork run reproduced the worker loss and invalid coverage result. Twelve forks preserve process headroom without becoming the single-host critical path. -The workflow retains three manual measurement suites. `suite=larger-runner-benchmark` compares isolated critical lanes across every size, `suite=consolidated-runner-benchmark` compares whole aggregates, and `suite=serial-reference` remains the unsharded cross-platform completeness oracle. `suite=optimized-larger-runners` dispatches the exact production topology against a branch ref when a pull request cannot form a merge commit. +The workflow retains two manual measurement suites. `suite=larger-runner-benchmark` compares isolated critical lanes across every size, and `suite=consolidated-runner-benchmark` compares whole aggregates. Complete serial Linux, macOS, and Windows references run only when `master` moves; pull requests run only the optimized jobs. ## Alternatives considered diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md index ffaf958560..bd2095b17b 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md @@ -16,9 +16,9 @@ Status: implemented 生产 CI 包含 5 次大型运行器执行和 1 个标准运行器聚合作业。主 Node 门禁清单不再分片： -- `node 24 / complete` 使用一台 96 核 Linux 运行器。只需执行一次代码检出、设置、缓存恢复和安装，即可供全部 42 项主门禁使用。`run-gates` 最多同时启动 32 项相互独立的门禁；ESLint 使用 32 个工作线程，覆盖率最多使用 12 个，快照回放最多使用 16 个。构建会立即与类型检查、覆盖率、lint 和文档工作同时启动，而快照回放和发布消费方仍显式依赖生成的 `lib/` 输出。 +- `node 24 / complete` 使用一台 96 核 Linux 运行器。只需执行一次代码检出、设置、缓存恢复和安装，即可供全部 42 项主门禁使用。`run-gates` 最多同时启动 16 项相互独立的门禁；ESLint 使用 16 个工作线程，覆盖率最多使用 12 个，快照回放最多使用 8 个。第一批短门禁释放调度器槽位后，构建会立即启动，而快照回放和发布消费方仍显式依赖生成的 `lib/` 输出。拉取请求会恢复 pnpm 和 ESLint 缓存但不保存，因此缓存压缩和上传不会延长必需作业；master 上的串行参考会在拉取请求关键路径之外刷新这些缓存。 - Node 22.19 和 Node 26 分别使用 4 核和 32 核 Linux 池运行各自的运行时兼容性冒烟测试。Python 3.10 使用 8 核 Linux 池运行完整的无密钥 SDK 套件。这些作业属于环境契约，并非主 Node 门禁清单的分片。 -- `windows node 24 / complete` 使用一台 32 核 Windows 运行器。一次设置供必需的包构建、必需的生产网站构建以及完整的观测性可移植性清单共用。任何必需项失败都会使作业失败；观测项失败则报告为非阻塞。ESLint 保持单线程，因为 16 个 ESLint 工作线程耗时 174.54 秒，外层调度器则保留 32 个槽位。 +- `windows node 24 / complete` 使用一台 32 核 Windows 运行器。一次设置供必需的包构建、必需的生产网站构建以及完整的观测性可移植性清单共用。任何必需项失败都会使作业失败；观测项失败则报告为非阻塞。ESLint 保持单线程，因为 16 个 ESLint 工作线程耗时 174.54 秒；覆盖率最多使用 12 个工作线程，外层调度器则保留 16 个槽位。该作业仅恢复由 master 刷新的较小 ESLint 缓存，并在干净环境中执行 pnpm 安装，而不恢复或保存包含大量文件的包存储。 原有的门禁级和粗粒度主流程分片作业已从工作流中移除。相应的静态、lint、覆盖率、快照和场景分片选择器也已从仓库中移除，因此未使用的诊断路径无法继续维系第二套 CI 架构。 @@ -38,11 +38,13 @@ Status: implemented Windows 仓库工作在超过 16 核后收益很小，但 32 核池可以让完整的外层清单同时启动。一次[重新定向的生产验证](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29907581119/attempts/2)在 173 秒内完成了单机 Windows 完整清单，其中包括覆盖率和快照回放，因此 Windows 继续采用合并执行方式。 +客户端包依赖图增大后，缓存机制和调度器压力也成为实测工作负载的一部分。在[一次分支头精确的生产运行](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29912577681)中，Linux 的仓库门禁耗时 39 秒，完整作业耗时 69 秒；Windows 的仓库门禁耗时 117 秒，完整作业耗时 228 秒。Windows pnpm 缓存的 154 MB 归档下载耗时约 2 秒，但解压耗时 27 秒，随后安装耗时 23 秒，作业结束后的保存又耗时 14 秒。一次[无缓存的全规格运行轨迹](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29913033155)在 27 秒内完成了同一台 32 核 Windows 运行器上的安装。因此，生产环境不使用 Windows 包存储缓存，在对延迟敏感的拉取请求作业中使用只恢复不保存的缓存，并限制外层并发度，以免类型检查、lint、覆盖率和构建在同一台主机上过度争用资源。 + 两项主机效应仍构成这项决策的依据。一个标准 Node 26 作业曾在总共 67 秒的耗时中，把 36 秒用在 `Set up job` 上，因此各项环境契约使用不同的大型运行器池，而非标准容量。一个 Linux 候选作业在注册 50 KB 的 Bubblewrap 包时耗时 18 秒，因为托管映像扫描了 202,507 个包数据库文件。[`scripts/prepare-ci-bubblewrap.sh`](../../../../scripts/prepare-ci-bubblewrap.sh) 改为验证固定包内容并将其解压到临时运行器目录，执行功能性隔离探针，并让这项准备工作与依赖安装重叠执行。 覆盖率仍限制为 12 个 fork。32 个 fork 曾两次导致 Node 24 的 CJS 词法分析器崩溃，后来一次使用 16 个 fork 的运行又复现了工作进程丢失和无效的覆盖率结果。12 个 fork 能保留进程余量，同时不会成为单机关键路径。 -工作流保留 3 项手动测量套件。`suite=larger-runner-benchmark` 比较所有规格下相互独立的关键通道，`suite=consolidated-runner-benchmark` 比较完整聚合流程，`suite=serial-reference` 则继续作为未分片的跨平台完整性判定基准。当拉取请求无法生成合并提交时，`suite=optimized-larger-runners` 会直接针对分支引用触发与生产环境完全相同的拓扑。 +工作流保留 2 项手动测量套件。`suite=larger-runner-benchmark` 比较所有规格下相互独立的关键通道，`suite=consolidated-runner-benchmark` 比较完整聚合流程。只有在 `master` 移动时，才运行完整的 Linux、macOS 和 Windows 串行参考；拉取请求只运行优化后的作业。 ## 曾考虑的替代方案 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a7ea422056..deba685f81 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,10 +37,10 @@ jobs: env: DSH_COVERAGE_MAX_WORKERS: '12' DSH_ESLINT_CACHE: '1' - DSH_ESLINT_CONCURRENCY: '32' - DSH_GATE_CONCURRENCY: '32' - DSH_PUBLINT_CONCURRENCY: '32' - DSH_SNAPSHOT_MAX_CONCURRENCY: '16' + DSH_ESLINT_CONCURRENCY: '16' + DSH_GATE_CONCURRENCY: '16' + DSH_PUBLINT_CONCURRENCY: '16' + DSH_SNAPSHOT_MAX_CONCURRENCY: '8' steps: - uses: actions/checkout@v6 @@ -54,14 +54,16 @@ jobs: corepack enable echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" - - uses: actions/cache@v4 + # Pull requests consume the default-branch cache but do not put cache + # compression and upload on the paid latency-critical path. + - uses: actions/cache/restore@v4 with: path: ${{ steps.pnpm-store.outputs.path }} key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} restore-keys: | ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm- - - uses: actions/cache@v4 + - uses: actions/cache/restore@v4 with: path: .cache/eslint key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-full-${{ hashFiles('pnpm-lock.yaml', 'eslint.config.mjs', 'tsconfig.json', 'packages/*/*/tsconfig.json', 'examples/*/tsconfig.json') }} @@ -160,10 +162,10 @@ jobs: env: # Keep ESLint itself single-threaded: 16 ESLint workers took 174 seconds on # this image. The outer scheduler still overlaps lint with the other gates. - DSH_COVERAGE_MAX_WORKERS: '4' + DSH_COVERAGE_MAX_WORKERS: '12' DSH_ESLINT_CACHE: '1' - DSH_GATE_CONCURRENCY: '32' - DSH_PUBLINT_CONCURRENCY: '32' + DSH_GATE_CONCURRENCY: '16' + DSH_PUBLINT_CONCURRENCY: '16' steps: - uses: actions/checkout@v6 @@ -177,30 +179,20 @@ jobs: with: node-version: ${{ env.PRIMARY_NODE_VERSION }} - - name: Enable corepack and resolve pnpm store path - id: pnpm-store - shell: pwsh - run: | - corepack enable - "path=$(pnpm store path --silent)" >> $env:GITHUB_OUTPUT - - - uses: actions/cache@v4 - with: - path: ${{ steps.pnpm-store.outputs.path }} - key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm- - - - uses: actions/cache@v4 + - uses: actions/cache/restore@v4 with: path: .cache/eslint key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-full-${{ hashFiles('pnpm-lock.yaml', 'eslint.config.mjs', 'tsconfig.json', 'packages/*/*/tsconfig.json', 'examples/*/tsconfig.json') }} restore-keys: | ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-full- - - name: Install (immutable) + # Extracting the many-file pnpm store cache is slower on this image than + # a clean parallel install, and saving it adds more latency after gates. + - name: Enable corepack and install (immutable) shell: pwsh - run: pnpm install --frozen-lockfile + run: | + corepack enable + pnpm install --frozen-lockfile - name: Run blocking and observational Windows gates concurrently shell: pwsh @@ -221,8 +213,26 @@ jobs: with: node-version: ${{ env.PRIMARY_NODE_VERSION }} - - name: Enable corepack (pnpm) - run: corepack enable + - name: Enable corepack and resolve pnpm store path + id: pnpm-store + run: | + corepack enable + echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" + + # Master refreshes the caches that pull requests restore without saving. + - uses: actions/cache@v4 + with: + path: ${{ steps.pnpm-store.outputs.path }} + key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm- + + - uses: actions/cache@v4 + with: + path: .cache/eslint + key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-full-${{ hashFiles('pnpm-lock.yaml', 'eslint.config.mjs', 'tsconfig.json', 'packages/*/*/tsconfig.json', 'examples/*/tsconfig.json') }} + restore-keys: | + ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-full- - name: Install (immutable) run: pnpm install --frozen-lockfile @@ -234,6 +244,7 @@ jobs: env: DSH_COVERAGE_MAX_WORKERS: '1' DSH_E2E_MAX_WORKERS: '1' + DSH_ESLINT_CACHE: '1' DSH_GATE_CONCURRENCY: '1' DSH_PUBLINT_CONCURRENCY: '1' DSH_SNAPSHOT_MAX_CONCURRENCY: '1' @@ -286,6 +297,15 @@ jobs: shell: pwsh run: corepack enable + # Master refreshes the small cache that pull requests restore without + # putting package-store extraction back on the Windows critical path. + - uses: actions/cache@v4 + with: + path: .cache/eslint + key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-full-${{ hashFiles('pnpm-lock.yaml', 'eslint.config.mjs', 'tsconfig.json', 'packages/*/*/tsconfig.json', 'examples/*/tsconfig.json') }} + restore-keys: | + ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-full- + - name: Install (immutable) shell: pwsh run: pnpm install --frozen-lockfile @@ -295,6 +315,7 @@ jobs: env: DSH_COVERAGE_MAX_WORKERS: '1' DSH_E2E_MAX_WORKERS: '1' + DSH_ESLINT_CACHE: '1' DSH_GATE_CONCURRENCY: '1' DSH_PUBLINT_CONCURRENCY: '1' DSH_SNAPSHOT_MAX_CONCURRENCY: '1' From 02f6d919c8f00afd156e5089c122bff02e98412f Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 19:13:07 +0800 Subject: [PATCH 44/49] ci: remove primary setup overhead --- ...evidence-based-larger-hosted-runners.i18n.yaml | 4 ++-- ...-07-22-evidence-based-larger-hosted-runners.md | 2 +- ...-22-evidence-based-larger-hosted-runners.zh.md | 2 +- .github/workflows/ci.yml | 15 +++++++-------- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml index d151088a0a..b53f173de2 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -2026-07-22-evidence-based-larger-hosted-runners.md: 6170aca3c2675069a74d951b11273ed5d2731de8 -2026-07-22-evidence-based-larger-hosted-runners.zh.md: bd2095b17bdb954382b455bd160405a10be56ddb +2026-07-22-evidence-based-larger-hosted-runners.md: f395a2d702e7d0bf2ad74fe0125952da220f799c +2026-07-22-evidence-based-larger-hosted-runners.zh.md: 476c13573b2fc9484e5c20adb0c398be4dfe80d6 diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md index 6170aca3c2..f395a2d702 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md @@ -16,7 +16,7 @@ The organization keeps twelve x64 larger-runner pools in the repo-restricted `ds Production CI uses five larger-runner executions and one standard-runner aggregator. The primary Node inventory is not sharded: -- `node 24 / complete` uses one 96-core Linux runner. One checkout, setup, cache restore, and install feeds all 42 primary gates. `run-gates` starts up to 16 independent gates; ESLint uses 16 workers, coverage uses at most 12, and snapshot replay uses at most 8. Build starts as soon as the first short gates release scheduler slots, while snapshot replay and publication consumers retain explicit dependencies on emitted `lib/` output. Pull requests restore pnpm and ESLint caches without saving them, so cache compression and upload do not extend the required job; the master serial references refresh those caches outside the pull-request critical path. +- `node 24 / complete` uses one 96-core Linux runner. One checkout, setup, cache restore, and install feeds all 42 primary gates. `run-gates` starts up to 12 independent gates; ESLint uses 16 workers, coverage uses at most 12, and snapshot replay uses at most 8. Build starts as soon as the first short gates release scheduler slots, while snapshot replay and publication consumers retain explicit dependencies on emitted `lib/` output. Pull requests restore pnpm and ESLint caches without saving them, so cache compression and upload do not extend the required job; the master serial references refresh those caches outside the pull-request critical path. The read-only job does not persist checkout credentials and restores the stable pnpm store root directly, avoiding a separate store-discovery step. - Node 22.19 and Node 26 use the 4- and 32-core Linux pools for their runtime compatibility smokes. Python 3.10 uses the 8-core Linux pool for the complete keyless SDK suite. These are environment contracts, not slices of the primary Node gate inventory. - `windows node 24 / complete` uses one 32-core Windows runner. One setup feeds the required package build, required production site build, and complete observational portability inventory. Required failures fail the job; observational failures are reported as non-blocking. ESLint stays single-threaded because 16 ESLint workers took 174.54 seconds, coverage uses at most 12 workers, and the outer scheduler retains 16 slots. The job restores only the small master-refreshed ESLint cache and performs a clean pnpm install instead of restoring or saving the many-file package store. diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md index bd2095b17b..476c13573b 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md @@ -16,7 +16,7 @@ Status: implemented 生产 CI 包含 5 次大型运行器执行和 1 个标准运行器聚合作业。主 Node 门禁清单不再分片： -- `node 24 / complete` 使用一台 96 核 Linux 运行器。只需执行一次代码检出、设置、缓存恢复和安装，即可供全部 42 项主门禁使用。`run-gates` 最多同时启动 16 项相互独立的门禁；ESLint 使用 16 个工作线程，覆盖率最多使用 12 个，快照回放最多使用 8 个。第一批短门禁释放调度器槽位后，构建会立即启动，而快照回放和发布消费方仍显式依赖生成的 `lib/` 输出。拉取请求会恢复 pnpm 和 ESLint 缓存但不保存，因此缓存压缩和上传不会延长必需作业；master 上的串行参考会在拉取请求关键路径之外刷新这些缓存。 +- `node 24 / complete` 使用一台 96 核 Linux 运行器。只需执行一次代码检出、设置、缓存恢复和安装，即可供全部 42 项主门禁使用。`run-gates` 最多同时启动 12 项相互独立的门禁；ESLint 使用 16 个工作线程，覆盖率最多使用 12 个，快照回放最多使用 8 个。第一批短门禁释放调度器槽位后，构建会立即启动，而快照回放和发布消费方仍显式依赖生成的 `lib/` 输出。拉取请求会恢复 pnpm 和 ESLint 缓存但不保存，因此缓存压缩和上传不会延长必需作业；master 上的串行参考会在拉取请求关键路径之外刷新这些缓存。该只读作业不会持久化代码检出凭据，并直接恢复固定的 pnpm 存储根目录，因此无需再单独执行存储路径探测步骤。 - Node 22.19 和 Node 26 分别使用 4 核和 32 核 Linux 池运行各自的运行时兼容性冒烟测试。Python 3.10 使用 8 核 Linux 池运行完整的无密钥 SDK 套件。这些作业属于环境契约，并非主 Node 门禁清单的分片。 - `windows node 24 / complete` 使用一台 32 核 Windows 运行器。一次设置供必需的包构建、必需的生产网站构建以及完整的观测性可移植性清单共用。任何必需项失败都会使作业失败；观测项失败则报告为非阻塞。ESLint 保持单线程，因为 16 个 ESLint 工作线程耗时 174.54 秒；覆盖率最多使用 12 个工作线程，外层调度器则保留 16 个槽位。该作业仅恢复由 master 刷新的较小 ESLint 缓存，并在干净环境中执行 pnpm 安装，而不恢复或保存包含大量文件的包存储。 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index deba685f81..ffc95ea849 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,27 +38,25 @@ jobs: DSH_COVERAGE_MAX_WORKERS: '12' DSH_ESLINT_CACHE: '1' DSH_ESLINT_CONCURRENCY: '16' - DSH_GATE_CONCURRENCY: '16' + DSH_GATE_CONCURRENCY: '12' DSH_PUBLINT_CONCURRENCY: '16' DSH_SNAPSHOT_MAX_CONCURRENCY: '8' steps: - uses: actions/checkout@v6 + with: + persist-credentials: false - uses: actions/setup-node@v6 with: node-version: ${{ env.PRIMARY_NODE_VERSION }} - - name: Enable corepack and resolve pnpm store path - id: pnpm-store - run: | - corepack enable - echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" - # Pull requests consume the default-branch cache but do not put cache # compression and upload on the paid latency-critical path. - uses: actions/cache/restore@v4 with: - path: ${{ steps.pnpm-store.outputs.path }} + # Cache the versioned store through its stable parent so selecting + # the current pnpm store does not require a separate runner step. + path: ~/.local/share/pnpm/store key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} restore-keys: | ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm- @@ -72,6 +70,7 @@ jobs: - name: Install dependencies and prepare bubblewrap run: | + corepack enable pnpm install --frozen-lockfile & install_pid=$! bash scripts/prepare-ci-bubblewrap.sh & From 17983d72bb525dc44852ede33b20cf41c23dce15 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 19:37:45 +0800 Subject: [PATCH 45/49] ci: use process-safe coverage pools --- ...ence-based-larger-hosted-runners.i18n.yaml | 4 +- ...22-evidence-based-larger-hosted-runners.md | 4 +- ...evidence-based-larger-hosted-runners.zh.md | 4 +- .github/workflows/ci.yml | 11 +++-- vitest.config.ts | 44 ++++++++++++++++++- 5 files changed, 56 insertions(+), 11 deletions(-) diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml index b53f173de2..3561f39a33 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -2026-07-22-evidence-based-larger-hosted-runners.md: f395a2d702e7d0bf2ad74fe0125952da220f799c -2026-07-22-evidence-based-larger-hosted-runners.zh.md: 476c13573b2fc9484e5c20adb0c398be4dfe80d6 +2026-07-22-evidence-based-larger-hosted-runners.md: 2fac4d1c14e3f1eabef9f915e53ff737c0e7ee20 +2026-07-22-evidence-based-larger-hosted-runners.zh.md: 33ca3b823d2bf235f9896b92868bd7a1f0688a76 diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md index f395a2d702..2fac4d1c14 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md @@ -16,7 +16,7 @@ The organization keeps twelve x64 larger-runner pools in the repo-restricted `ds Production CI uses five larger-runner executions and one standard-runner aggregator. The primary Node inventory is not sharded: -- `node 24 / complete` uses one 96-core Linux runner. One checkout, setup, cache restore, and install feeds all 42 primary gates. `run-gates` starts up to 12 independent gates; ESLint uses 16 workers, coverage uses at most 12, and snapshot replay uses at most 8. Build starts as soon as the first short gates release scheduler slots, while snapshot replay and publication consumers retain explicit dependencies on emitted `lib/` output. Pull requests restore pnpm and ESLint caches without saving them, so cache compression and upload do not extend the required job; the master serial references refresh those caches outside the pull-request critical path. The read-only job does not persist checkout credentials and restores the stable pnpm store root directly, avoiding a separate store-discovery step. +- `node 24 / complete` uses one 96-core Linux runner. One checkout, setup, cache restore, and install feeds all 42 primary gates. `run-gates` starts up to 12 independent gates; ESLint uses 16 workers, coverage uses at most 12, and snapshot replay uses at most 8. Build starts as soon as the first short gates release scheduler slots, while snapshot replay and publication consumers retain explicit dependencies on emitted `lib/` output. Pull requests restore pnpm and ESLint caches without saving them, so cache compression and upload do not extend the required job; the master serial references refresh those caches outside the pull-request critical path. The read-only job does not persist checkout credentials. - Node 22.19 and Node 26 use the 4- and 32-core Linux pools for their runtime compatibility smokes. Python 3.10 uses the 8-core Linux pool for the complete keyless SDK suite. These are environment contracts, not slices of the primary Node gate inventory. - `windows node 24 / complete` uses one 32-core Windows runner. One setup feeds the required package build, required production site build, and complete observational portability inventory. Required failures fail the job; observational failures are reported as non-blocking. ESLint stays single-threaded because 16 ESLint workers took 174.54 seconds, coverage uses at most 12 workers, and the outer scheduler retains 16 slots. The job restores only the small master-refreshed ESLint cache and performs a clean pnpm install instead of restoring or saving the many-file package store. @@ -42,7 +42,7 @@ The larger client package graph makes cache mechanics and scheduler pressure par Two host effects remain part of the decision. A standard Node 26 job once spent 36 of its 67 seconds in `Set up job`, which is why environment contracts use distinct larger-runner pools instead of standard capacity. A Linux candidate spent 18 seconds registering a 50 KB Bubblewrap package because the hosted image scanned 202,507 package-database files. [`scripts/prepare-ci-bubblewrap.sh`](../../../../scripts/prepare-ci-bubblewrap.sh) instead verifies and extracts the pinned payload into the ephemeral runner directory, runs a functional confinement probe, and overlaps that preparation with dependency installation. -Coverage remains capped at 12 forks. Thirty-two forks crashed Node 24's CJS lexer twice, and a later 16-fork run reproduced the worker loss and invalid coverage result. Twelve forks preserve process headroom without becoming the single-host critical path. +Each coverage project is capped at 12 workers, and the process-bound project contains exactly four suite files. Thirty-two forks crashed Node 24's CJS lexer twice, and a later 16-fork run reproduced the worker loss and invalid coverage result. The single Vitest invocation therefore uses threads for the broad inventory and reserves forks for suites that exercise process-global state or `process` APIs. That narrow fork inventory includes the local bash process-plumbing suite: under aggregate gate contention its thread worker completed every test but intermittently missed the stdin-error callback needed for per-file function coverage. This preserves the suites' isolation contracts and deterministic coverage while avoiding forked execution for ordinary test files. The workflow retains two manual measurement suites. `suite=larger-runner-benchmark` compares isolated critical lanes across every size, and `suite=consolidated-runner-benchmark` compares whole aggregates. Complete serial Linux, macOS, and Windows references run only when `master` moves; pull requests run only the optimized jobs. diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md index 476c13573b..33ca3b823d 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md @@ -16,7 +16,7 @@ Status: implemented 生产 CI 包含 5 次大型运行器执行和 1 个标准运行器聚合作业。主 Node 门禁清单不再分片： -- `node 24 / complete` 使用一台 96 核 Linux 运行器。只需执行一次代码检出、设置、缓存恢复和安装，即可供全部 42 项主门禁使用。`run-gates` 最多同时启动 12 项相互独立的门禁；ESLint 使用 16 个工作线程，覆盖率最多使用 12 个，快照回放最多使用 8 个。第一批短门禁释放调度器槽位后，构建会立即启动，而快照回放和发布消费方仍显式依赖生成的 `lib/` 输出。拉取请求会恢复 pnpm 和 ESLint 缓存但不保存，因此缓存压缩和上传不会延长必需作业；master 上的串行参考会在拉取请求关键路径之外刷新这些缓存。该只读作业不会持久化代码检出凭据，并直接恢复固定的 pnpm 存储根目录，因此无需再单独执行存储路径探测步骤。 +- `node 24 / complete` 使用一台 96 核 Linux 运行器。只需执行一次代码检出、设置、缓存恢复和安装，即可供全部 42 项主门禁使用。`run-gates` 最多同时启动 12 项相互独立的门禁；ESLint 使用 16 个工作线程，覆盖率最多使用 12 个，快照回放最多使用 8 个。第一批短门禁释放调度器槽位后，构建会立即启动，而快照回放和发布消费方仍显式依赖生成的 `lib/` 输出。拉取请求会恢复 pnpm 和 ESLint 缓存但不保存，因此缓存压缩和上传不会延长必需作业；master 上的串行参考会在拉取请求关键路径之外刷新这些缓存。该只读作业不会持久化代码检出凭据。 - Node 22.19 和 Node 26 分别使用 4 核和 32 核 Linux 池运行各自的运行时兼容性冒烟测试。Python 3.10 使用 8 核 Linux 池运行完整的无密钥 SDK 套件。这些作业属于环境契约，并非主 Node 门禁清单的分片。 - `windows node 24 / complete` 使用一台 32 核 Windows 运行器。一次设置供必需的包构建、必需的生产网站构建以及完整的观测性可移植性清单共用。任何必需项失败都会使作业失败；观测项失败则报告为非阻塞。ESLint 保持单线程，因为 16 个 ESLint 工作线程耗时 174.54 秒；覆盖率最多使用 12 个工作线程，外层调度器则保留 16 个槽位。该作业仅恢复由 master 刷新的较小 ESLint 缓存，并在干净环境中执行 pnpm 安装，而不恢复或保存包含大量文件的包存储。 @@ -42,7 +42,7 @@ Windows 仓库工作在超过 16 核后收益很小，但 32 核池可以让完 两项主机效应仍构成这项决策的依据。一个标准 Node 26 作业曾在总共 67 秒的耗时中，把 36 秒用在 `Set up job` 上，因此各项环境契约使用不同的大型运行器池，而非标准容量。一个 Linux 候选作业在注册 50 KB 的 Bubblewrap 包时耗时 18 秒，因为托管映像扫描了 202,507 个包数据库文件。[`scripts/prepare-ci-bubblewrap.sh`](../../../../scripts/prepare-ci-bubblewrap.sh) 改为验证固定包内容并将其解压到临时运行器目录，执行功能性隔离探针，并让这项准备工作与依赖安装重叠执行。 -覆盖率仍限制为 12 个 fork。32 个 fork 曾两次导致 Node 24 的 CJS 词法分析器崩溃，后来一次使用 16 个 fork 的运行又复现了工作进程丢失和无效的覆盖率结果。12 个 fork 能保留进程余量，同时不会成为单机关键路径。 +每个覆盖率项目均将工作线程上限设为 12 个，其中进程约束项目恰好包含 4 个套件文件。32 个 fork 曾两次导致 Node 24 的 CJS 词法分析器崩溃，后来一次使用 16 个 fork 的运行又复现了工作进程丢失和无效的覆盖率结果。因此，单次 Vitest 调用会对大范围测试清单使用线程，只为涉及进程全局状态或 `process` API 的套件保留 fork。这份有限的 fork 清单还包含本地 bash 进程通路套件：在聚合门禁争用资源时，该套件的工作线程虽然完成并通过了所有测试，却会间歇性漏记逐文件函数覆盖率所需的 stdin 错误回调。这样既能保留这些套件的隔离契约和覆盖率结果的确定性，又能避免以 fork 方式执行普通测试文件。 工作流保留 2 项手动测量套件。`suite=larger-runner-benchmark` 比较所有规格下相互独立的关键通道，`suite=consolidated-runner-benchmark` 比较完整聚合流程。只有在 `master` 移动时，才运行完整的 Linux、macOS 和 Windows 串行参考；拉取请求只运行优化后的作业。 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ffc95ea849..4a0476c824 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,13 +50,17 @@ jobs: with: node-version: ${{ env.PRIMARY_NODE_VERSION }} + - name: Enable corepack and resolve pnpm store path + id: pnpm-store + run: | + corepack enable + echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" + # Pull requests consume the default-branch cache but do not put cache # compression and upload on the paid latency-critical path. - uses: actions/cache/restore@v4 with: - # Cache the versioned store through its stable parent so selecting - # the current pnpm store does not require a separate runner step. - path: ~/.local/share/pnpm/store + path: ${{ steps.pnpm-store.outputs.path }} key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} restore-keys: | ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm- @@ -70,7 +74,6 @@ jobs: - name: Install dependencies and prepare bubblewrap run: | - corepack enable pnpm install --frozen-lockfile & install_pid=$! bash scripts/prepare-ci-bubblewrap.sh & diff --git a/vitest.config.ts b/vitest.config.ts index dc9061c45a..51ced4b19f 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -22,6 +22,21 @@ const windowsCoverageExclusions = process.platform === 'win32' ] : [] +const testIncludes = [ + 'packages/*/*/tests/**/*.spec.{ts,tsx}', + 'examples/*/tests/**/*.spec.ts', + 'scripts/**/*.spec.ts', +] + +// These suites exercise process-global state or process APIs that worker threads cannot isolate. +// Keep the narrow exception in forks while the rest of the inventory avoids per-file processes. +const processBoundTests = [ + 'packages/bash/bash-local/tests/run.spec.ts', + 'packages/context/time-context/tests/time-context.spec.ts', + 'packages/ui/app-boot/tests/app-boot.spec.ts', + 'packages/workflow/workflow-workerthread/tests/session.spec.ts', +] + export default defineConfig({ // Native path resolution reads each package's nearest tsconfig, but only the root defines // workspace paths. Keep this plugin pinned to the root map so bare package imports resolve @@ -32,8 +47,35 @@ export default defineConfig({ test: { setupFiles: ['./scripts/test-invariants.ts'], // .tsx: client component specs (jsdom via per-file @vitest-environment pragma). - include: ['packages/*/*/tests/**/*.spec.{ts,tsx}', 'examples/*/tests/**/*.spec.ts', 'scripts/**/*.spec.ts'], + include: testIncludes, exclude: windowsUnsupportedPackages.map(path => `${path}/tests/**/*.spec.ts`), + // One coverage invocation aggregates both projects. Most suites use threads + // for lower startup/IPC overhead; only explicit process-bound suites fork. + projects: [ + { + plugins: [tsconfigPaths({ projects: ['./tsconfig.vitest.json'] })], + test: { + name: 'thread-safe', + pool: 'threads', + setupFiles: ['./scripts/test-invariants.ts'], + include: testIncludes, + exclude: [ + ...windowsUnsupportedPackages.map(path => `${path}/tests/**/*.spec.ts`), + ...processBoundTests, + ], + }, + }, + { + plugins: [tsconfigPaths({ projects: ['./tsconfig.vitest.json'] })], + test: { + name: 'process-bound', + pool: 'forks', + setupFiles: ['./scripts/test-invariants.ts'], + include: processBoundTests, + exclude: windowsUnsupportedPackages.map(path => `${path}/tests/**/*.spec.ts`), + }, + }, + ], coverage: { provider: 'v8', // Coverage measures OUR runtime source. Types-only files carry no From a952aaed72de940e27dcf44db9e142a9ee3db366 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 20:00:15 +0800 Subject: [PATCH 46/49] ci: trim remaining hosted setup --- ...vidence-based-larger-hosted-runners.i18n.yaml | 4 ++-- ...07-22-evidence-based-larger-hosted-runners.md | 6 +++--- ...22-evidence-based-larger-hosted-runners.zh.md | 6 +++--- .github/workflows/ci.yml | 16 +--------------- 4 files changed, 9 insertions(+), 23 deletions(-) diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml index 3561f39a33..7ad1361050 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -2026-07-22-evidence-based-larger-hosted-runners.md: 2fac4d1c14e3f1eabef9f915e53ff737c0e7ee20 -2026-07-22-evidence-based-larger-hosted-runners.zh.md: 33ca3b823d2bf235f9896b92868bd7a1f0688a76 +2026-07-22-evidence-based-larger-hosted-runners.md: 959ee8516ca4a4990ca4565966d637cfa84d8d7a +2026-07-22-evidence-based-larger-hosted-runners.zh.md: 52f5394d99c4fe9897653b43f9ba6f8ef2cbadfe diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md index 2fac4d1c14..959ee8516c 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md @@ -16,9 +16,9 @@ The organization keeps twelve x64 larger-runner pools in the repo-restricted `ds Production CI uses five larger-runner executions and one standard-runner aggregator. The primary Node inventory is not sharded: -- `node 24 / complete` uses one 96-core Linux runner. One checkout, setup, cache restore, and install feeds all 42 primary gates. `run-gates` starts up to 12 independent gates; ESLint uses 16 workers, coverage uses at most 12, and snapshot replay uses at most 8. Build starts as soon as the first short gates release scheduler slots, while snapshot replay and publication consumers retain explicit dependencies on emitted `lib/` output. Pull requests restore pnpm and ESLint caches without saving them, so cache compression and upload do not extend the required job; the master serial references refresh those caches outside the pull-request critical path. The read-only job does not persist checkout credentials. +- `node 24 / complete` uses one 96-core Linux runner. One checkout, setup, pnpm-cache restore, and install feeds all 42 primary gates. `run-gates` starts up to 12 independent gates; ESLint and coverage use at most 16 workers, and snapshot replay uses at most 8. Build starts as soon as the first short gates release scheduler slots, while snapshot replay and publication consumers retain explicit dependencies on emitted `lib/` output. Pull requests restore the pnpm cache without saving it, so cache compression and upload do not extend the required job; the master serial reference refreshes that cache outside the pull-request critical path. ESLint remains well below the critical path without a separate cache action. The read-only job does not persist checkout credentials. - Node 22.19 and Node 26 use the 4- and 32-core Linux pools for their runtime compatibility smokes. Python 3.10 uses the 8-core Linux pool for the complete keyless SDK suite. These are environment contracts, not slices of the primary Node gate inventory. -- `windows node 24 / complete` uses one 32-core Windows runner. One setup feeds the required package build, required production site build, and complete observational portability inventory. Required failures fail the job; observational failures are reported as non-blocking. ESLint stays single-threaded because 16 ESLint workers took 174.54 seconds, coverage uses at most 12 workers, and the outer scheduler retains 16 slots. The job restores only the small master-refreshed ESLint cache and performs a clean pnpm install instead of restoring or saving the many-file package store. +- `windows node 24 / complete` uses one 32-core Windows runner. One setup feeds the required package build, required production site build, and complete observational portability inventory. Required failures fail the job; observational failures are reported as non-blocking. ESLint stays single-threaded because 16 ESLint workers took 174.54 seconds, coverage uses at most 12 workers, and the outer scheduler retains 16 slots. The job restores only the small master-refreshed ESLint cache and performs a clean pnpm install instead of restoring or saving the many-file package store. All six Windows larger-runner sizes completed install and the production-site benchmark without mutating the machine-wide Developer Mode registry key, so the pull-request critical path omits that redundant step. The former gate-level and coarse primary shard jobs are absent from the workflow. Their static, lint, coverage, snapshot, and scenario shard selectors are also absent from the repository, so an unused diagnostic path cannot preserve a second CI architecture. @@ -42,7 +42,7 @@ The larger client package graph makes cache mechanics and scheduler pressure par Two host effects remain part of the decision. A standard Node 26 job once spent 36 of its 67 seconds in `Set up job`, which is why environment contracts use distinct larger-runner pools instead of standard capacity. A Linux candidate spent 18 seconds registering a 50 KB Bubblewrap package because the hosted image scanned 202,507 package-database files. [`scripts/prepare-ci-bubblewrap.sh`](../../../../scripts/prepare-ci-bubblewrap.sh) instead verifies and extracts the pinned payload into the ephemeral runner directory, runs a functional confinement probe, and overlaps that preparation with dependency installation. -Each coverage project is capped at 12 workers, and the process-bound project contains exactly four suite files. Thirty-two forks crashed Node 24's CJS lexer twice, and a later 16-fork run reproduced the worker loss and invalid coverage result. The single Vitest invocation therefore uses threads for the broad inventory and reserves forks for suites that exercise process-global state or `process` APIs. That narrow fork inventory includes the local bash process-plumbing suite: under aggregate gate contention its thread worker completed every test but intermittently missed the stdin-error callback needed for per-file function coverage. This preserves the suites' isolation contracts and deterministic coverage while avoiding forked execution for ordinary test files. +Linux coverage caps each project at 16 workers, while Windows keeps the 12-worker cap. The process-bound project contains exactly four suite files, so its fork count cannot reach either cap. Thirty-two forks crashed Node 24's CJS lexer twice, and a later 16-fork run reproduced the worker loss and invalid coverage result. The single Vitest invocation therefore uses threads for the broad inventory and reserves forks for suites that exercise process-global state or `process` APIs. That narrow fork inventory includes the local bash process-plumbing suite: under aggregate gate contention its thread worker completed every test but intermittently missed the stdin-error callback needed for per-file function coverage. A 32-worker all-gate run on the 96-core host slowed coverage to 44.6 seconds and made a compute-budget regression cross its one-second threshold, so production stops at 16. This preserves the suites' isolation contracts and deterministic coverage while avoiding forked execution for ordinary test files. The workflow retains two manual measurement suites. `suite=larger-runner-benchmark` compares isolated critical lanes across every size, and `suite=consolidated-runner-benchmark` compares whole aggregates. Complete serial Linux, macOS, and Windows references run only when `master` moves; pull requests run only the optimized jobs. diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md index 33ca3b823d..52f5394d99 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md @@ -16,9 +16,9 @@ Status: implemented 生产 CI 包含 5 次大型运行器执行和 1 个标准运行器聚合作业。主 Node 门禁清单不再分片： -- `node 24 / complete` 使用一台 96 核 Linux 运行器。只需执行一次代码检出、设置、缓存恢复和安装，即可供全部 42 项主门禁使用。`run-gates` 最多同时启动 12 项相互独立的门禁；ESLint 使用 16 个工作线程，覆盖率最多使用 12 个，快照回放最多使用 8 个。第一批短门禁释放调度器槽位后，构建会立即启动，而快照回放和发布消费方仍显式依赖生成的 `lib/` 输出。拉取请求会恢复 pnpm 和 ESLint 缓存但不保存，因此缓存压缩和上传不会延长必需作业；master 上的串行参考会在拉取请求关键路径之外刷新这些缓存。该只读作业不会持久化代码检出凭据。 +- `node 24 / complete` 使用一台 96 核 Linux 运行器。只需执行一次代码检出、设置、pnpm 缓存恢复和安装，即可供全部 42 项主门禁使用。`run-gates` 最多同时启动 12 项相互独立的门禁；ESLint 和覆盖率最多使用 16 个工作线程，快照回放最多使用 8 个。第一批短门禁释放调度器槽位后，构建会立即启动，而快照回放和发布消费方仍显式依赖生成的 `lib/` 输出。拉取请求会恢复 pnpm 缓存但不保存，因此缓存压缩和上传不会延长必需作业；master 上的串行参考会在拉取请求关键路径之外刷新该缓存。无需单独执行缓存操作，ESLint 的耗时仍远低于关键路径耗时。该只读作业不会持久化代码检出凭据。 - Node 22.19 和 Node 26 分别使用 4 核和 32 核 Linux 池运行各自的运行时兼容性冒烟测试。Python 3.10 使用 8 核 Linux 池运行完整的无密钥 SDK 套件。这些作业属于环境契约，并非主 Node 门禁清单的分片。 -- `windows node 24 / complete` 使用一台 32 核 Windows 运行器。一次设置供必需的包构建、必需的生产网站构建以及完整的观测性可移植性清单共用。任何必需项失败都会使作业失败；观测项失败则报告为非阻塞。ESLint 保持单线程，因为 16 个 ESLint 工作线程耗时 174.54 秒；覆盖率最多使用 12 个工作线程，外层调度器则保留 16 个槽位。该作业仅恢复由 master 刷新的较小 ESLint 缓存，并在干净环境中执行 pnpm 安装，而不恢复或保存包含大量文件的包存储。 +- `windows node 24 / complete` 使用一台 32 核 Windows 运行器。一次设置供必需的包构建、必需的生产网站构建以及完整的观测性可移植性清单共用。任何必需项失败都会使作业失败；观测项失败则报告为非阻塞。ESLint 保持单线程，因为 16 个 ESLint 工作线程耗时 174.54 秒；覆盖率最多使用 12 个工作线程，外层调度器则保留 16 个槽位。该作业仅恢复由 master 刷新的较小 ESLint 缓存，并在干净环境中执行 pnpm 安装，而不恢复或保存包含大量文件的包存储。全部 6 种 Windows 大型运行器规格都在未修改系统级 Developer Mode 注册表项的情况下完成了安装和生产网站基准测试，因此拉取请求关键路径省略了这个多余步骤。 原有的门禁级和粗粒度主流程分片作业已从工作流中移除。相应的静态、lint、覆盖率、快照和场景分片选择器也已从仓库中移除，因此未使用的诊断路径无法继续维系第二套 CI 架构。 @@ -42,7 +42,7 @@ Windows 仓库工作在超过 16 核后收益很小，但 32 核池可以让完 两项主机效应仍构成这项决策的依据。一个标准 Node 26 作业曾在总共 67 秒的耗时中，把 36 秒用在 `Set up job` 上，因此各项环境契约使用不同的大型运行器池，而非标准容量。一个 Linux 候选作业在注册 50 KB 的 Bubblewrap 包时耗时 18 秒，因为托管映像扫描了 202,507 个包数据库文件。[`scripts/prepare-ci-bubblewrap.sh`](../../../../scripts/prepare-ci-bubblewrap.sh) 改为验证固定包内容并将其解压到临时运行器目录，执行功能性隔离探针，并让这项准备工作与依赖安装重叠执行。 -每个覆盖率项目均将工作线程上限设为 12 个，其中进程约束项目恰好包含 4 个套件文件。32 个 fork 曾两次导致 Node 24 的 CJS 词法分析器崩溃，后来一次使用 16 个 fork 的运行又复现了工作进程丢失和无效的覆盖率结果。因此，单次 Vitest 调用会对大范围测试清单使用线程，只为涉及进程全局状态或 `process` API 的套件保留 fork。这份有限的 fork 清单还包含本地 bash 进程通路套件：在聚合门禁争用资源时，该套件的工作线程虽然完成并通过了所有测试，却会间歇性漏记逐文件函数覆盖率所需的 stdin 错误回调。这样既能保留这些套件的隔离契约和覆盖率结果的确定性，又能避免以 fork 方式执行普通测试文件。 +Linux 覆盖率把每个项目的工作线程上限设为 16 个，Windows 则保留 12 个工作线程的上限。进程约束项目恰好包含 4 个套件文件，因此它的 fork 数量不可能达到任一上限。32 个 fork 曾两次导致 Node 24 的 CJS 词法分析器崩溃，后来一次使用 16 个 fork 的运行又复现了工作进程丢失和无效的覆盖率结果。因此，单次 Vitest 调用会对大范围测试清单使用线程，只为涉及进程全局状态或 `process` API 的套件保留 fork。这份有限的 fork 清单还包含本地 bash 进程通路套件：在聚合门禁争用资源时，该套件的工作线程虽然完成了所有测试，却会间歇性漏记逐文件函数覆盖率所需的 stdin 错误回调。在 96 核主机上使用 32 个工作线程运行全部门禁时，覆盖率耗时变慢至 44.6 秒，还使一项计算预算回归超过其 1 秒阈值，因此生产环境将工作线程数限制在 16 个以内。这样既能保留这些套件的隔离契约和覆盖率结果的确定性，又能避免以 fork 方式执行普通测试文件。 工作流保留 2 项手动测量套件。`suite=larger-runner-benchmark` 比较所有规格下相互独立的关键通道，`suite=consolidated-runner-benchmark` 比较完整聚合流程。只有在 `master` 移动时，才运行完整的 Linux、macOS 和 Windows 串行参考；拉取请求只运行优化后的作业。 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a0476c824..3d3f329018 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,8 +35,7 @@ jobs: runs-on: dsh-ubuntu-24-04-96core name: node 24 / complete env: - DSH_COVERAGE_MAX_WORKERS: '12' - DSH_ESLINT_CACHE: '1' + DSH_COVERAGE_MAX_WORKERS: '16' DSH_ESLINT_CONCURRENCY: '16' DSH_GATE_CONCURRENCY: '12' DSH_PUBLINT_CONCURRENCY: '16' @@ -65,13 +64,6 @@ jobs: restore-keys: | ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm- - - uses: actions/cache/restore@v4 - with: - path: .cache/eslint - key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-full-${{ hashFiles('pnpm-lock.yaml', 'eslint.config.mjs', 'tsconfig.json', 'packages/*/*/tsconfig.json', 'examples/*/tsconfig.json') }} - restore-keys: | - ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-full- - - name: Install dependencies and prepare bubblewrap run: | pnpm install --frozen-lockfile & @@ -171,12 +163,6 @@ jobs: steps: - uses: actions/checkout@v6 - - name: Enable Developer Mode (symlink support) - shell: pwsh - run: >- - reg add "HKLM\SOFTWARE\Microsoft\Windows\CurrentVersion\AppModelUnlock" - /t REG_DWORD /f /v "AllowDevelopmentWithoutDevLicense" /d "1" - - uses: actions/setup-node@v6 with: node-version: ${{ env.PRIMARY_NODE_VERSION }} From 9b8320cb941ac07c92e9e2937e6e32951b2d9d88 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 20:06:50 +0800 Subject: [PATCH 47/49] ci: remove the final Linux lint tail --- .../2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml | 4 ++-- .../2026-07-22-evidence-based-larger-hosted-runners.md | 2 +- .../2026-07-22-evidence-based-larger-hosted-runners.zh.md | 2 +- .github/workflows/ci.yml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml index 7ad1361050..f0fdb86b15 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -2026-07-22-evidence-based-larger-hosted-runners.md: 959ee8516ca4a4990ca4565966d637cfa84d8d7a -2026-07-22-evidence-based-larger-hosted-runners.zh.md: 52f5394d99c4fe9897653b43f9ba6f8ef2cbadfe +2026-07-22-evidence-based-larger-hosted-runners.md: d0894b8ba239026046602e7909bb521e848ed0bd +2026-07-22-evidence-based-larger-hosted-runners.zh.md: a308fc76f37f3ea831266164d99ca9184464d895 diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md index 959ee8516c..d0894b8ba2 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md @@ -16,7 +16,7 @@ The organization keeps twelve x64 larger-runner pools in the repo-restricted `ds Production CI uses five larger-runner executions and one standard-runner aggregator. The primary Node inventory is not sharded: -- `node 24 / complete` uses one 96-core Linux runner. One checkout, setup, pnpm-cache restore, and install feeds all 42 primary gates. `run-gates` starts up to 12 independent gates; ESLint and coverage use at most 16 workers, and snapshot replay uses at most 8. Build starts as soon as the first short gates release scheduler slots, while snapshot replay and publication consumers retain explicit dependencies on emitted `lib/` output. Pull requests restore the pnpm cache without saving it, so cache compression and upload do not extend the required job; the master serial reference refreshes that cache outside the pull-request critical path. ESLint remains well below the critical path without a separate cache action. The read-only job does not persist checkout credentials. +- `node 24 / complete` uses one 96-core Linux runner. One checkout, setup, pnpm-cache restore, and install feeds all 42 primary gates. `run-gates` starts up to 12 independent gates; ESLint uses at most 32 workers, coverage uses at most 16, and snapshot replay uses at most 8. Build starts as soon as the first short gates release scheduler slots, while snapshot replay and publication consumers retain explicit dependencies on emitted `lib/` output. Pull requests restore the pnpm cache without saving it, so cache compression and upload do not extend the required job; the master serial reference refreshes that cache outside the pull-request critical path. The earlier 96-core trace measured uncached 32-worker ESLint at 33.30 seconds, below the coverage critical path, so a separate ESLint cache action is unnecessary. The read-only job does not persist checkout credentials. - Node 22.19 and Node 26 use the 4- and 32-core Linux pools for their runtime compatibility smokes. Python 3.10 uses the 8-core Linux pool for the complete keyless SDK suite. These are environment contracts, not slices of the primary Node gate inventory. - `windows node 24 / complete` uses one 32-core Windows runner. One setup feeds the required package build, required production site build, and complete observational portability inventory. Required failures fail the job; observational failures are reported as non-blocking. ESLint stays single-threaded because 16 ESLint workers took 174.54 seconds, coverage uses at most 12 workers, and the outer scheduler retains 16 slots. The job restores only the small master-refreshed ESLint cache and performs a clean pnpm install instead of restoring or saving the many-file package store. All six Windows larger-runner sizes completed install and the production-site benchmark without mutating the machine-wide Developer Mode registry key, so the pull-request critical path omits that redundant step. diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md index 52f5394d99..a308fc76f3 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md @@ -16,7 +16,7 @@ Status: implemented 生产 CI 包含 5 次大型运行器执行和 1 个标准运行器聚合作业。主 Node 门禁清单不再分片： -- `node 24 / complete` 使用一台 96 核 Linux 运行器。只需执行一次代码检出、设置、pnpm 缓存恢复和安装，即可供全部 42 项主门禁使用。`run-gates` 最多同时启动 12 项相互独立的门禁；ESLint 和覆盖率最多使用 16 个工作线程，快照回放最多使用 8 个。第一批短门禁释放调度器槽位后，构建会立即启动，而快照回放和发布消费方仍显式依赖生成的 `lib/` 输出。拉取请求会恢复 pnpm 缓存但不保存，因此缓存压缩和上传不会延长必需作业；master 上的串行参考会在拉取请求关键路径之外刷新该缓存。无需单独执行缓存操作，ESLint 的耗时仍远低于关键路径耗时。该只读作业不会持久化代码检出凭据。 +- `node 24 / complete` 使用一台 96 核 Linux 运行器。只需执行一次代码检出、设置、pnpm 缓存恢复和安装，即可供全部 42 项主门禁使用。`run-gates` 最多同时启动 12 项相互独立的门禁；ESLint 最多使用 32 个工作线程，覆盖率最多使用 16 个，快照回放最多使用 8 个。第一批短门禁释放调度器槽位后，构建会立即启动，而快照回放和发布消费方仍显式依赖生成的 `lib/` 输出。拉取请求会恢复 pnpm 缓存但不保存，因此缓存压缩和上传不会延长必需作业；master 上的串行参考会在拉取请求关键路径之外刷新该缓存。早先的 96 核运行轨迹测得，在未使用缓存且启动 32 个工作线程的情况下，ESLint 耗时 33.30 秒，短于覆盖率关键路径，因此无需为 ESLint 单独执行缓存操作。该只读作业不会持久化代码检出凭据。 - Node 22.19 和 Node 26 分别使用 4 核和 32 核 Linux 池运行各自的运行时兼容性冒烟测试。Python 3.10 使用 8 核 Linux 池运行完整的无密钥 SDK 套件。这些作业属于环境契约，并非主 Node 门禁清单的分片。 - `windows node 24 / complete` 使用一台 32 核 Windows 运行器。一次设置供必需的包构建、必需的生产网站构建以及完整的观测性可移植性清单共用。任何必需项失败都会使作业失败；观测项失败则报告为非阻塞。ESLint 保持单线程，因为 16 个 ESLint 工作线程耗时 174.54 秒；覆盖率最多使用 12 个工作线程，外层调度器则保留 16 个槽位。该作业仅恢复由 master 刷新的较小 ESLint 缓存，并在干净环境中执行 pnpm 安装，而不恢复或保存包含大量文件的包存储。全部 6 种 Windows 大型运行器规格都在未修改系统级 Developer Mode 注册表项的情况下完成了安装和生产网站基准测试，因此拉取请求关键路径省略了这个多余步骤。 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d3f329018..f6c00943e1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,7 @@ jobs: name: node 24 / complete env: DSH_COVERAGE_MAX_WORKERS: '16' - DSH_ESLINT_CONCURRENCY: '16' + DSH_ESLINT_CONCURRENCY: '32' DSH_GATE_CONCURRENCY: '12' DSH_PUBLINT_CONCURRENCY: '16' DSH_SNAPSHOT_MAX_CONCURRENCY: '8' From dcf9eee3c597ed4a5fffd109943f89fbed3c418c Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 20:19:56 +0800 Subject: [PATCH 48/49] ci: bound Linux gate contention --- ...22-evidence-based-larger-hosted-runners.i18n.yaml | 4 ++-- ...026-07-22-evidence-based-larger-hosted-runners.md | 4 +++- ...-07-22-evidence-based-larger-hosted-runners.zh.md | 4 +++- .github/workflows/ci.yml | 12 ++++++++++-- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml index f0fdb86b15..06d52fb36a 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -2026-07-22-evidence-based-larger-hosted-runners.md: d0894b8ba239026046602e7909bb521e848ed0bd -2026-07-22-evidence-based-larger-hosted-runners.zh.md: a308fc76f37f3ea831266164d99ca9184464d895 +2026-07-22-evidence-based-larger-hosted-runners.md: 360c8f58f9650b466e7fbc8455a4a234cfc86e70 +2026-07-22-evidence-based-larger-hosted-runners.zh.md: 2a82c5fd0124c9d8d1dba72b2ca6eed92415580c diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md index d0894b8ba2..360c8f58f9 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md @@ -16,7 +16,7 @@ The organization keeps twelve x64 larger-runner pools in the repo-restricted `ds Production CI uses five larger-runner executions and one standard-runner aggregator. The primary Node inventory is not sharded: -- `node 24 / complete` uses one 96-core Linux runner. One checkout, setup, pnpm-cache restore, and install feeds all 42 primary gates. `run-gates` starts up to 12 independent gates; ESLint uses at most 32 workers, coverage uses at most 16, and snapshot replay uses at most 8. Build starts as soon as the first short gates release scheduler slots, while snapshot replay and publication consumers retain explicit dependencies on emitted `lib/` output. Pull requests restore the pnpm cache without saving it, so cache compression and upload do not extend the required job; the master serial reference refreshes that cache outside the pull-request critical path. The earlier 96-core trace measured uncached 32-worker ESLint at 33.30 seconds, below the coverage critical path, so a separate ESLint cache action is unnecessary. The read-only job does not persist checkout credentials. +- `node 24 / complete` uses one 96-core Linux runner. One checkout, setup, pnpm- and ESLint-cache restore, and install feeds all 42 primary gates. `run-gates` starts up to 8 independent gates; ESLint and coverage use at most 16 workers, and snapshot replay uses at most 8. Build starts as soon as the first short gates release scheduler slots, while snapshot replay and publication consumers retain explicit dependencies on emitted `lib/` output. Pull requests restore both caches without saving them, so cache compression and upload do not extend the required job; the master serial reference refreshes those caches outside the pull-request critical path. An uncached exact-head trace put ESLint at 38.11 seconds and coverage at 37.10 seconds, so the small ESLint restore remains useful on the critical path. The read-only job does not persist checkout credentials. - Node 22.19 and Node 26 use the 4- and 32-core Linux pools for their runtime compatibility smokes. Python 3.10 uses the 8-core Linux pool for the complete keyless SDK suite. These are environment contracts, not slices of the primary Node gate inventory. - `windows node 24 / complete` uses one 32-core Windows runner. One setup feeds the required package build, required production site build, and complete observational portability inventory. Required failures fail the job; observational failures are reported as non-blocking. ESLint stays single-threaded because 16 ESLint workers took 174.54 seconds, coverage uses at most 12 workers, and the outer scheduler retains 16 slots. The job restores only the small master-refreshed ESLint cache and performs a clean pnpm install instead of restoring or saving the many-file package store. All six Windows larger-runner sizes completed install and the production-site benchmark without mutating the machine-wide Developer Mode registry key, so the pull-request critical path omits that redundant step. @@ -42,6 +42,8 @@ The larger client package graph makes cache mechanics and scheduler pressure par Two host effects remain part of the decision. A standard Node 26 job once spent 36 of its 67 seconds in `Set up job`, which is why environment contracts use distinct larger-runner pools instead of standard capacity. A Linux candidate spent 18 seconds registering a 50 KB Bubblewrap package because the hosted image scanned 202,507 package-database files. [`scripts/prepare-ci-bubblewrap.sh`](../../../../scripts/prepare-ci-bubblewrap.sh) instead verifies and extracts the pinned payload into the ephemeral runner directory, runs a functional confinement probe, and overlaps that preparation with dependency installation. +Inner and outer worker limits are separate controls. An [exact-head 32-worker ESLint experiment](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29918329463) slowed lint to 52.28 seconds and coverage to 42.71 seconds, where an adapter idle-timeout test failed. Production therefore retains 16 ESLint workers and admits only 8 independent repository gates at once, leaving capacity for the worker pools owned by those gates. + Linux coverage caps each project at 16 workers, while Windows keeps the 12-worker cap. The process-bound project contains exactly four suite files, so its fork count cannot reach either cap. Thirty-two forks crashed Node 24's CJS lexer twice, and a later 16-fork run reproduced the worker loss and invalid coverage result. The single Vitest invocation therefore uses threads for the broad inventory and reserves forks for suites that exercise process-global state or `process` APIs. That narrow fork inventory includes the local bash process-plumbing suite: under aggregate gate contention its thread worker completed every test but intermittently missed the stdin-error callback needed for per-file function coverage. A 32-worker all-gate run on the 96-core host slowed coverage to 44.6 seconds and made a compute-budget regression cross its one-second threshold, so production stops at 16. This preserves the suites' isolation contracts and deterministic coverage while avoiding forked execution for ordinary test files. The workflow retains two manual measurement suites. `suite=larger-runner-benchmark` compares isolated critical lanes across every size, and `suite=consolidated-runner-benchmark` compares whole aggregates. Complete serial Linux, macOS, and Windows references run only when `master` moves; pull requests run only the optimized jobs. diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md index a308fc76f3..2a82c5fd01 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md @@ -16,7 +16,7 @@ Status: implemented 生产 CI 包含 5 次大型运行器执行和 1 个标准运行器聚合作业。主 Node 门禁清单不再分片： -- `node 24 / complete` 使用一台 96 核 Linux 运行器。只需执行一次代码检出、设置、pnpm 缓存恢复和安装，即可供全部 42 项主门禁使用。`run-gates` 最多同时启动 12 项相互独立的门禁；ESLint 最多使用 32 个工作线程，覆盖率最多使用 16 个，快照回放最多使用 8 个。第一批短门禁释放调度器槽位后，构建会立即启动，而快照回放和发布消费方仍显式依赖生成的 `lib/` 输出。拉取请求会恢复 pnpm 缓存但不保存，因此缓存压缩和上传不会延长必需作业；master 上的串行参考会在拉取请求关键路径之外刷新该缓存。早先的 96 核运行轨迹测得，在未使用缓存且启动 32 个工作线程的情况下，ESLint 耗时 33.30 秒，短于覆盖率关键路径，因此无需为 ESLint 单独执行缓存操作。该只读作业不会持久化代码检出凭据。 +- `node 24 / complete` 使用一台 96 核 Linux 运行器。只需执行一次代码检出、设置、pnpm 和 ESLint 缓存恢复以及安装，即可供全部 42 项主门禁使用。`run-gates` 最多同时启动 8 项相互独立的门禁；ESLint 和覆盖率最多使用 16 个工作线程，快照回放最多使用 8 个。第一批短门禁释放调度器槽位后，构建会立即启动，而快照回放和发布消费方仍显式依赖生成的 `lib/` 输出。拉取请求会恢复这两项缓存但不保存，因此缓存压缩和上传不会延长必需作业；master 上的串行参考会在拉取请求关键路径之外刷新这两项缓存。一次未使用缓存的分支头精确运行轨迹显示，ESLint 耗时 38.11 秒，覆盖率耗时 37.10 秒，因此在关键路径上恢复这个较小的 ESLint 缓存仍有价值。该只读作业不会持久化代码检出凭据。 - Node 22.19 和 Node 26 分别使用 4 核和 32 核 Linux 池运行各自的运行时兼容性冒烟测试。Python 3.10 使用 8 核 Linux 池运行完整的无密钥 SDK 套件。这些作业属于环境契约，并非主 Node 门禁清单的分片。 - `windows node 24 / complete` 使用一台 32 核 Windows 运行器。一次设置供必需的包构建、必需的生产网站构建以及完整的观测性可移植性清单共用。任何必需项失败都会使作业失败；观测项失败则报告为非阻塞。ESLint 保持单线程，因为 16 个 ESLint 工作线程耗时 174.54 秒；覆盖率最多使用 12 个工作线程，外层调度器则保留 16 个槽位。该作业仅恢复由 master 刷新的较小 ESLint 缓存，并在干净环境中执行 pnpm 安装，而不恢复或保存包含大量文件的包存储。全部 6 种 Windows 大型运行器规格都在未修改系统级 Developer Mode 注册表项的情况下完成了安装和生产网站基准测试，因此拉取请求关键路径省略了这个多余步骤。 @@ -42,6 +42,8 @@ Windows 仓库工作在超过 16 核后收益很小，但 32 核池可以让完 两项主机效应仍构成这项决策的依据。一个标准 Node 26 作业曾在总共 67 秒的耗时中，把 36 秒用在 `Set up job` 上，因此各项环境契约使用不同的大型运行器池，而非标准容量。一个 Linux 候选作业在注册 50 KB 的 Bubblewrap 包时耗时 18 秒，因为托管映像扫描了 202,507 个包数据库文件。[`scripts/prepare-ci-bubblewrap.sh`](../../../../scripts/prepare-ci-bubblewrap.sh) 改为验证固定包内容并将其解压到临时运行器目录，执行功能性隔离探针，并让这项准备工作与依赖安装重叠执行。 +内层与外层工作线程上限是相互独立的控制机制。一次[分支头精确、使用 32 个工作线程的 ESLint 实验](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29918329463)使 lint 耗时增至 52.28 秒、覆盖率耗时增至 42.71 秒；同一次运行中，一项适配器空闲超时测试失败。因此，生产环境将 ESLint 工作线程上限维持在 16 个，并且同时最多只运行 8 项相互独立的仓库门禁，为这些门禁自身的工作线程池留出容量。 + Linux 覆盖率把每个项目的工作线程上限设为 16 个，Windows 则保留 12 个工作线程的上限。进程约束项目恰好包含 4 个套件文件，因此它的 fork 数量不可能达到任一上限。32 个 fork 曾两次导致 Node 24 的 CJS 词法分析器崩溃，后来一次使用 16 个 fork 的运行又复现了工作进程丢失和无效的覆盖率结果。因此，单次 Vitest 调用会对大范围测试清单使用线程，只为涉及进程全局状态或 `process` API 的套件保留 fork。这份有限的 fork 清单还包含本地 bash 进程通路套件：在聚合门禁争用资源时，该套件的工作线程虽然完成了所有测试，却会间歇性漏记逐文件函数覆盖率所需的 stdin 错误回调。在 96 核主机上使用 32 个工作线程运行全部门禁时，覆盖率耗时变慢至 44.6 秒，还使一项计算预算回归超过其 1 秒阈值，因此生产环境将工作线程数限制在 16 个以内。这样既能保留这些套件的隔离契约和覆盖率结果的确定性，又能避免以 fork 方式执行普通测试文件。 工作流保留 2 项手动测量套件。`suite=larger-runner-benchmark` 比较所有规格下相互独立的关键通道，`suite=consolidated-runner-benchmark` 比较完整聚合流程。只有在 `master` 移动时，才运行完整的 Linux、macOS 和 Windows 串行参考；拉取请求只运行优化后的作业。 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f6c00943e1..42d1c2b1f6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,8 +36,9 @@ jobs: name: node 24 / complete env: DSH_COVERAGE_MAX_WORKERS: '16' - DSH_ESLINT_CONCURRENCY: '32' - DSH_GATE_CONCURRENCY: '12' + DSH_ESLINT_CACHE: '1' + DSH_ESLINT_CONCURRENCY: '16' + DSH_GATE_CONCURRENCY: '8' DSH_PUBLINT_CONCURRENCY: '16' DSH_SNAPSHOT_MAX_CONCURRENCY: '8' steps: @@ -64,6 +65,13 @@ jobs: restore-keys: | ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm- + - uses: actions/cache/restore@v4 + with: + path: .cache/eslint + key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-full-${{ hashFiles('pnpm-lock.yaml', 'eslint.config.mjs', 'tsconfig.json', 'packages/*/*/tsconfig.json', 'examples/*/tsconfig.json') }} + restore-keys: | + ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-full- + - name: Install dependencies and prepare bubblewrap run: | pnpm install --frozen-lockfile & From 3c397e594fc921ea93b739a3f1d56671d7a3d91a Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 20:29:33 +0800 Subject: [PATCH 49/49] ci: remove critical setup variance --- ...ence-based-larger-hosted-runners.i18n.yaml | 4 +- ...22-evidence-based-larger-hosted-runners.md | 10 ++-- ...evidence-based-larger-hosted-runners.zh.md | 10 ++-- .github/workflows/ci.yml | 47 ++++++++++++------- vitest.config.ts | 4 +- 5 files changed, 44 insertions(+), 31 deletions(-) diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml index 06d52fb36a..6277617a73 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write -2026-07-22-evidence-based-larger-hosted-runners.md: 360c8f58f9650b466e7fbc8455a4a234cfc86e70 -2026-07-22-evidence-based-larger-hosted-runners.zh.md: 2a82c5fd0124c9d8d1dba72b2ca6eed92415580c +2026-07-22-evidence-based-larger-hosted-runners.md: c0fae2841f21c431d6416cd5d421929d70197abb +2026-07-22-evidence-based-larger-hosted-runners.zh.md: 51c73a8a631af4f1254c795d09585770fc4e68bb diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md index 360c8f58f9..c0fae2841f 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.md @@ -16,9 +16,9 @@ The organization keeps twelve x64 larger-runner pools in the repo-restricted `ds Production CI uses five larger-runner executions and one standard-runner aggregator. The primary Node inventory is not sharded: -- `node 24 / complete` uses one 96-core Linux runner. One checkout, setup, pnpm- and ESLint-cache restore, and install feeds all 42 primary gates. `run-gates` starts up to 8 independent gates; ESLint and coverage use at most 16 workers, and snapshot replay uses at most 8. Build starts as soon as the first short gates release scheduler slots, while snapshot replay and publication consumers retain explicit dependencies on emitted `lib/` output. Pull requests restore both caches without saving them, so cache compression and upload do not extend the required job; the master serial reference refreshes those caches outside the pull-request critical path. An uncached exact-head trace put ESLint at 38.11 seconds and coverage at 37.10 seconds, so the small ESLint restore remains useful on the critical path. The read-only job does not persist checkout credentials. +- `node 24 / complete` uses one 96-core Linux runner. One checkout, direct selection of the image's preinstalled Node 24 toolcache, pnpm- and ESLint-cache restore, and install feeds all 42 primary gates. `run-gates` starts up to 10 independent gates; ESLint and coverage use at most 16 workers, and snapshot replay uses at most 8. Build starts as soon as the first short gates release scheduler slots, while snapshot replay and publication consumers retain explicit dependencies on emitted `lib/` output. Pull requests restore both caches without saving them, so cache compression and upload do not extend the required job; the master serial reference refreshes those caches outside the pull-request critical path. An uncached exact-head trace put ESLint at 38.11 seconds and coverage at 37.10 seconds, so the small ESLint restore remains useful on the critical path. The read-only job does not persist checkout credentials. - Node 22.19 and Node 26 use the 4- and 32-core Linux pools for their runtime compatibility smokes. Python 3.10 uses the 8-core Linux pool for the complete keyless SDK suite. These are environment contracts, not slices of the primary Node gate inventory. -- `windows node 24 / complete` uses one 32-core Windows runner. One setup feeds the required package build, required production site build, and complete observational portability inventory. Required failures fail the job; observational failures are reported as non-blocking. ESLint stays single-threaded because 16 ESLint workers took 174.54 seconds, coverage uses at most 12 workers, and the outer scheduler retains 16 slots. The job restores only the small master-refreshed ESLint cache and performs a clean pnpm install instead of restoring or saving the many-file package store. All six Windows larger-runner sizes completed install and the production-site benchmark without mutating the machine-wide Developer Mode registry key, so the pull-request critical path omits that redundant step. +- `windows node 24 / complete` uses one 32-core Windows runner. One preparation wave feeds the required package build, required production site build, and complete observational portability inventory. Required failures fail the job; observational failures are reported as non-blocking. ESLint stays single-threaded because 16 ESLint workers took 174.54 seconds, coverage uses at most 12 workers, and the outer scheduler retains 16 slots. The job restores only the small master-refreshed ESLint cache and performs a clean pnpm install instead of restoring or saving the many-file package store. All six Windows larger-runner sizes completed install and the production-site benchmark without mutating the machine-wide Developer Mode registry key, so the pull-request critical path omits that redundant step. The former gate-level and coarse primary shard jobs are absent from the workflow. Their static, lint, coverage, snapshot, and scenario shard selectors are also absent from the repository, so an unused diagnostic path cannot preserve a second CI architecture. @@ -40,11 +40,11 @@ Repository work gains little above 16 Windows cores, but the 32-core pool can st The larger client package graph makes cache mechanics and scheduler pressure part of the measured workload. In [one exact-head production run](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29912577681), Linux spent 39 seconds in repository gates but 69 seconds in the complete job, while Windows spent 117 seconds in repository gates and 228 seconds in the complete job. The Windows pnpm cache downloaded its 154 MB archive in about two seconds but spent 27 seconds extracting it, followed by a 23-second install and a 14-second post-job save. A [cacheless all-size trace](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29913033155) completed the same 32-core Windows install in 27 seconds. Production therefore avoids the Windows package-store cache, uses restore-only caches on latency-critical pull-request jobs, and bounds outer concurrency so typecheck, lint, coverage, and build do not oversubscribe one host. -Two host effects remain part of the decision. A standard Node 26 job once spent 36 of its 67 seconds in `Set up job`, which is why environment contracts use distinct larger-runner pools instead of standard capacity. A Linux candidate spent 18 seconds registering a 50 KB Bubblewrap package because the hosted image scanned 202,507 package-database files. [`scripts/prepare-ci-bubblewrap.sh`](../../../../scripts/prepare-ci-bubblewrap.sh) instead verifies and extracts the pinned payload into the ephemeral runner directory, runs a functional confinement probe, and overlaps that preparation with dependency installation. +Three host effects remain part of the decision. A standard Node 26 job once spent 36 of its 67 seconds in `Set up job`, which is why environment contracts use distinct larger-runner pools instead of standard capacity. The setup-node action later spent 3.68 seconds printing cached Linux environment details and 46.56 seconds doing the same on Windows after both had already found Node 24.18.0 in the hosted toolcache. The two latency-critical jobs select the newest preinstalled 24.x directory directly, verify its major, and fail loud if the image no longer carries it; compatibility jobs retain setup-node because selecting a non-default runtime is their contract. A Linux candidate also spent 18 seconds registering a 50 KB Bubblewrap package because the hosted image scanned 202,507 package-database files. [`scripts/prepare-ci-bubblewrap.sh`](../../../../scripts/prepare-ci-bubblewrap.sh) instead verifies and extracts the pinned payload into the ephemeral runner directory, runs a functional confinement probe, and overlaps that preparation with dependency installation. -Inner and outer worker limits are separate controls. An [exact-head 32-worker ESLint experiment](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29918329463) slowed lint to 52.28 seconds and coverage to 42.71 seconds, where an adapter idle-timeout test failed. Production therefore retains 16 ESLint workers and admits only 8 independent repository gates at once, leaving capacity for the worker pools owned by those gates. +Inner and outer worker limits are separate controls. An [exact-head 32-worker ESLint experiment](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29918329463) slowed lint to 52.28 seconds and coverage to 42.71 seconds, where an adapter idle-timeout test failed. A later 8-gate trace reduced coverage to 35.17 seconds but delayed the production-site build until the aggregate reached 41.06 seconds. Production therefore retains 16 ESLint workers and admits 10 independent repository gates at once, leaving capacity for the worker pools owned by those gates without starving later independent work. -Linux coverage caps each project at 16 workers, while Windows keeps the 12-worker cap. The process-bound project contains exactly four suite files, so its fork count cannot reach either cap. Thirty-two forks crashed Node 24's CJS lexer twice, and a later 16-fork run reproduced the worker loss and invalid coverage result. The single Vitest invocation therefore uses threads for the broad inventory and reserves forks for suites that exercise process-global state or `process` APIs. That narrow fork inventory includes the local bash process-plumbing suite: under aggregate gate contention its thread worker completed every test but intermittently missed the stdin-error callback needed for per-file function coverage. A 32-worker all-gate run on the 96-core host slowed coverage to 44.6 seconds and made a compute-budget regression cross its one-second threshold, so production stops at 16. This preserves the suites' isolation contracts and deterministic coverage while avoiding forked execution for ordinary test files. +Linux coverage caps each project at 16 workers, while Windows keeps the 12-worker cap. The process-bound project contains exactly five suite files, so its fork count cannot reach either cap. Thirty-two forks crashed Node 24's CJS lexer twice, and a later 16-fork run reproduced the worker loss and invalid coverage result. The single Vitest invocation therefore uses threads for the broad inventory and reserves forks for suites that exercise process-global state, `process` APIs, or timing-sensitive process I/O. That narrow fork inventory includes the local bash process-plumbing suite: under aggregate gate contention its thread worker completed every test but intermittently missed the stdin-error callback needed for per-file function coverage. It also includes the pi-ai adapter suite after two hosted aggregate runs delayed an idle-watchdog socket-close observation past its 100-millisecond test deadline. A 32-worker all-gate run on the 96-core host slowed coverage to 44.6 seconds and made a compute-budget regression cross its one-second threshold, so production stops at 16. This preserves the suites' isolation contracts and deterministic coverage while avoiding forked execution for ordinary test files. The workflow retains two manual measurement suites. `suite=larger-runner-benchmark` compares isolated critical lanes across every size, and `suite=consolidated-runner-benchmark` compares whole aggregates. Complete serial Linux, macOS, and Windows references run only when `master` moves; pull requests run only the optimized jobs. diff --git a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md index 2a82c5fd01..51c73a8a63 100644 --- a/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md +++ b/.agents/notes/implemented/process/2026-07-22-evidence-based-larger-hosted-runners.zh.md @@ -16,9 +16,9 @@ Status: implemented 生产 CI 包含 5 次大型运行器执行和 1 个标准运行器聚合作业。主 Node 门禁清单不再分片： -- `node 24 / complete` 使用一台 96 核 Linux 运行器。只需执行一次代码检出、设置、pnpm 和 ESLint 缓存恢复以及安装，即可供全部 42 项主门禁使用。`run-gates` 最多同时启动 8 项相互独立的门禁；ESLint 和覆盖率最多使用 16 个工作线程，快照回放最多使用 8 个。第一批短门禁释放调度器槽位后，构建会立即启动，而快照回放和发布消费方仍显式依赖生成的 `lib/` 输出。拉取请求会恢复这两项缓存但不保存，因此缓存压缩和上传不会延长必需作业；master 上的串行参考会在拉取请求关键路径之外刷新这两项缓存。一次未使用缓存的分支头精确运行轨迹显示，ESLint 耗时 38.11 秒，覆盖率耗时 37.10 秒，因此在关键路径上恢复这个较小的 ESLint 缓存仍有价值。该只读作业不会持久化代码检出凭据。 +- `node 24 / complete` 使用一台 96 核 Linux 运行器。只需执行一次代码检出、直接选择托管映像中预装的 Node 24 toolcache、恢复 pnpm 和 ESLint 缓存以及安装，即可供全部 42 项主门禁使用。`run-gates` 最多同时启动 10 项相互独立的门禁；ESLint 和覆盖率最多使用 16 个工作线程，快照回放最多使用 8 个。第一批短门禁释放调度器槽位后，构建会立即启动，而快照回放和发布消费方仍显式依赖生成的 `lib/` 输出。拉取请求会恢复这两项缓存但不保存，因此缓存压缩和上传不会延长必需作业；master 上的串行参考会在拉取请求关键路径之外刷新这两项缓存。一次未使用缓存的分支头精确运行轨迹显示，ESLint 耗时 38.11 秒，覆盖率耗时 37.10 秒，因此在关键路径上恢复这个较小的 ESLint 缓存仍有价值。该只读作业不会持久化代码检出凭据。 - Node 22.19 和 Node 26 分别使用 4 核和 32 核 Linux 池运行各自的运行时兼容性冒烟测试。Python 3.10 使用 8 核 Linux 池运行完整的无密钥 SDK 套件。这些作业属于环境契约，并非主 Node 门禁清单的分片。 -- `windows node 24 / complete` 使用一台 32 核 Windows 运行器。一次设置供必需的包构建、必需的生产网站构建以及完整的观测性可移植性清单共用。任何必需项失败都会使作业失败；观测项失败则报告为非阻塞。ESLint 保持单线程，因为 16 个 ESLint 工作线程耗时 174.54 秒；覆盖率最多使用 12 个工作线程，外层调度器则保留 16 个槽位。该作业仅恢复由 master 刷新的较小 ESLint 缓存，并在干净环境中执行 pnpm 安装，而不恢复或保存包含大量文件的包存储。全部 6 种 Windows 大型运行器规格都在未修改系统级 Developer Mode 注册表项的情况下完成了安装和生产网站基准测试，因此拉取请求关键路径省略了这个多余步骤。 +- `windows node 24 / complete` 使用一台 32 核 Windows 运行器。一轮准备工作供必需的包构建、必需的生产网站构建以及完整的观测性可移植性清单共用。任何必需项失败都会使作业失败；观测项失败则报告为非阻塞。ESLint 保持单线程，因为 16 个 ESLint 工作线程耗时 174.54 秒；覆盖率最多使用 12 个工作线程，外层调度器则保留 16 个槽位。该作业仅恢复由 master 刷新的较小 ESLint 缓存，并在干净环境中执行 pnpm 安装，而不恢复或保存包含大量文件的包存储。全部 6 种 Windows 大型运行器规格都在未修改系统级 Developer Mode 注册表项的情况下完成了安装和生产网站基准测试，因此拉取请求关键路径省略了这个多余步骤。 原有的门禁级和粗粒度主流程分片作业已从工作流中移除。相应的静态、lint、覆盖率、快照和场景分片选择器也已从仓库中移除，因此未使用的诊断路径无法继续维系第二套 CI 架构。 @@ -40,11 +40,11 @@ Windows 仓库工作在超过 16 核后收益很小，但 32 核池可以让完 客户端包依赖图增大后，缓存机制和调度器压力也成为实测工作负载的一部分。在[一次分支头精确的生产运行](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29912577681)中，Linux 的仓库门禁耗时 39 秒，完整作业耗时 69 秒；Windows 的仓库门禁耗时 117 秒，完整作业耗时 228 秒。Windows pnpm 缓存的 154 MB 归档下载耗时约 2 秒，但解压耗时 27 秒，随后安装耗时 23 秒，作业结束后的保存又耗时 14 秒。一次[无缓存的全规格运行轨迹](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29913033155)在 27 秒内完成了同一台 32 核 Windows 运行器上的安装。因此，生产环境不使用 Windows 包存储缓存，在对延迟敏感的拉取请求作业中使用只恢复不保存的缓存，并限制外层并发度，以免类型检查、lint、覆盖率和构建在同一台主机上过度争用资源。 -两项主机效应仍构成这项决策的依据。一个标准 Node 26 作业曾在总共 67 秒的耗时中，把 36 秒用在 `Set up job` 上，因此各项环境契约使用不同的大型运行器池，而非标准容量。一个 Linux 候选作业在注册 50 KB 的 Bubblewrap 包时耗时 18 秒，因为托管映像扫描了 202,507 个包数据库文件。[`scripts/prepare-ci-bubblewrap.sh`](../../../../scripts/prepare-ci-bubblewrap.sh) 改为验证固定包内容并将其解压到临时运行器目录，执行功能性隔离探针，并让这项准备工作与依赖安装重叠执行。 +3 项主机效应仍构成这项决策的依据。一个标准 Node 26 作业曾在总共 67 秒的耗时中，把 36 秒用在 `Set up job` 上，因此各项环境契约使用不同的大型运行器池，而非标准容量。setup-node action 在 Linux 和 Windows 均已从托管 toolcache 找到 Node 24.18.0 后，仍分别花费 3.68 秒和 46.56 秒输出缓存的环境详情。两个延迟关键作业会直接选择最新的预装 24.x 目录并验证其主版本号；如果映像不再提供该目录，作业会明确报错并失败。兼容性作业仍使用 setup-node，因为选择非默认运行时正是它们的契约。一个 Linux 候选作业还在注册 50 KB 的 Bubblewrap 包时耗时 18 秒，因为托管映像扫描了 202,507 个包数据库文件。[`scripts/prepare-ci-bubblewrap.sh`](../../../../scripts/prepare-ci-bubblewrap.sh) 改为验证固定包内容并将其解压到临时运行器目录，执行功能性隔离探针，并让这项准备工作与依赖安装重叠执行。 -内层与外层工作线程上限是相互独立的控制机制。一次[分支头精确、使用 32 个工作线程的 ESLint 实验](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29918329463)使 lint 耗时增至 52.28 秒、覆盖率耗时增至 42.71 秒；同一次运行中，一项适配器空闲超时测试失败。因此，生产环境将 ESLint 工作线程上限维持在 16 个，并且同时最多只运行 8 项相互独立的仓库门禁，为这些门禁自身的工作线程池留出容量。 +内层与外层工作线程上限是相互独立的控制机制。一次[分支头精确、使用 32 个工作线程的 ESLint 实验](https://github.com/deepseek-harness/deepseek-harness/actions/runs/29918329463)使 lint 耗时增至 52.28 秒、覆盖率耗时增至 42.71 秒；同一次运行中，一项适配器空闲超时测试失败。后来一次同时运行 8 项门禁的运行轨迹将覆盖率耗时降至 35.17 秒，但生产网站构建被延后，直到聚合流程耗时达到 41.06 秒时才完成。因此，生产环境将 ESLint 工作线程上限维持在 16 个，并且同时最多运行 10 项相互独立的仓库门禁，既为这些门禁自身的工作线程池留出容量，又避免后续独立工作因资源不足而迟迟无法启动。 -Linux 覆盖率把每个项目的工作线程上限设为 16 个，Windows 则保留 12 个工作线程的上限。进程约束项目恰好包含 4 个套件文件，因此它的 fork 数量不可能达到任一上限。32 个 fork 曾两次导致 Node 24 的 CJS 词法分析器崩溃，后来一次使用 16 个 fork 的运行又复现了工作进程丢失和无效的覆盖率结果。因此，单次 Vitest 调用会对大范围测试清单使用线程，只为涉及进程全局状态或 `process` API 的套件保留 fork。这份有限的 fork 清单还包含本地 bash 进程通路套件：在聚合门禁争用资源时，该套件的工作线程虽然完成了所有测试，却会间歇性漏记逐文件函数覆盖率所需的 stdin 错误回调。在 96 核主机上使用 32 个工作线程运行全部门禁时，覆盖率耗时变慢至 44.6 秒，还使一项计算预算回归超过其 1 秒阈值，因此生产环境将工作线程数限制在 16 个以内。这样既能保留这些套件的隔离契约和覆盖率结果的确定性，又能避免以 fork 方式执行普通测试文件。 +Linux 覆盖率把每个项目的工作线程上限设为 16 个，Windows 则保留 12 个工作线程的上限。进程约束项目恰好包含 5 个套件文件，因此它的 fork 数量不可能达到任一上限。32 个 fork 曾两次导致 Node 24 的 CJS 词法分析器崩溃，后来一次使用 16 个 fork 的运行又复现了工作进程丢失和无效的覆盖率结果。因此，单次 Vitest 调用会对大范围测试清单使用线程，只为涉及进程全局状态、`process` API 或对时间敏感的进程 I/O 的套件保留 fork。这份有限的 fork 清单还包含本地 bash 进程通路套件：在聚合门禁争用资源时，该套件的工作线程虽然完成了所有测试，却会间歇性漏记逐文件函数覆盖率所需的 stdin 错误回调。两次托管聚合运行都将空闲看门狗对套接字关闭的观测延迟到超过其 100 毫秒测试截止时间，因此这份清单还包含 pi-ai 适配器套件。在 96 核主机上使用 32 个工作线程运行全部门禁时，覆盖率耗时变慢至 44.6 秒，还使一项计算预算回归超过其 1 秒阈值，因此生产环境将工作线程数限制在 16 个以内。这样既能保留这些套件的隔离契约和覆盖率结果的确定性，又能避免以 fork 方式执行普通测试文件。 工作流保留 2 项手动测量套件。`suite=larger-runner-benchmark` 比较所有规格下相互独立的关键通道，`suite=consolidated-runner-benchmark` 比较完整聚合流程。只有在 `master` 移动时，才运行完整的 Linux、macOS 和 Windows 串行参考；拉取请求只运行优化后的作业。 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 42d1c2b1f6..f52274cddd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,7 +38,7 @@ jobs: DSH_COVERAGE_MAX_WORKERS: '16' DSH_ESLINT_CACHE: '1' DSH_ESLINT_CONCURRENCY: '16' - DSH_GATE_CONCURRENCY: '8' + DSH_GATE_CONCURRENCY: '10' DSH_PUBLINT_CONCURRENCY: '16' DSH_SNAPSHOT_MAX_CONCURRENCY: '8' steps: @@ -46,21 +46,11 @@ jobs: with: persist-credentials: false - - uses: actions/setup-node@v6 - with: - node-version: ${{ env.PRIMARY_NODE_VERSION }} - - - name: Enable corepack and resolve pnpm store path - id: pnpm-store - run: | - corepack enable - echo "path=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" - # Pull requests consume the default-branch cache but do not put cache # compression and upload on the paid latency-critical path. - uses: actions/cache/restore@v4 with: - path: ${{ steps.pnpm-store.outputs.path }} + path: /home/runner/.local/share/pnpm/store/v11 key: ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm-${{ hashFiles('pnpm-lock.yaml') }} restore-keys: | ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-pnpm- @@ -72,8 +62,17 @@ jobs: restore-keys: | ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-full- - - name: Install dependencies and prepare bubblewrap + - name: Select preinstalled Node, install dependencies, and prepare bubblewrap run: | + node_root="$(printf '%s\n' "$RUNNER_TOOL_CACHE"/node/"${PRIMARY_NODE_VERSION}".*/x64 | sort -V | tail -n 1)" + if [[ ! -d "$node_root" ]]; then + echo "preinstalled Node ${PRIMARY_NODE_VERSION}.x not found in $RUNNER_TOOL_CACHE" >&2 + exit 1 + fi + echo "$node_root/bin" >> "$GITHUB_PATH" + export PATH="$node_root/bin:$PATH" + [[ "$(node --version)" == "v${PRIMARY_NODE_VERSION}."* ]] + corepack enable pnpm install --frozen-lockfile & install_pid=$! bash scripts/prepare-ci-bubblewrap.sh & @@ -171,10 +170,6 @@ jobs: steps: - uses: actions/checkout@v6 - - uses: actions/setup-node@v6 - with: - node-version: ${{ env.PRIMARY_NODE_VERSION }} - - uses: actions/cache/restore@v4 with: path: .cache/eslint @@ -184,9 +179,25 @@ jobs: # Extracting the many-file pnpm store cache is slower on this image than # a clean parallel install, and saving it adds more latency after gates. - - name: Enable corepack and install (immutable) + - name: Select preinstalled Node and install (immutable) shell: pwsh run: | + $nodeRoot = Get-ChildItem -Path "$env:RUNNER_TOOL_CACHE\node" -Directory | + Where-Object { $_.Name -like "$env:PRIMARY_NODE_VERSION.*" } | + Sort-Object { [version]$_.Name } | + Select-Object -Last 1 + if ($null -eq $nodeRoot) { + throw "preinstalled Node $env:PRIMARY_NODE_VERSION.x not found in $env:RUNNER_TOOL_CACHE" + } + $nodeBin = Join-Path $nodeRoot.FullName 'x64' + if (-not (Test-Path $nodeBin -PathType Container)) { + throw "preinstalled Node x64 directory not found at $nodeBin" + } + Add-Content -Path $env:GITHUB_PATH -Value $nodeBin + $env:PATH = "$nodeBin;$env:PATH" + if ((node --version) -notlike "v$env:PRIMARY_NODE_VERSION.*") { + throw "selected unexpected Node version $(node --version)" + } corepack enable pnpm install --frozen-lockfile diff --git a/vitest.config.ts b/vitest.config.ts index 51ced4b19f..47ac2674a4 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -28,11 +28,13 @@ const testIncludes = [ 'scripts/**/*.spec.ts', ] -// These suites exercise process-global state or process APIs that worker threads cannot isolate. +// These suites exercise process-global state, process APIs, or timing-sensitive process I/O +// that worker threads cannot isolate reliably under aggregate gate contention. // Keep the narrow exception in forks while the rest of the inventory avoids per-file processes. const processBoundTests = [ 'packages/bash/bash-local/tests/run.spec.ts', 'packages/context/time-context/tests/time-context.spec.ts', + 'packages/llm/llm-pi-ai/tests/adapter.spec.ts', 'packages/ui/app-boot/tests/app-boot.spec.ts', 'packages/workflow/workflow-workerthread/tests/session.spec.ts', ]