From 7813dbcf0f227c7a7c1807e02a35c9b52a796bb8 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Tue, 21 Jul 2026 21:14:55 +0800 Subject: [PATCH] ci: bound hosted fanout --- .../2026-07-06-parallel-github-ci-gates.md | 4 +- .github/workflows/ci.yml | 70 ++++--------------- scripts/coverage-shards.ts | 17 ++--- scripts/run-gates.ts | 3 +- scripts/snapshot-shards.spec.ts | 4 +- scripts/snapshot-shards.ts | 13 ++-- 6 files changed, 26 insertions(+), 85 deletions(-) diff --git a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md index a77493d8a9..5321db9bd7 100644 --- a/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md +++ b/.agents/notes/implemented/process/2026-07-06-parallel-github-ci-gates.md @@ -16,9 +16,9 @@ The artifact boundary remains load-bearing. `publint`, `verify-node-next-types`, [scripts/run-gates.ts](../../../../scripts/run-gates.ts) remains the common bounded scheduler, but GitHub supplies explicit shard names for the expensive gate families. [scripts/static-shards.ts](../../../../scripts/static-shards.ts) partitions static gates into foundation, API-contract, catalog, prose, documentation-projection, and documentation-build lanes and rejects a missing or duplicate gate assignment. Lint uses disjoint package-source, package-test, and repository-complement lanes; the complement still starts from `.` so a new top-level lint target cannot disappear between shards, and it owns the single cross-file duplication run. [scripts/coverage-shards.ts](../../../../scripts/coverage-shards.ts) assigns every workspace package to exactly one source-coverage lane; its test expands the live package tree, so a new package makes CI red until it has an owner. Each coverage lane includes only its owned source files, repeats the exhaustive companion topology test, and runs without a preceding build because the complete coverage suite passes from a tree with every generated `lib/` removed. -Snapshot replay uses three explicit multi-file lanes and four scenario partitions of the large ACP file. [scripts/snapshot-shards.ts](../../../../scripts/snapshot-shards.ts) owns that inventory, and its test discovers every file admitted by the snapshot config so a new file cannot land outside CI. Each snapshot job builds the shipped runtime while its Linux runner installs bubblewrap from the hosted image's existing package index, then runs only its assigned replay surface. CI explicitly retains the suite's bounded concurrency of five subprocesses instead of clamping it to the runner's two logical CPUs, because replay spends most of its time waiting on child protocol I/O. Fixture guards still inspect the complete ACP scenario table in every partition. Static, lint, coverage, and snapshot sharding changes only GitHub scheduling: the ordinary local package scripts still run their complete suites. +Snapshot replay uses two explicit multi-file lanes and two scenario partitions of the large ACP file. [scripts/snapshot-shards.ts](../../../../scripts/snapshot-shards.ts) owns that inventory, and its test discovers every file admitted by the snapshot config so a new file cannot land outside CI. Each snapshot job builds the shipped runtime while its Linux runner installs bubblewrap from the hosted image's existing package index, then runs only its assigned replay surface. CI explicitly retains the suite's bounded concurrency of five subprocesses instead of clamping it to the runner's two logical CPUs, because replay spends most of its time waiting on child protocol I/O. Fixture guards still inspect the complete ACP scenario table in every partition. Static, lint, coverage, and snapshot sharding changes only GitHub scheduling: the ordinary local package scripts still run their complete suites. -Artifacts use three lanes: one metadata lane for `publint`, NodeNext declarations, and compiled invariant loading, plus two Vitest shards for built-bin smoke. Each lane produces its own build before its consumers. Repeating the short build costs runner minutes but avoids an upload/download dependency and keeps each job's critical path bounded. +Artifacts use two lanes: one metadata lane for `publint`, NodeNext declarations, and compiled invariant loading, plus one built-bin smoke lane. Each lane produces its own build before its consumers. Repeating the short build costs runner minutes but avoids an upload/download dependency and keeps each job's critical path bounded. [scripts/publint-all.ts](../../../../scripts/publint-all.ts) calls publint's supported API in-process against an in-memory publication view made from each manifest's declared files and npm's mandatory metadata files. This preserves the distinction between workspace files and published files without spawning a package-manager pack command 103 times. [scripts/verify-built-package-invariants.mjs](../../../../scripts/verify-built-package-invariants.mjs) stages those structurally validated manifest-declared `lib/` files below the real package, then imports the compiled self-reference through plain Node and Cordis Loader normalization. A companion that reaches an undeclared runtime chunk still fails. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d78cb52af4..cb08cefae7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,16 +82,16 @@ jobs: gate_concurrency: '1' coverage_max_workers: '4' coverage_shard: core - - lane: coverage-models + - lane: coverage-models-protocol command: pnpm run check:ci:coverage gate_concurrency: '1' coverage_max_workers: '4' - coverage_shard: models - - lane: coverage-sdk + coverage_shard: models-protocol + - lane: coverage-sdk-codex command: pnpm run check:ci:coverage gate_concurrency: '1' coverage_max_workers: '4' - coverage_shard: sdk + coverage_shard: sdk-codex - lane: coverage-interfaces command: pnpm run check:ci:coverage gate_concurrency: '1' @@ -122,36 +122,16 @@ jobs: gate_concurrency: '1' coverage_max_workers: '4' coverage_shard: repository - - lane: coverage-scripts + - lane: coverage-integrations-claude command: pnpm run check:ci:coverage gate_concurrency: '1' coverage_max_workers: '4' - coverage_shard: scripts - - lane: coverage-integrations + coverage_shard: integrations-claude + - lane: coverage-session-scripts command: pnpm run check:ci:coverage gate_concurrency: '1' coverage_max_workers: '4' - coverage_shard: integrations - - lane: coverage-session-state - command: pnpm run check:ci:coverage - gate_concurrency: '1' - coverage_max_workers: '4' - coverage_shard: session-state - - lane: coverage-hooks-claude - command: pnpm run check:ci:coverage - gate_concurrency: '1' - coverage_max_workers: '4' - coverage_shard: hooks-claude - - lane: coverage-hook-protocol - command: pnpm run check:ci:coverage - gate_concurrency: '1' - coverage_max_workers: '4' - coverage_shard: hook-protocol - - lane: coverage-hooks-codex - command: pnpm run check:ci:coverage - gate_concurrency: '1' - coverage_max_workers: '4' - coverage_shard: hooks-codex + coverage_shard: session-scripts - lane: coverage-capabilities command: pnpm run check:ci:coverage gate_concurrency: '1' @@ -163,12 +143,6 @@ jobs: snapshot_lane: support snapshot_max_concurrency: '5' snapshot_prebuilt: '1' - - lane: snapshot-demos - command: pnpm run check:ci:snapshot - gate_concurrency: '1' - snapshot_lane: demos - snapshot_max_concurrency: '5' - snapshot_prebuilt: '1' - lane: snapshot-agents command: pnpm run check:ci:snapshot gate_concurrency: '1' @@ -187,31 +161,15 @@ jobs: snapshot_lane: acp-2 snapshot_max_concurrency: '5' snapshot_prebuilt: '1' - - lane: snapshot-acp-3 - command: pnpm run check:ci:snapshot - gate_concurrency: '1' - snapshot_lane: acp-3 - snapshot_max_concurrency: '5' - snapshot_prebuilt: '1' - - lane: snapshot-acp-4 - command: pnpm run check:ci:snapshot - gate_concurrency: '1' - snapshot_lane: acp-4 - snapshot_max_concurrency: '5' - snapshot_prebuilt: '1' - lane: artifacts-metadata command: pnpm run check:ci:artifacts gate_concurrency: '3' publint_concurrency: '8' artifact_shard: metadata - - lane: artifacts-smoke-1 + - lane: artifacts-smoke command: pnpm run check:ci:artifacts gate_concurrency: '1' - artifact_shard: smoke-1 - - lane: artifacts-smoke-2 - command: pnpm run check:ci:artifacts - gate_concurrency: '1' - artifact_shard: smoke-2 + artifact_shard: smoke steps: - uses: actions/checkout@v6 @@ -409,14 +367,10 @@ jobs: gate_concurrency: '3' publint_concurrency: '8' artifact_shard: metadata - - lane: artifacts-smoke-1 + - lane: artifacts-smoke command: pnpm run check:ci:artifacts gate_concurrency: '1' - artifact_shard: smoke-1 - - lane: artifacts-smoke-2 - command: pnpm run check:ci:artifacts - gate_concurrency: '1' - artifact_shard: smoke-2 + artifact_shard: smoke steps: - uses: actions/checkout@v6 diff --git a/scripts/coverage-shards.ts b/scripts/coverage-shards.ts index 5ac8f999f3..f866bbb04d 100644 --- a/scripts/coverage-shards.ts +++ b/scripts/coverage-shards.ts @@ -17,8 +17,8 @@ export const coverageShards = [ packageRoots: ['core', 'context'], extraTestRoots: ['packages/examples/cli-demo/tests'], }, - { name: 'models', packageRoots: ['llm', 'compact'] }, - { name: 'sdk', packageRoots: ['sdk'] }, + { name: 'models-protocol', packageRoots: ['llm', 'compact', 'hooks/hook-protocol'] }, + { name: 'sdk-codex', packageRoots: ['sdk', 'hooks/hooks-codex'] }, { name: 'interfaces', packageRoots: ['ui', 'examples', 'goal'], @@ -42,18 +42,11 @@ export const coverageShards = [ ], }, { - name: 'scripts', - packageRoots: ['support/invariants'], + name: 'session-scripts', + packageRoots: ['session-persistence', 'session-query', 'support/invariants'], extraTestRoots: ['scripts'], }, - { name: 'integrations', packageRoots: ['lsp', 'mcp'] }, - { - name: 'session-state', - packageRoots: ['session-persistence', 'session-query'], - }, - { name: 'hook-protocol', packageRoots: ['hooks/hook-protocol'] }, - { name: 'hooks-claude', packageRoots: ['hooks/hooks-claude'] }, - { name: 'hooks-codex', packageRoots: ['hooks/hooks-codex'] }, + { name: 'integrations-claude', packageRoots: ['lsp', 'mcp', 'hooks/hooks-claude'] }, { name: 'capabilities', packageRoots: ['web', 'skill', 'spill', 'util', 'guard', 'todo', 'timeout'], diff --git a/scripts/run-gates.ts b/scripts/run-gates.ts index e45a254905..efd94c92a0 100644 --- a/scripts/run-gates.ts +++ b/scripts/run-gates.ts @@ -263,8 +263,7 @@ function ciArtifactGates(): Gate[] { builtPackageInvariantsGate(['build']), ] if (shard === 'metadata') return metadataGates - if (shard === 'smoke-1') return [pnpmScript('build', 'build'), builtBinSmokeGate('1/2')] - if (shard === 'smoke-2') return [pnpmScript('build', 'build'), builtBinSmokeGate('2/2')] + if (shard === 'smoke') return [pnpmScript('build', 'build'), builtBinSmokeGate()] if (shard !== undefined && shard !== '') { throw new Error(`run-gates: unknown DSH_ARTIFACT_SHARD ${JSON.stringify(shard)}.`) } diff --git a/scripts/snapshot-shards.spec.ts b/scripts/snapshot-shards.spec.ts index dc4bef62ad..9f3c4e7cbd 100644 --- a/scripts/snapshot-shards.spec.ts +++ b/scripts/snapshot-shards.spec.ts @@ -26,10 +26,10 @@ describe('snapshot lanes', () => { expect(new Set(ordinary).size).toBe(ordinary.length) expect(acp.map(lane => lane.files)).toEqual(Array.from( - { length: 4 }, + { length: 2 }, () => ['examples/acp-agent/tests/acp.snapshot.ts'], )) - expect(acp.map(lane => lane.scenarioShard)).toEqual(['1/4', '2/4', '3/4', '4/4']) + expect(acp.map(lane => lane.scenarioShard)).toEqual(['1/2', '2/2']) expect([...ordinary, 'examples/acp-agent/tests/acp.snapshot.ts'].sort()).toEqual(discovered) }) diff --git a/scripts/snapshot-shards.ts b/scripts/snapshot-shards.ts index a6126f23c2..3106297a2e 100644 --- a/scripts/snapshot-shards.ts +++ b/scripts/snapshot-shards.ts @@ -16,27 +16,22 @@ export const snapshotLanes: readonly SnapshotLane[] = [ name: 'support', files: [ 'packages/sdk/scripts/tests/config.snapshot.ts', - 'packages/ui/tui/tests/tui.snapshot.ts', - ], - }, - { - name: 'demos', - files: [ - 'examples/tui-agent/tests/tui.snapshot.ts', 'packages/sdk/create-sdk/tests/create.snapshot.ts', + 'packages/ui/tui/tests/tui.snapshot.ts', ], }, { name: 'agents', files: [ + 'examples/tui-agent/tests/tui.snapshot.ts', 'examples/acp-agent/tests/goal.snapshot.ts', 'examples/headless-agent/tests/headless.snapshot.ts', ], }, - ...Array.from({ length: 4 }, (_, offset) => ({ + ...Array.from({ length: 2 }, (_, offset) => ({ name: `acp-${offset + 1}`, files: ['examples/acp-agent/tests/acp.snapshot.ts'], - scenarioShard: `${offset + 1}/4`, + scenarioShard: `${offset + 1}/2`, })), ]