From 9d81078f2f8476f60a22bde645f4a11c69b16bbc Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 16:10:52 +0800 Subject: [PATCH] ci: split primary cold-path budget --- .github/workflows/ci.yml | 61 +++++++++++++++++++++++++++++++--------- package.json | 1 + scripts/run-gates.ts | 59 +++++++++++++++++++++++++++----------- 3 files changed, 91 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41a49c4269..17cd71ffde 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -319,19 +319,16 @@ jobs: run: ${{ matrix.command }} # One large runner pays hosted setup once, then the repository scheduler - # overlaps the unsharded primary inventory except the production site build. + # overlaps the unsharded core inventory. CPU and site chains use coarse lanes. node-24: if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' runs-on: dsh-ubuntu-24-04-96core name: node 24 / core env: - # Thirty-two Vitest forks intermittently crash Node's CJS lexer on this image. - DSH_COVERAGE_MAX_WORKERS: '16' DSH_ESLINT_CACHE: '1' DSH_ESLINT_CONCURRENCY: '32' DSH_GATE_CONCURRENCY: '32' DSH_PUBLINT_CONCURRENCY: '32' - DSH_SNAPSHOT_MAX_CONCURRENCY: '32' steps: - uses: actions/checkout@v6 @@ -383,8 +380,8 @@ jobs: - name: Run unsharded primary Node core CI concurrently run: pnpm run check:ci:large-runner - # Keep only the longest independent Linux gate on a second coarse-grained - # runner so cold install variance does not push the primary box over a minute. + # Keep the longest independent Linux gate on its own coarse-grained runner so + # cold install variance does not push either primary aggregate over a minute. node-24-site: if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' runs-on: dsh-ubuntu-24-04-16core @@ -418,24 +415,39 @@ jobs: node-compat: if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' # Distinct larger-runner pools avoid both standard-runner setup outliers and - # the delayed second allocation observed when two jobs shared one pool. + # the delayed second allocation observed when two jobs shared one pool. The + # primary Node row also owns the CPU/dependency-critical primary gate chain. runs-on: ${{ matrix.runner }} - name: node ${{ matrix.node }} + name: ${{ matrix.name }} env: - DSH_GATE_CONCURRENCY: '2' + DSH_COVERAGE_MAX_WORKERS: ${{ matrix.coverage_workers }} + DSH_GATE_CONCURRENCY: ${{ matrix.gate_concurrency }} DSH_NODE_COMPAT_SKIP_TYPECHECK: ${{ matrix.skip_typecheck }} + DSH_SNAPSHOT_MAX_CONCURRENCY: ${{ matrix.snapshot_workers }} strategy: fail-fast: false matrix: include: - node: '22.19' + name: node 22.19 runner: dsh-ubuntu-24-04-4core + command: pnpm run check:node-compat + gate_concurrency: '2' skip_typecheck: '1' - node: 24 - runner: dsh-ubuntu-24-04-8core + name: node 24 / cpu + runner: dsh-ubuntu-24-04-64core + command: pnpm run check:ci:primary-cpu + coverage_workers: '16' + gate_concurrency: '6' + primary_cpu: true + snapshot_workers: '16' skip_typecheck: '1' - node: 26 + name: node 26 runner: dsh-ubuntu-24-04-32core + command: pnpm run check:node-compat + gate_concurrency: '2' skip_typecheck: '1' steps: - uses: actions/checkout@v6 @@ -458,14 +470,37 @@ jobs: ${{ runner.os }}-node-${{ matrix.node }}-pnpm- - name: Install (immutable) + if: matrix.primary_cpu != true run: pnpm install --frozen-lockfile - - name: Run compatibility gates - run: pnpm run check:node-compat + - name: Install and prepare bubblewrap + if: matrix.primary_cpu == true + run: | + pnpm install --frozen-lockfile & + install_pid=$! + ( + if ! sudo apt-get install -yq --no-install-recommends bubblewrap; then + echo "initial bubblewrap install failed; refreshing APT indexes and retrying" + sudo apt-get update -q + sudo apt-get install -yq --no-install-recommends bubblewrap + fi + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 \ + || echo "apparmor userns knob absent — the functional probe decides" + ) & + sandbox_pid=$! + install_status=0 + wait "$install_pid" || install_status=$? + sandbox_status=0 + wait "$sandbox_pid" || sandbox_status=$? + if (( install_status != 0 )); then exit "$install_status"; fi + exit "$sandbox_status" + + - name: Run assigned gates + run: ${{ matrix.command }} python-sdk: if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' - runs-on: dsh-ubuntu-24-04-64core + runs-on: dsh-ubuntu-24-04-8core name: python 3.10 / keyless SDK steps: - uses: actions/checkout@v6 diff --git a/package.json b/package.json index c9e5e01a71..c318a3a94d 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "test:snapshot:record": "DSH_SNAPSHOT=record vitest run --config vitest.snapshot.config.ts --update", "test:snapshot:refresh": "DSH_SNAPSHOT=refresh vitest run --config vitest.snapshot.config.ts", "check:ci": "tsx scripts/run-gates.ts ci-primary", + "check:ci:primary-cpu": "tsx scripts/run-gates.ts ci-primary-cpu", "check:ci:large-runner": "tsx scripts/run-gates.ts ci-primary-large-runner", "check:ci:static": "tsx scripts/run-gates.ts ci-static", "check:ci:lint": "tsx scripts/run-gates.ts ci-lint", diff --git a/scripts/run-gates.ts b/scripts/run-gates.ts index 11155551db..ff4b1aab2d 100644 --- a/scripts/run-gates.ts +++ b/scripts/run-gates.ts @@ -15,6 +15,7 @@ import { selectStaticGates } from './static-shards.ts' type Mode = | 'ci-primary' + | 'ci-primary-cpu' | 'ci-primary-large-runner' | 'ci-static' | 'ci-lint' @@ -92,6 +93,7 @@ if (results.some(result => result.gate.allowFailure !== true && (result.status = function parseMode(raw: string | undefined): Mode { switch (raw) { case 'ci-primary': + case 'ci-primary-cpu': case 'ci-primary-large-runner': case 'ci-static': case 'ci-lint': @@ -107,7 +109,7 @@ function parseMode(raw: string | undefined): Mode { return raw default: throw new Error( - `run-gates: expected mode ci-primary | ci-primary-large-runner | ci-static | ci-lint | ci-coverage | ci-snapshot | ci-artifacts | ci-windows-blocking | ci-windows-complete | ci-windows-observational | node-compat | pre-push | doc-sync, got ${JSON.stringify(raw)}.`, + `run-gates: expected mode ci-primary | ci-primary-cpu | ci-primary-large-runner | ci-static | ci-lint | ci-coverage | ci-snapshot | ci-artifacts | ci-windows-blocking | ci-windows-complete | ci-windows-observational | node-compat | pre-push | doc-sync, got ${JSON.stringify(raw)}.`, ) } } @@ -173,6 +175,8 @@ function gatesForMode(selected: Mode): Gate[] { switch (selected) { case 'ci-primary': return ciPrimaryGates() + case 'ci-primary-cpu': + return ciPrimaryCpuGates() case 'ci-primary-large-runner': return ciPrimaryLargeRunnerGates() case 'ci-static': @@ -199,19 +203,7 @@ function gatesForMode(selected: Mode): Gate[] { case 'ci-windows-observational': return ciWindowsObservationalGates() case 'node-compat': - return [ - ...flagEnabled('DSH_NODE_COMPAT_SKIP_TYPECHECK') ? [] : [pnpmScript('typecheck', 'typecheck')], - pnpmExec('source-worker-smoke', [ - 'vitest', - 'run', - 'packages/workflow/workflow-workerthread/tests/source-worker.compat.spec.ts', - ], { label: 'source worker smoke' }), - pnpmExec('jsonl-zstd-smoke', [ - 'vitest', - 'run', - 'packages/session-persistence/session-persistence-jsonl/tests/zstd.compat.spec.ts', - ], { label: 'JSONL Zstandard smoke' }), - ] + return nodeCompatGates() case 'pre-push': return [ pnpmScript('runtime-closure', 'verify-runtime-closure', { label: 'runtime closure' }), @@ -258,10 +250,10 @@ function ciPrimaryGates(): Gate[] { } function ciPrimaryLargeRunnerGates(): Gate[] { - // Typecheck does not consume build output, so a large runner can start both - // together while snapshot and artifact consumers still wait for the build. + // The CPU lane owns typecheck, coverage, and the build-to-snapshot chain. + // This core lane starts its own build eagerly for the remaining artifact consumers. return ciPrimaryGates() - .filter(gate => gate.id !== 'docs-site-build') + .filter(gate => !['coverage', 'docs-site-build', 'snapshot', 'typecheck'].includes(gate.id)) .map((gate) => { if (gate.id !== 'build') return gate const eagerBuild = { ...gate } @@ -270,6 +262,39 @@ function ciPrimaryLargeRunnerGates(): Gate[] { }) } +function ciPrimaryCpuGates(): Gate[] { + // Build and snapshot stay together so the dependent replay consumes this lane's output. + return [ + pnpmScript('typecheck', 'typecheck'), + coverageGate(), + pnpmScript('build', 'build'), + snapshotGate(), + ...nodeCompatSmokeGates(), + ] +} + +function nodeCompatGates(): Gate[] { + return [ + ...flagEnabled('DSH_NODE_COMPAT_SKIP_TYPECHECK') ? [] : [pnpmScript('typecheck', 'typecheck')], + ...nodeCompatSmokeGates(), + ] +} + +function nodeCompatSmokeGates(): Gate[] { + return [ + pnpmExec('source-worker-smoke', [ + 'vitest', + 'run', + 'packages/workflow/workflow-workerthread/tests/source-worker.compat.spec.ts', + ], { label: 'source worker smoke' }), + pnpmExec('jsonl-zstd-smoke', [ + 'vitest', + 'run', + 'packages/session-persistence/session-persistence-jsonl/tests/zstd.compat.spec.ts', + ], { label: 'JSONL Zstandard smoke' }), + ] +} + function ciStaticGates(): Gate[] { const gates = [ pnpmScript('runtime-closure', 'verify-runtime-closure', { label: 'runtime closure' }),