From 990f82b85cf748638c04272f8cada333ca2dee39 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Wed, 22 Jul 2026 15:17:17 +0800 Subject: [PATCH] ci: tune consolidated runner concurrency --- .github/workflows/ci.yml | 28 ++++++++++------------------ package.json | 2 +- scripts/run-gates.ts | 24 ++++++++++++++---------- 3 files changed, 25 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e1135f447..121d82d701 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -325,7 +325,8 @@ jobs: runs-on: dsh-ubuntu-24-04-96core name: node 24 / complete env: - DSH_COVERAGE_MAX_WORKERS: '32' + # Thirty-two Vitest forks intermittently crash Node's CJS lexer on this image. + DSH_COVERAGE_MAX_WORKERS: '16' DSH_ESLINT_CACHE: '1' DSH_ESLINT_CONCURRENCY: '32' DSH_GATE_CONCURRENCY: '32' @@ -358,7 +359,7 @@ jobs: restore-keys: | ${{ runner.os }}-node-${{ env.PRIMARY_NODE_VERSION }}-eslint-full- - - name: Install, build, and prepare bubblewrap + - name: Install and prepare bubblewrap run: | pnpm install --frozen-lockfile & install_pid=$! @@ -374,18 +375,13 @@ jobs: sandbox_pid=$! install_status=0 wait "$install_pid" || install_status=$? - build_status=0 - if (( install_status == 0 )); then - pnpm run build || build_status=$? - fi sandbox_status=0 wait "$sandbox_pid" || sandbox_status=$? if (( install_status != 0 )); then exit "$install_status"; fi - if (( build_status != 0 )); then exit "$build_status"; fi exit "$sandbox_status" - name: Run complete unsharded primary Node CI concurrently - run: pnpm run check:ci:prebuilt + run: pnpm run check:ci:large-runner node-compat: if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' @@ -628,13 +624,14 @@ jobs: windows-gates: if: github.event_name != 'workflow_dispatch' || inputs.suite == 'optimized-larger-runners' continue-on-error: true - runs-on: dsh-windows-2025-16core + runs-on: dsh-windows-2025-32core name: windows node 24 / observational env: + # Keep ESLint itself single-threaded: 16 ESLint workers took 174 seconds on + # this image. The outer scheduler still overlaps lint with the other gates. DSH_ESLINT_CACHE: '1' - DSH_ESLINT_CONCURRENCY: '16' - DSH_GATE_CONCURRENCY: '16' - DSH_PUBLINT_CONCURRENCY: '16' + DSH_GATE_CONCURRENCY: '32' + DSH_PUBLINT_CONCURRENCY: '32' steps: - uses: actions/checkout@v6 @@ -995,14 +992,9 @@ jobs: sandbox_pid=$! install_status=0 wait "$install_pid" || install_status=$? - build_status=0 - if (( install_status == 0 )); then - pnpm run build || build_status=$? - fi sandbox_status=0 wait "$sandbox_pid" || sandbox_status=$? if (( install_status != 0 )); then exit "$install_status"; fi - if (( build_status != 0 )); then exit "$build_status"; fi exit "$sandbox_status" - name: Install (immutable) @@ -1019,7 +1011,7 @@ jobs: DSH_GATE_CONCURRENCY: ${{ matrix.workers }} DSH_PUBLINT_CONCURRENCY: ${{ matrix.workers }} DSH_SNAPSHOT_MAX_CONCURRENCY: ${{ matrix.workers }} - run: pnpm run check:ci:prebuilt + run: pnpm run check:ci:large-runner - name: Run blocking Windows builds concurrently if: matrix.platform == 'windows' diff --git a/package.json b/package.json index a8ee301da3..c3aaa3c29d 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "test:snapshot:record": "DSH_SNAPSHOT=record vitest run --config vitest.snapshot.config.ts --update", "test:snapshot:refresh": "DSH_SNAPSHOT=refresh vitest run --config vitest.snapshot.config.ts", "check:ci": "tsx scripts/run-gates.ts ci-primary", - "check:ci:prebuilt": "tsx scripts/run-gates.ts ci-primary-prebuilt", + "check:ci:large-runner": "tsx scripts/run-gates.ts ci-primary-large-runner", "check:ci:static": "tsx scripts/run-gates.ts ci-static", "check:ci:lint": "tsx scripts/run-gates.ts ci-lint", "check:ci:coverage": "tsx scripts/run-gates.ts ci-coverage", diff --git a/scripts/run-gates.ts b/scripts/run-gates.ts index c9d058518b..80c9c9bc5c 100644 --- a/scripts/run-gates.ts +++ b/scripts/run-gates.ts @@ -15,7 +15,7 @@ import { selectStaticGates } from './static-shards.ts' type Mode = | 'ci-primary' - | 'ci-primary-prebuilt' + | 'ci-primary-large-runner' | 'ci-static' | 'ci-lint' | 'ci-coverage' @@ -88,7 +88,7 @@ if (results.some(result => result.status === 'failed' || result.status === 'skip function parseMode(raw: string | undefined): Mode { switch (raw) { case 'ci-primary': - case 'ci-primary-prebuilt': + case 'ci-primary-large-runner': case 'ci-static': case 'ci-lint': case 'ci-coverage': @@ -102,7 +102,7 @@ function parseMode(raw: string | undefined): Mode { return raw default: throw new Error( - `run-gates: expected mode ci-primary | ci-primary-prebuilt | ci-static | ci-lint | ci-coverage | ci-snapshot | ci-artifacts | ci-windows-blocking | ci-windows-observational | node-compat | pre-push | doc-sync, got ${JSON.stringify(raw)}.`, + `run-gates: expected mode ci-primary | ci-primary-large-runner | ci-static | ci-lint | ci-coverage | ci-snapshot | ci-artifacts | ci-windows-blocking | ci-windows-observational | node-compat | pre-push | doc-sync, got ${JSON.stringify(raw)}.`, ) } } @@ -168,8 +168,8 @@ function gatesForMode(selected: Mode): Gate[] { switch (selected) { case 'ci-primary': return ciPrimaryGates() - case 'ci-primary-prebuilt': - return ciPrimaryPrebuiltGates() + case 'ci-primary-large-runner': + return ciPrimaryLargeRunnerGates() case 'ci-static': return ciStaticGates() case 'ci-lint': { @@ -250,12 +250,16 @@ function ciPrimaryGates(): Gate[] { ] } -function ciPrimaryPrebuiltGates(): Gate[] { +function ciPrimaryLargeRunnerGates(): Gate[] { + // Typecheck does not consume build output, so a large runner can start both + // together while snapshot and artifact consumers still wait for the build. return ciPrimaryGates() - .filter(gate => gate.id !== 'build') - .map(gate => gate.needs?.includes('build') === true - ? { ...gate, needs: gate.needs.filter(id => id !== 'build') } - : gate) + .map((gate) => { + if (gate.id !== 'build') return gate + const eagerBuild = { ...gate } + delete eagerBuild.needs + return eagerBuild + }) } function ciStaticGates(): Gate[] {