From e23c201da87955e8c50f75c3ee4504fd6866d369 Mon Sep 17 00:00:00 2001 From: Chinesezjc Date: Wed, 22 Jul 2026 13:21:18 +0800 Subject: [PATCH 1/2] perf(ci): parallelize packed-companion probes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit verify-built-package-invariants ran 100+ npm-pack + plain-Node probes serially, dominating the CI artifacts lane (~4min of a ~5min job; ~7.5min on Windows). Each probe stages its packed view inside its own package and spawns its own processes, so the probes are independent — run them through the same bounded worker pool shape as publint-all, capped by DSH_BUILT_INVARIANTS_CONCURRENCY (default availableParallelism), failures kept in manifest order. Measured on the gate alone: 2m07s serial -> 17s at concurrency 8. CI lanes pin the cap to 8, matching DSH_PUBLINT_CONCURRENCY. --- .../2026-07-06-parallel-pre-push-gates.md | 2 + .github/workflows/ci.yml | 12 +++ scripts/verify-built-package-invariants.mjs | 93 ++++++++++++++----- 3 files changed, 85 insertions(+), 22 deletions(-) diff --git a/.agents/notes/implemented/process/2026-07-06-parallel-pre-push-gates.md b/.agents/notes/implemented/process/2026-07-06-parallel-pre-push-gates.md index 710c37cb3a..e649c8c2c9 100644 --- a/.agents/notes/implemented/process/2026-07-06-parallel-pre-push-gates.md +++ b/.agents/notes/implemented/process/2026-07-06-parallel-pre-push-gates.md @@ -20,6 +20,8 @@ The build gate makes the hook self-contained from a clean worktree. `publint`, ` [scripts/publint-all.ts](../../../../scripts/publint-all.ts) discovers the package list from `packages//` and runs `publint` with a worker pool sized from `availableParallelism()`. `DSH_PUBLINT_CONCURRENCY` can cap or raise the worker count for local machines and CI runners with different resource profiles. Results are buffered per package and printed in deterministic package order, so parallel execution does not scramble each package's log block. +[scripts/verify-built-package-invariants.mjs](../../../../scripts/verify-built-package-invariants.mjs) has the same per-package independence — each probe stages its packed view inside its own package and spawns its own `npm pack` and Node processes — so it uses the same bounded-pool shape with `DSH_BUILT_INVARIANTS_CONCURRENCY` as its cap. Serially it dominated the CI artifacts lane (about 4 minutes for 100+ packages, over half the lane's wall clock); the pool collapses that to the slowest probe batch, and failures keep manifest order. + The per-gate package scripts remain the vocabulary for ad hoc local runs. `hygiene` stays an aggregate `&&` chain the scheduler mirrors, while `doc-sync` has since moved its member list into the scheduler itself ([doc-sync through the gate scheduler](2026-07-21-doc-sync-through-gate-scheduler.md)). ## Alternatives considered diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de9a03288b..3aabcb60cf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,7 @@ jobs: env: DSH_GATE_CONCURRENCY: ${{ matrix.gate_concurrency }} DSH_PUBLINT_CONCURRENCY: ${{ matrix.publint_concurrency }} + DSH_BUILT_INVARIANTS_CONCURRENCY: ${{ matrix.built_invariants_concurrency }} DSH_COVERAGE_MAX_WORKERS: ${{ matrix.coverage_max_workers }} DSH_ESLINT_CACHE: ${{ matrix.eslint_cache }} strategy: @@ -32,30 +33,35 @@ jobs: command: pnpm run check:ci:static gate_concurrency: '4' publint_concurrency: '8' + built_invariants_concurrency: '8' coverage_max_workers: '' eslint_cache: '' - lane: lint command: pnpm run check:ci:lint gate_concurrency: '1' publint_concurrency: '8' + built_invariants_concurrency: '8' coverage_max_workers: '' eslint_cache: '1' - lane: coverage command: pnpm run check:ci:coverage gate_concurrency: '1' publint_concurrency: '8' + built_invariants_concurrency: '8' coverage_max_workers: '4' eslint_cache: '' - lane: snapshot command: pnpm run check:ci:snapshot gate_concurrency: '1' publint_concurrency: '8' + built_invariants_concurrency: '8' coverage_max_workers: '' eslint_cache: '' - lane: artifacts command: pnpm run check:ci:artifacts gate_concurrency: '3' publint_concurrency: '8' + built_invariants_concurrency: '8' coverage_max_workers: '' eslint_cache: '' steps: @@ -192,6 +198,7 @@ jobs: env: DSH_GATE_CONCURRENCY: ${{ matrix.gate_concurrency }} DSH_PUBLINT_CONCURRENCY: ${{ matrix.publint_concurrency }} + DSH_BUILT_INVARIANTS_CONCURRENCY: ${{ matrix.built_invariants_concurrency }} DSH_COVERAGE_MAX_WORKERS: ${{ matrix.coverage_max_workers }} DSH_ESLINT_CACHE: ${{ matrix.eslint_cache }} strategy: @@ -202,30 +209,35 @@ jobs: command: pnpm run check:ci:static gate_concurrency: '4' publint_concurrency: '8' + built_invariants_concurrency: '8' coverage_max_workers: '' eslint_cache: '' - lane: lint command: pnpm run check:ci:lint gate_concurrency: '1' publint_concurrency: '8' + built_invariants_concurrency: '8' coverage_max_workers: '' eslint_cache: '1' - lane: coverage command: pnpm run check:ci:coverage gate_concurrency: '1' publint_concurrency: '8' + built_invariants_concurrency: '8' coverage_max_workers: '4' eslint_cache: '' - lane: snapshot command: pnpm run check:ci:snapshot gate_concurrency: '1' publint_concurrency: '8' + built_invariants_concurrency: '8' coverage_max_workers: '' eslint_cache: '' - lane: artifacts command: pnpm run check:ci:artifacts gate_concurrency: '3' publint_concurrency: '8' + built_invariants_concurrency: '8' coverage_max_workers: '' eslint_cache: '' steps: diff --git a/scripts/verify-built-package-invariants.mjs b/scripts/verify-built-package-invariants.mjs index 4b298946d1..7b9aa0c187 100644 --- a/scripts/verify-built-package-invariants.mjs +++ b/scripts/verify-built-package-invariants.mjs @@ -1,6 +1,6 @@ /** Verify every packed companion through its package self-reference under plain Node. */ -import { spawnSync } from 'node:child_process' +import { execFile } from 'node:child_process' import { copyFileSync, globSync, @@ -9,12 +9,16 @@ import { readFileSync, rmSync, } from 'node:fs' +import { availableParallelism } from 'node:os' import { dirname, resolve } from 'node:path' import { pathToFileURL } from 'node:url' +import { promisify } from 'node:util' + +const execFileAsync = promisify(execFile) +const CONCURRENCY_ENV = 'DSH_BUILT_INVARIANTS_CONCURRENCY' const root = resolve(import.meta.dirname, '..') const loaderUrl = pathToFileURL(resolve(root, 'vendor/loader/lib/index.js')).href -const failures = [] const manifests = globSync('packages/*/*/package.json', { cwd: root }).sort() const packArgs = ['pack', '--dry-run', '--json', '--ignore-scripts'] // Windows cannot spawn npm's .cmd shim directly; setup-node installs this JS @@ -23,24 +27,54 @@ const npmInvocation = process.platform === 'win32' ? [process.execPath, [resolve(dirname(process.execPath), 'node_modules/npm/bin/npm-cli.js'), ...packArgs]] : ['npm', packArgs] -for (const manifestPath of manifests) { +function probeConcurrency(total) { + if (total === 0) return 0 + + const raw = process.env[CONCURRENCY_ENV] + if (raw !== undefined && raw !== '') { + const parsed = Number.parseInt(raw, 10) + if (!Number.isSafeInteger(parsed) || parsed < 1) { + throw new Error(`verify-built-package-invariants: ${CONCURRENCY_ENV} must be a positive integer, got ${JSON.stringify(raw)}.`) + } + return Math.min(total, parsed) + } + + return Math.min(total, availableParallelism()) +} + +async function runCommand(command, args, cwd) { + try { + const { stdout, stderr } = await execFileAsync(command, args, { + cwd, + encoding: 'utf8', + maxBuffer: 10 * 1024 * 1024, + }) + return { status: 0, stdout, stderr, message: undefined } + } catch (error) { + const failed = /** @type {{ code?: number; stdout?: unknown; stderr?: unknown; message?: string }} */ (error) + return { + status: typeof failed.code === 'number' ? failed.code : 1, + stdout: typeof failed.stdout === 'string' ? failed.stdout : '', + stderr: typeof failed.stderr === 'string' ? failed.stderr : '', + message: failed.message ?? 'command failed', + } + } +} + +/** Probe one manifest's packed companion; resolves to a failure string or undefined. */ +async function verifyManifest(manifestPath) { const packageDir = dirname(resolve(root, manifestPath)) const manifest = JSON.parse(readFileSync(resolve(root, manifestPath), 'utf8')) const packageName = manifest.name if (typeof packageName !== 'string' || packageName.length === 0) { - failures.push(`${manifestPath}: missing package name`) - continue + return `${manifestPath}: missing package name` } - const pack = spawnSync(npmInvocation[0], npmInvocation[1], { - cwd: packageDir, - encoding: 'utf8', - }) + const pack = await runCommand(npmInvocation[0], npmInvocation[1], packageDir) if (pack.status !== 0) { - const detail = pack.error?.message - ?? (pack.stderr.trim() || pack.stdout.trim() || `npm pack exited ${pack.status}`) - failures.push(`${packageName}: ${detail}`) - continue + const detail = pack.stderr.trim() || pack.stdout.trim() || pack.message + || `npm pack exited ${pack.status}` + return `${packageName}: ${detail}` } let files @@ -49,8 +83,7 @@ for (const manifestPath of manifests) { files = result[0]?.files if (!Array.isArray(files)) throw new Error('npm pack returned no file inventory') } catch (error) { - failures.push(`${packageName}: cannot parse npm pack inventory: ${String(error)}`) - continue + return `${packageName}: cannot parse npm pack inventory: ${String(error)}` } // Keep the packed view below its owning package so Node reaches the real @@ -79,20 +112,36 @@ for (const manifestPath of manifests) { } if (typeof unwrapped.apply !== 'function') throw new Error('companion apply is missing'); ` - const result = spawnSync(process.execPath, ['--input-type=module', '--eval', probe], { - cwd: stagedPackageDir, - encoding: 'utf8', - }) + const result = await runCommand(process.execPath, ['--input-type=module', '--eval', probe], stagedPackageDir) if (result.status !== 0) { - const detail = result.error?.message - ?? (result.stderr.trim() || result.stdout.trim() || `node exited ${result.status}`) - failures.push(`${packageName}: ${detail}`) + const detail = result.stderr.trim() || result.stdout.trim() || result.message + || `node exited ${result.status}` + return `${packageName}: ${detail}` } + return undefined } finally { rmSync(stagedPackageDir, { recursive: true, force: true }) } } +/** Run every manifest probe through a bounded worker pool, keeping failures in manifest order. */ +async function runAll(paths, concurrency) { + let next = 0 + const results = new Array(paths.length) + const workers = Array.from({ length: concurrency }, async () => { + for (;;) { + const index = next + next += 1 + if (index >= paths.length) return + results[index] = await verifyManifest(paths[index]) + } + }) + await Promise.all(workers) + return results.filter(failure => failure !== undefined) +} + +const failures = await runAll(manifests, probeConcurrency(manifests.length)) + if (failures.length > 0) { console.error('verify-built-package-invariants: packed companion failures:') for (const failure of failures) console.error(` ${failure}`) From 0580bc9068b042e6d1557dea298005139a572cae Mon Sep 17 00:00:00 2001 From: Chinesezjc Date: Wed, 22 Jul 2026 15:30:53 +0800 Subject: [PATCH 2/2] fix(ci): reject partially parsed concurrency limits Number.parseInt accepts a numeric prefix, so values like 1.5 or 8junk silently ran an unintended worker count. Require the full string to round-trip (same pattern as run-gates' positiveIntArg). --- scripts/verify-built-package-invariants.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/verify-built-package-invariants.mjs b/scripts/verify-built-package-invariants.mjs index 7b9aa0c187..7412404a97 100644 --- a/scripts/verify-built-package-invariants.mjs +++ b/scripts/verify-built-package-invariants.mjs @@ -33,7 +33,7 @@ function probeConcurrency(total) { const raw = process.env[CONCURRENCY_ENV] if (raw !== undefined && raw !== '') { const parsed = Number.parseInt(raw, 10) - if (!Number.isSafeInteger(parsed) || parsed < 1) { + if (!Number.isSafeInteger(parsed) || parsed < 1 || String(parsed) !== raw) { throw new Error(`verify-built-package-invariants: ${CONCURRENCY_ENV} must be a positive integer, got ${JSON.stringify(raw)}.`) } return Math.min(total, parsed)