perf(ci): parallelize packed-companion probes

verify-built-package-invariants ran 100+ npm-pack + plain-Node probes
serially, dominating the CI artifacts lane (~4min of a ~5min job; ~7.5min
on Windows). Each probe stages its packed view inside its own package and
spawns its own processes, so the probes are independent — run them through
the same bounded worker pool shape as publint-all, capped by
DSH_BUILT_INVARIANTS_CONCURRENCY (default availableParallelism), failures
kept in manifest order.

Measured on the gate alone: 2m07s serial -> 17s at concurrency 8.
CI lanes pin the cap to 8, matching DSH_PUBLINT_CONCURRENCY.
This commit is contained in:
Chinesezjc
2026-07-22 13:21:18 +08:00
parent 86f115a07c
commit e23c201da8
3 changed files with 85 additions and 22 deletions

View File

@@ -1,6 +1,6 @@
/** Verify every packed companion through its package self-reference under plain Node. */
import { spawnSync } from 'node:child_process'
import { execFile } from 'node:child_process'
import {
copyFileSync,
globSync,
@@ -9,12 +9,16 @@ import {
readFileSync,
rmSync,
} from 'node:fs'
import { availableParallelism } from 'node:os'
import { dirname, resolve } from 'node:path'
import { pathToFileURL } from 'node:url'
import { promisify } from 'node:util'
const execFileAsync = promisify(execFile)
const CONCURRENCY_ENV = 'DSH_BUILT_INVARIANTS_CONCURRENCY'
const root = resolve(import.meta.dirname, '..')
const loaderUrl = pathToFileURL(resolve(root, 'vendor/loader/lib/index.js')).href
const failures = []
const manifests = globSync('packages/*/*/package.json', { cwd: root }).sort()
const packArgs = ['pack', '--dry-run', '--json', '--ignore-scripts']
// Windows cannot spawn npm's .cmd shim directly; setup-node installs this JS
@@ -23,24 +27,54 @@ const npmInvocation = process.platform === 'win32'
? [process.execPath, [resolve(dirname(process.execPath), 'node_modules/npm/bin/npm-cli.js'), ...packArgs]]
: ['npm', packArgs]
for (const manifestPath of manifests) {
function probeConcurrency(total) {
if (total === 0) return 0
const raw = process.env[CONCURRENCY_ENV]
if (raw !== undefined && raw !== '') {
const parsed = Number.parseInt(raw, 10)
if (!Number.isSafeInteger(parsed) || parsed < 1) {
throw new Error(`verify-built-package-invariants: ${CONCURRENCY_ENV} must be a positive integer, got ${JSON.stringify(raw)}.`)
}
return Math.min(total, parsed)
}
return Math.min(total, availableParallelism())
}
async function runCommand(command, args, cwd) {
try {
const { stdout, stderr } = await execFileAsync(command, args, {
cwd,
encoding: 'utf8',
maxBuffer: 10 * 1024 * 1024,
})
return { status: 0, stdout, stderr, message: undefined }
} catch (error) {
const failed = /** @type {{ code?: number; stdout?: unknown; stderr?: unknown; message?: string }} */ (error)
return {
status: typeof failed.code === 'number' ? failed.code : 1,
stdout: typeof failed.stdout === 'string' ? failed.stdout : '',
stderr: typeof failed.stderr === 'string' ? failed.stderr : '',
message: failed.message ?? 'command failed',
}
}
}
/** Probe one manifest's packed companion; resolves to a failure string or undefined. */
async function verifyManifest(manifestPath) {
const packageDir = dirname(resolve(root, manifestPath))
const manifest = JSON.parse(readFileSync(resolve(root, manifestPath), 'utf8'))
const packageName = manifest.name
if (typeof packageName !== 'string' || packageName.length === 0) {
failures.push(`${manifestPath}: missing package name`)
continue
return `${manifestPath}: missing package name`
}
const pack = spawnSync(npmInvocation[0], npmInvocation[1], {
cwd: packageDir,
encoding: 'utf8',
})
const pack = await runCommand(npmInvocation[0], npmInvocation[1], packageDir)
if (pack.status !== 0) {
const detail = pack.error?.message
?? (pack.stderr.trim() || pack.stdout.trim() || `npm pack exited ${pack.status}`)
failures.push(`${packageName}: ${detail}`)
continue
const detail = pack.stderr.trim() || pack.stdout.trim() || pack.message
|| `npm pack exited ${pack.status}`
return `${packageName}: ${detail}`
}
let files
@@ -49,8 +83,7 @@ for (const manifestPath of manifests) {
files = result[0]?.files
if (!Array.isArray(files)) throw new Error('npm pack returned no file inventory')
} catch (error) {
failures.push(`${packageName}: cannot parse npm pack inventory: ${String(error)}`)
continue
return `${packageName}: cannot parse npm pack inventory: ${String(error)}`
}
// Keep the packed view below its owning package so Node reaches the real
@@ -79,20 +112,36 @@ for (const manifestPath of manifests) {
}
if (typeof unwrapped.apply !== 'function') throw new Error('companion apply is missing');
`
const result = spawnSync(process.execPath, ['--input-type=module', '--eval', probe], {
cwd: stagedPackageDir,
encoding: 'utf8',
})
const result = await runCommand(process.execPath, ['--input-type=module', '--eval', probe], stagedPackageDir)
if (result.status !== 0) {
const detail = result.error?.message
?? (result.stderr.trim() || result.stdout.trim() || `node exited ${result.status}`)
failures.push(`${packageName}: ${detail}`)
const detail = result.stderr.trim() || result.stdout.trim() || result.message
|| `node exited ${result.status}`
return `${packageName}: ${detail}`
}
return undefined
} finally {
rmSync(stagedPackageDir, { recursive: true, force: true })
}
}
/** Run every manifest probe through a bounded worker pool, keeping failures in manifest order. */
async function runAll(paths, concurrency) {
let next = 0
const results = new Array(paths.length)
const workers = Array.from({ length: concurrency }, async () => {
for (;;) {
const index = next
next += 1
if (index >= paths.length) return
results[index] = await verifyManifest(paths[index])
}
})
await Promise.all(workers)
return results.filter(failure => failure !== undefined)
}
const failures = await runAll(manifests, probeConcurrency(manifests.length))
if (failures.length > 0) {
console.error('verify-built-package-invariants: packed companion failures:')
for (const failure of failures) console.error(` ${failure}`)