mirror of
https://github.com/deepseek-ai/deepseek-harness
synced 2026-08-15 21:04:50 +00:00
perf(ci): parallelize packed-companion probes
verify-built-package-invariants ran 100+ npm-pack + plain-Node probes serially, dominating the CI artifacts lane (~4min of a ~5min job; ~7.5min on Windows). Each probe stages its packed view inside its own package and spawns its own processes, so the probes are independent — run them through the same bounded worker pool shape as publint-all, capped by DSH_BUILT_INVARIANTS_CONCURRENCY (default availableParallelism), failures kept in manifest order. Measured on the gate alone: 2m07s serial -> 17s at concurrency 8. CI lanes pin the cap to 8, matching DSH_PUBLINT_CONCURRENCY.
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
/** Verify every packed companion through its package self-reference under plain Node. */
|
||||
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { execFile } from 'node:child_process'
|
||||
import {
|
||||
copyFileSync,
|
||||
globSync,
|
||||
@@ -9,12 +9,16 @@ import {
|
||||
readFileSync,
|
||||
rmSync,
|
||||
} from 'node:fs'
|
||||
import { availableParallelism } from 'node:os'
|
||||
import { dirname, resolve } from 'node:path'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { promisify } from 'node:util'
|
||||
|
||||
const execFileAsync = promisify(execFile)
|
||||
const CONCURRENCY_ENV = 'DSH_BUILT_INVARIANTS_CONCURRENCY'
|
||||
|
||||
const root = resolve(import.meta.dirname, '..')
|
||||
const loaderUrl = pathToFileURL(resolve(root, 'vendor/loader/lib/index.js')).href
|
||||
const failures = []
|
||||
const manifests = globSync('packages/*/*/package.json', { cwd: root }).sort()
|
||||
const packArgs = ['pack', '--dry-run', '--json', '--ignore-scripts']
|
||||
// Windows cannot spawn npm's .cmd shim directly; setup-node installs this JS
|
||||
@@ -23,24 +27,54 @@ const npmInvocation = process.platform === 'win32'
|
||||
? [process.execPath, [resolve(dirname(process.execPath), 'node_modules/npm/bin/npm-cli.js'), ...packArgs]]
|
||||
: ['npm', packArgs]
|
||||
|
||||
for (const manifestPath of manifests) {
|
||||
function probeConcurrency(total) {
|
||||
if (total === 0) return 0
|
||||
|
||||
const raw = process.env[CONCURRENCY_ENV]
|
||||
if (raw !== undefined && raw !== '') {
|
||||
const parsed = Number.parseInt(raw, 10)
|
||||
if (!Number.isSafeInteger(parsed) || parsed < 1) {
|
||||
throw new Error(`verify-built-package-invariants: ${CONCURRENCY_ENV} must be a positive integer, got ${JSON.stringify(raw)}.`)
|
||||
}
|
||||
return Math.min(total, parsed)
|
||||
}
|
||||
|
||||
return Math.min(total, availableParallelism())
|
||||
}
|
||||
|
||||
async function runCommand(command, args, cwd) {
|
||||
try {
|
||||
const { stdout, stderr } = await execFileAsync(command, args, {
|
||||
cwd,
|
||||
encoding: 'utf8',
|
||||
maxBuffer: 10 * 1024 * 1024,
|
||||
})
|
||||
return { status: 0, stdout, stderr, message: undefined }
|
||||
} catch (error) {
|
||||
const failed = /** @type {{ code?: number; stdout?: unknown; stderr?: unknown; message?: string }} */ (error)
|
||||
return {
|
||||
status: typeof failed.code === 'number' ? failed.code : 1,
|
||||
stdout: typeof failed.stdout === 'string' ? failed.stdout : '',
|
||||
stderr: typeof failed.stderr === 'string' ? failed.stderr : '',
|
||||
message: failed.message ?? 'command failed',
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Probe one manifest's packed companion; resolves to a failure string or undefined. */
|
||||
async function verifyManifest(manifestPath) {
|
||||
const packageDir = dirname(resolve(root, manifestPath))
|
||||
const manifest = JSON.parse(readFileSync(resolve(root, manifestPath), 'utf8'))
|
||||
const packageName = manifest.name
|
||||
if (typeof packageName !== 'string' || packageName.length === 0) {
|
||||
failures.push(`${manifestPath}: missing package name`)
|
||||
continue
|
||||
return `${manifestPath}: missing package name`
|
||||
}
|
||||
|
||||
const pack = spawnSync(npmInvocation[0], npmInvocation[1], {
|
||||
cwd: packageDir,
|
||||
encoding: 'utf8',
|
||||
})
|
||||
const pack = await runCommand(npmInvocation[0], npmInvocation[1], packageDir)
|
||||
if (pack.status !== 0) {
|
||||
const detail = pack.error?.message
|
||||
?? (pack.stderr.trim() || pack.stdout.trim() || `npm pack exited ${pack.status}`)
|
||||
failures.push(`${packageName}: ${detail}`)
|
||||
continue
|
||||
const detail = pack.stderr.trim() || pack.stdout.trim() || pack.message
|
||||
|| `npm pack exited ${pack.status}`
|
||||
return `${packageName}: ${detail}`
|
||||
}
|
||||
|
||||
let files
|
||||
@@ -49,8 +83,7 @@ for (const manifestPath of manifests) {
|
||||
files = result[0]?.files
|
||||
if (!Array.isArray(files)) throw new Error('npm pack returned no file inventory')
|
||||
} catch (error) {
|
||||
failures.push(`${packageName}: cannot parse npm pack inventory: ${String(error)}`)
|
||||
continue
|
||||
return `${packageName}: cannot parse npm pack inventory: ${String(error)}`
|
||||
}
|
||||
|
||||
// Keep the packed view below its owning package so Node reaches the real
|
||||
@@ -79,20 +112,36 @@ for (const manifestPath of manifests) {
|
||||
}
|
||||
if (typeof unwrapped.apply !== 'function') throw new Error('companion apply is missing');
|
||||
`
|
||||
const result = spawnSync(process.execPath, ['--input-type=module', '--eval', probe], {
|
||||
cwd: stagedPackageDir,
|
||||
encoding: 'utf8',
|
||||
})
|
||||
const result = await runCommand(process.execPath, ['--input-type=module', '--eval', probe], stagedPackageDir)
|
||||
if (result.status !== 0) {
|
||||
const detail = result.error?.message
|
||||
?? (result.stderr.trim() || result.stdout.trim() || `node exited ${result.status}`)
|
||||
failures.push(`${packageName}: ${detail}`)
|
||||
const detail = result.stderr.trim() || result.stdout.trim() || result.message
|
||||
|| `node exited ${result.status}`
|
||||
return `${packageName}: ${detail}`
|
||||
}
|
||||
return undefined
|
||||
} finally {
|
||||
rmSync(stagedPackageDir, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
/** Run every manifest probe through a bounded worker pool, keeping failures in manifest order. */
|
||||
async function runAll(paths, concurrency) {
|
||||
let next = 0
|
||||
const results = new Array(paths.length)
|
||||
const workers = Array.from({ length: concurrency }, async () => {
|
||||
for (;;) {
|
||||
const index = next
|
||||
next += 1
|
||||
if (index >= paths.length) return
|
||||
results[index] = await verifyManifest(paths[index])
|
||||
}
|
||||
})
|
||||
await Promise.all(workers)
|
||||
return results.filter(failure => failure !== undefined)
|
||||
}
|
||||
|
||||
const failures = await runAll(manifests, probeConcurrency(manifests.length))
|
||||
|
||||
if (failures.length > 0) {
|
||||
console.error('verify-built-package-invariants: packed companion failures:')
|
||||
for (const failure of failures) console.error(` ${failure}`)
|
||||
|
||||
Reference in New Issue
Block a user