chore: parallelize pre-push gates

This commit is contained in:
Tianyi Cui
2026-07-06 00:51:50 +08:00
parent d10d94a640
commit 72c83c771a
5 changed files with 153 additions and 19 deletions

View File

@@ -2,5 +2,5 @@
# side as of the last confirmed-consistent state. Both languages carry equal authority;
# after editing either side, bring the other along and re-record with:
# pnpm run verify-translation-pairing --write
development.md: f032764fff29baaca007211db8b69d9a5129078f
development.zh.md: 3a650d03ce7cafd0e34290ae918e5a303c2ad8a9
development.md: 2c39e0c531638eb0fa3561ec32e6d5c00d87a8be
development.zh.md: 6c9c2a5480db3b71a2d9e25e89c6afd34d24fdb0

View File

@@ -59,7 +59,7 @@ DEEPSEEK_BASE_URL=https://... # optional
lefthook is configured in `lefthook.yml` as an early local checkpoint before review:
- `pre-commit` runs staged-file ESLint fixes, `pnpm run typecheck`, and the vendor manifest guard.
- `pre-push` runs `pnpm run test`, `pnpm run test:snapshot`, `pnpm run hygiene`, `pnpm run doc-sync`, and `pnpm run verify-module-graph`.
- `pre-push` runs unit tests, snapshot tests, module-graph freshness, and the member gates of `pnpm run hygiene` and `pnpm run doc-sync` as parallel lefthook jobs.
The vendor manifest guard checks that changes under `vendor/*/src` are staged with the matching `vendor/README.md` manifest update. See `vendor/README.md` before editing vendored code.

View File

@@ -59,7 +59,7 @@ DEEPSEEK_BASE_URL=https://... # optional
lefthook 在 `lefthook.yml` 中配置,作为评审前的本地早期检查点:
- `pre-commit` 运行对暂存文件的 ESLint 修复、`pnpm run typecheck` 和 vendor manifest 守卫。
- `pre-push` 运行 `pnpm run test``pnpm run test:snapshot``pnpm run hygiene``pnpm run doc-sync` `pnpm run verify-module-graph`
- `pre-push` 将单元测试、快照测试、module graph 新鲜度,以及 `pnpm run hygiene``pnpm run doc-sync` 的成员门禁拆成并行 lefthook job 运行
vendor manifest 守卫检查 `vendor/*/src` 下的改动是否连同对应的 `vendor/README.md` manifest 更新一起暂存。编辑 vendor 代码前先看 `vendor/README.md`

View File

@@ -28,11 +28,61 @@ pre-push:
- name: snapshot
run: pnpm run test:snapshot
- name: hygiene
run: pnpm run hygiene
# Flatten `pnpm run hygiene` and `pnpm run doc-sync` so lefthook can run
# their independent member gates concurrently before a push.
- name: knip
run: pnpm run knip
- name: doc-sync
run: pnpm run doc-sync
- name: publint
run: pnpm run publint
- name: constraints
run: pnpm run constraints
- name: node-next types
run: pnpm run verify-node-next-types
- name: doc typecheck
run: pnpm run doc-typecheck
- name: cordis catalog
run: pnpm run verify-cordis-catalog
- name: tool catalog
run: pnpm run verify-tool-catalog
- name: persistence catalog
run: pnpm run verify-persistence-catalog
- name: doc graphs
run: pnpm run verify-doc-graphs
- name: markdown wrap
run: pnpm run verify-md-wrap
- name: markdown links
run: pnpm run verify-md-links
- name: doc refs
run: pnpm run verify-doc-refs
- name: package paths
run: pnpm run verify-package-paths
- name: mermaid
run: pnpm run verify-mermaid
- name: rfc classification
run: pnpm run verify-rfc-classification
- name: type equivalence
run: pnpm run verify-type-equiv
- name: translation pairing
run: pnpm run verify-translation-pairing
- name: doc budgets
run: pnpm run verify-doc-budgets
- name: module-graph freshness
run: pnpm run verify-module-graph

View File

@@ -1,6 +1,11 @@
import { execFileSync } from 'node:child_process'
import { execFile } from 'node:child_process'
import { existsSync, readdirSync } from 'node:fs'
import { availableParallelism } from 'node:os'
import { resolve } from 'node:path'
import { promisify } from 'node:util'
const execFileAsync = promisify(execFile)
const CONCURRENCY_ENV = 'DSH_PUBLINT_CONCURRENCY'
// publint every harness package. Packages live at packages/<group>/<pkg>
// (the group dirs — core/llm/bash/… — are pure containers); vendor/ is private
@@ -9,15 +14,94 @@ import { resolve } from 'node:path'
const root = resolve(import.meta.dirname, '..')
const packagesRoot = resolve(root, 'packages')
const packages = readdirSync(packagesRoot, { withFileTypes: true })
.filter(group => group.isDirectory())
.flatMap(group =>
readdirSync(resolve(packagesRoot, group.name), { withFileTypes: true })
.filter(pkg => pkg.isDirectory())
.filter(pkg => existsSync(resolve(packagesRoot, group.name, pkg.name, 'package.json')))
.map(pkg => `packages/${group.name}/${pkg.name}`),
)
type PublintResult =
| { path: string; status: 'passed'; stdout: string; stderr: string }
| { path: string; status: 'failed'; stdout: string; stderr: string; message: string }
for (const path of packages) {
execFileSync('node_modules/.bin/publint', [path], { cwd: root, stdio: 'inherit' })
function workspacePackages(): string[] {
return readdirSync(packagesRoot, { withFileTypes: true })
.filter(group => group.isDirectory())
.flatMap(group =>
readdirSync(resolve(packagesRoot, group.name), { withFileTypes: true })
.filter(pkg => pkg.isDirectory())
.filter(pkg => existsSync(resolve(packagesRoot, group.name, pkg.name, 'package.json')))
.map(pkg => `packages/${group.name}/${pkg.name}`),
)
}
function publintConcurrency(total: number): number {
if (total === 0) return 0
const raw = process.env[CONCURRENCY_ENV]
if (raw !== undefined) {
const parsed = Number.parseInt(raw, 10)
if (!Number.isSafeInteger(parsed) || parsed < 1) {
throw new Error(`publint-all: ${CONCURRENCY_ENV} must be a positive integer, got ${JSON.stringify(raw)}.`)
}
return Math.min(total, parsed)
}
return Math.min(total, availableParallelism())
}
function outputText(value: unknown): string {
if (typeof value === 'string') return value
if (Buffer.isBuffer(value)) return value.toString()
return ''
}
async function runPublint(path: string): Promise<PublintResult> {
try {
const { stdout, stderr } = await execFileAsync('node_modules/.bin/publint', [path], {
cwd: root,
encoding: 'utf8',
maxBuffer: 10 * 1024 * 1024,
})
return { path, status: 'passed', stdout, stderr }
} catch (error: unknown) {
const failed = error as { stdout?: unknown; stderr?: unknown; message?: string }
return {
path,
status: 'failed',
stdout: outputText(failed.stdout),
stderr: outputText(failed.stderr),
message: failed.message ?? 'publint failed',
}
}
}
async function runAll(paths: string[], concurrency: number): Promise<PublintResult[]> {
let next = 0
const results: Array<PublintResult | undefined> = []
await Promise.all(Array.from({ length: concurrency }, async () => {
for (;;) {
const index = next
next += 1
const path = paths[index]
if (path === undefined) return
results[index] = await runPublint(path)
}
}))
return paths.map((path, index) => {
const result = results[index]
if (result === undefined) throw new Error(`publint-all: missing result for ${path}.`)
return result
})
}
function printResult(result: PublintResult): void {
console.log(`Running publint for ${result.path}...`)
process.stdout.write(result.stdout)
process.stderr.write(result.stderr)
if (result.status === 'failed') console.error(result.message)
}
const packages = workspacePackages()
const concurrency = publintConcurrency(packages.length)
console.log(`publint-all: linting ${packages.length} package(s) with ${concurrency} worker(s).`)
const results = await runAll(packages, concurrency)
for (const result of results) printResult(result)
if (results.some(result => result.status === 'failed')) process.exit(1)