mirror of
https://github.com/deepseek-ai/deepseek-harness
synced 2026-08-15 21:04:50 +00:00
feat(gates): print actionable failure output
This commit is contained in:
@@ -24,6 +24,7 @@ type GateStatus = 'pending' | 'running' | 'passed' | 'failed' | 'skipped'
|
||||
interface Gate {
|
||||
id: string
|
||||
label: string
|
||||
displayCommand: string
|
||||
command: string
|
||||
args: string[]
|
||||
needs?: string[]
|
||||
@@ -38,10 +39,16 @@ interface GateResult {
|
||||
durationMs: number
|
||||
stdout: string
|
||||
stderr: string
|
||||
output: GateOutputChunk[]
|
||||
exitCode: number | null
|
||||
error?: string
|
||||
}
|
||||
|
||||
interface GateOutputChunk {
|
||||
stream: 'stdout' | 'stderr'
|
||||
text: string
|
||||
}
|
||||
|
||||
interface RunningGate {
|
||||
gate: Gate
|
||||
promise: Promise<GateResult>
|
||||
@@ -58,6 +65,7 @@ const gates = gatesForMode(mode)
|
||||
const concurrencyDefault = defaultConcurrency(mode, gates.length)
|
||||
const concurrencyOverride = process.env.DSH_GATE_CONCURRENCY
|
||||
const maxConcurrency = concurrencyFromEnv('DSH_GATE_CONCURRENCY', concurrencyDefault.workers)
|
||||
const verbose = process.env.DSH_GATE_VERBOSE === '1'
|
||||
const startedAt = performance.now()
|
||||
|
||||
const concurrencySource = concurrencyOverride === undefined || concurrencyOverride === ''
|
||||
@@ -113,6 +121,7 @@ function pnpmScript(id: string, script: string, options: Partial<Gate> = {}): Ga
|
||||
return {
|
||||
id,
|
||||
label: options.label ?? script,
|
||||
displayCommand: `pnpm run ${script}`,
|
||||
...pnpmInvocation(['run', script]),
|
||||
...options,
|
||||
}
|
||||
@@ -122,6 +131,7 @@ function pnpmExec(id: string, args: string[], options: Partial<Gate> = {}): Gate
|
||||
return {
|
||||
id,
|
||||
label: options.label ?? `pnpm exec ${args.join(' ')}`,
|
||||
displayCommand: `pnpm exec ${args.join(' ')}`,
|
||||
...pnpmInvocation(['exec', ...args]),
|
||||
...options,
|
||||
}
|
||||
@@ -332,6 +342,7 @@ function demoSmokeGate(options: { needs?: string[] } = {}): Gate {
|
||||
return {
|
||||
id: 'demo-smoke',
|
||||
label: 'demo smoke',
|
||||
displayCommand: 'pnpm run demo:echo',
|
||||
...pnpmInvocation(['run', 'demo:echo']),
|
||||
input: 'echo ci smoke\n',
|
||||
...dependencyOptions,
|
||||
@@ -408,6 +419,7 @@ async function runGates(allGates: Gate[], maxActive: number): Promise<GateResult
|
||||
durationMs: 0,
|
||||
stdout: '',
|
||||
stderr: '',
|
||||
output: [],
|
||||
exitCode: null,
|
||||
error: `dependency failed or skipped: ${failedDeps.join(', ')}`,
|
||||
}
|
||||
@@ -442,8 +454,10 @@ async function runGate(gate: Gate): Promise<GateResult> {
|
||||
const started = performance.now()
|
||||
let stdout = ''
|
||||
let stderr = ''
|
||||
const output: GateOutputChunk[] = []
|
||||
let spawnError: string | undefined
|
||||
|
||||
const exitCode = await new Promise<number | null>((resolveExit, reject) => {
|
||||
const exitCode = await new Promise<number | null>((resolveExit) => {
|
||||
const child = spawn(gate.command, gate.args, {
|
||||
cwd: root,
|
||||
env: { ...process.env, ...gate.env },
|
||||
@@ -451,19 +465,28 @@ async function runGate(gate: Gate): Promise<GateResult> {
|
||||
})
|
||||
child.stdout.setEncoding('utf8')
|
||||
child.stderr.setEncoding('utf8')
|
||||
child.stdout.on('data', (chunk: string) => { stdout += chunk })
|
||||
child.stderr.on('data', (chunk: string) => { stderr += chunk })
|
||||
child.on('error', reject)
|
||||
child.stdout.on('data', (chunk: string) => {
|
||||
stdout += chunk
|
||||
output.push({ stream: 'stdout', text: chunk })
|
||||
})
|
||||
child.stderr.on('data', (chunk: string) => {
|
||||
stderr += chunk
|
||||
output.push({ stream: 'stderr', text: chunk })
|
||||
})
|
||||
child.on('error', (error) => {
|
||||
spawnError = `failed to start command: ${error.message}`
|
||||
resolveExit(null)
|
||||
})
|
||||
child.on('close', resolveExit)
|
||||
if (gate.input !== undefined) child.stdin.end(gate.input)
|
||||
else child.stdin.end()
|
||||
})
|
||||
|
||||
let status: GateStatus = exitCode === 0 ? 'passed' : 'failed'
|
||||
let error: string | undefined
|
||||
let status: GateStatus = exitCode === 0 && spawnError === undefined ? 'passed' : 'failed'
|
||||
let error = spawnError
|
||||
if (status === 'passed' && gate.verify !== undefined) {
|
||||
try {
|
||||
await gate.verify({ gate, status, durationMs: performance.now() - started, stdout, stderr, exitCode })
|
||||
await gate.verify({ gate, status, durationMs: performance.now() - started, stdout, stderr, output, exitCode })
|
||||
} catch (verifyError: unknown) {
|
||||
status = 'failed'
|
||||
error = verifyError instanceof Error ? verifyError.message : String(verifyError)
|
||||
@@ -476,6 +499,7 @@ async function runGate(gate: Gate): Promise<GateResult> {
|
||||
durationMs: performance.now() - started,
|
||||
stdout,
|
||||
stderr,
|
||||
output,
|
||||
exitCode,
|
||||
}
|
||||
if (error !== undefined) result.error = error
|
||||
@@ -484,9 +508,16 @@ async function runGate(gate: Gate): Promise<GateResult> {
|
||||
|
||||
function printResult(result: GateResult): void {
|
||||
const seconds = (result.durationMs / 1000).toFixed(2)
|
||||
console.log(`\n== ${result.status.toUpperCase()} ${result.gate.label} (${seconds}s) ==`)
|
||||
process.stdout.write(result.stdout)
|
||||
process.stderr.write(result.stderr)
|
||||
if (result.status === 'passed' && !verbose) {
|
||||
console.log(`run-gates: PASS ${result.gate.label} (${seconds}s)`)
|
||||
return
|
||||
}
|
||||
|
||||
const heading = `${result.status.toUpperCase()} ${result.gate.label} (${seconds}s)`
|
||||
const writeHeading = result.status === 'passed' ? console.log : console.error
|
||||
writeHeading(`\n== ${heading} ==`)
|
||||
if (result.status !== 'passed') console.error(`command: ${result.gate.displayCommand}`)
|
||||
printOutput(result.output)
|
||||
if (result.error !== undefined) console.error(result.error)
|
||||
}
|
||||
|
||||
@@ -496,4 +527,22 @@ function printSummary(results: GateResult[], durationMs: number): void {
|
||||
const skipped = results.filter(result => result.status === 'skipped').length
|
||||
const seconds = (durationMs / 1000).toFixed(2)
|
||||
console.log(`\nrun-gates: ${passed} passed, ${failed} failed, ${skipped} skipped in ${seconds}s.`)
|
||||
|
||||
const unsuccessful = results.filter(result => result.status === 'failed' || result.status === 'skipped')
|
||||
if (unsuccessful.length === 0) return
|
||||
|
||||
console.error('run-gates: unsuccessful gates:')
|
||||
for (const result of unsuccessful) {
|
||||
const duration = (result.durationMs / 1000).toFixed(2)
|
||||
const reason = result.error ?? (result.exitCode === null ? 'no exit code' : `exit ${result.exitCode}`)
|
||||
console.error(` - ${result.status.toUpperCase()} ${result.gate.label} (${duration}s, ${reason})`)
|
||||
console.error(` ${result.gate.displayCommand}`)
|
||||
}
|
||||
}
|
||||
|
||||
function printOutput(output: GateOutputChunk[]): void {
|
||||
for (const chunk of output) {
|
||||
if (chunk.stream === 'stdout') process.stdout.write(chunk.text)
|
||||
else process.stderr.write(chunk.text)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user