mirror of
https://github.com/deepseek-ai/deepseek-harness
synced 2026-08-15 21:04:50 +00:00
ec601ca13d
Machine-produced by `pnpm run rescope-vendor --apply` plus the regeneration it prints: `pnpm install` for the lockfile, `pnpm run gen-third-party-notices`, `verify-translation-pairing --write` for the touched bilingual pairs, `gen-doc-graphs`, and one typert snapshot whose ids embed character offsets. `pnpm run rescope-vendor --check` verifies the result. Renames nine vendored packages (cordis, cosmokit, schemastery and the six @cordisjs plugins) and every reference that resolves them: manifest names and dependency keys, module specifiers including declare-module merges, cordis.yml plugin names, tsconfig paths, every Markdown fence, and `docs/` prose. Directory names, upstream versions, and dependency ranges are unchanged, so vendor/README.md still reads as an upstream snapshot; its manifest table gains an upstream-name column so THIRD_PARTY_NOTICES keeps MIT attribution pointed at each fork's origin. The tutorial tier follows the rename end to end: its yaml fences named plugins the Loader can no longer resolve, its `ts ignore-check` fences disagreed with the compiled fences beside them, and its prose quoted both. The contracts that told readers to keep upstream names — the root convention and the vendoring cookbook's tree comment and manifest invariant — now say to rescope instead. Two rules read `@deepseek-ai/` as "another workspace plugin": the client bundle purity gate now names the vendored libraries a browser bundle inlines, and the files where a bare `cordis` is an agent-preset id keep that product data.
87 lines
3.3 KiB
TypeScript
87 lines
3.3 KiB
TypeScript
import { createUserMessage } from '@deepseek-ai/dsh-llm'
|
|
import { spawnSync } from 'node:child_process'
|
|
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
|
|
import { tmpdir } from 'node:os'
|
|
import { join } from 'node:path'
|
|
import { afterEach, describe, expect, it } from 'vitest'
|
|
import type { Context } from '@deepseek-ai/cordis'
|
|
import { codingHarness, finalText, SYSTEM_PROMPT, waitForIdle } from './harness.ts'
|
|
import { SessionId } from '@deepseek-ai/dsh-session'
|
|
|
|
/**
|
|
* The swebench-style smoke test: a real model fixes a real bug in a temp
|
|
* directory using only the bash tool, and the fix is verified OUTSIDE the
|
|
* agent by re-running the test script. Key-gated.
|
|
*/
|
|
|
|
const TEST_FILE = [
|
|
"const assert = require('node:assert');",
|
|
"const { add } = require('./add.js');",
|
|
'assert.strictEqual(add(2, 3), 5);',
|
|
'assert.strictEqual(add(-1, 1), 0);',
|
|
"console.log('PASS');",
|
|
'',
|
|
].join('\n')
|
|
|
|
const BUGGY_ADD = [
|
|
'// A tiny module with an obvious bug.',
|
|
'function add(a, b) {',
|
|
' return a - b;',
|
|
'}',
|
|
'module.exports = { add };',
|
|
'',
|
|
].join('\n')
|
|
|
|
let workdir: string | undefined
|
|
let ctx: Context | undefined
|
|
|
|
afterEach(async () => {
|
|
// Dispose the harness even on failure/retry: agent-loop teardown stops the
|
|
// loop and LocalBashExecutor teardown kills anything the model left running.
|
|
await ctx?.fiber.dispose()
|
|
ctx = undefined
|
|
if (workdir !== undefined) await rm(workdir, { recursive: true, force: true })
|
|
workdir = undefined
|
|
})
|
|
|
|
describe.skipIf(!process.env.DEEPSEEK_API_KEY)('coding task: fix a failing test via bash', () => {
|
|
it('repairs add.js so node add.test.js passes', async () => {
|
|
workdir = await mkdtemp(join(tmpdir(), 'dsh-coding-task-'))
|
|
await writeFile(join(workdir, 'add.js'), BUGGY_ADD)
|
|
await writeFile(join(workdir, 'add.test.js'), TEST_FILE)
|
|
|
|
// Confirm the fixture actually fails before the agent touches it.
|
|
const before = spawnSync('node', ['add.test.js'], { cwd: workdir })
|
|
expect(before.status).not.toBe(0)
|
|
|
|
ctx = await codingHarness(workdir, { persona: SYSTEM_PROMPT })
|
|
const agent = ctx.agentLoop.create(SessionId('e2e-task'), { provider: 'deepseek-official', model: 'deepseek-v4-flash' })
|
|
|
|
agent.followup(createUserMessage({
|
|
content: [{
|
|
type: 'text',
|
|
text: 'In the current directory, `node add.test.js` fails because add.js has a bug. '
|
|
+ 'Fix add.js so the test passes, run `node add.test.js` to verify, and report the result. '
|
|
+ 'Do not modify add.test.js.',
|
|
}], source: { kind: 'user' } }))
|
|
await waitForIdle(ctx, agent)
|
|
|
|
// The agent claims success…
|
|
const summary = finalText([...agent.session.events]).toLowerCase()
|
|
expect(summary.length).toBeGreaterThan(0)
|
|
|
|
// …and the world agrees: the test passes when WE run it, and the test
|
|
// file is byte-identical (an agent that neutered the test instead of
|
|
// fixing the bug fails here, not just on a keyword probe).
|
|
const untouchedTest = await readFile(join(workdir, 'add.test.js'), 'utf8')
|
|
expect(untouchedTest).toBe(TEST_FILE)
|
|
|
|
const after = spawnSync('node', ['add.test.js'], { cwd: workdir, encoding: 'utf8' })
|
|
expect(after.stdout).toContain('PASS')
|
|
expect(after.status).toBe(0)
|
|
|
|
const fixed = await readFile(join(workdir, 'add.js'), 'utf8')
|
|
expect(fixed).not.toMatch(/a\s*-\s*b/)
|
|
}, 180_000)
|
|
})
|