mirror of
https://github.com/deepseek-ai/deepseek-harness
synced 2026-08-15 21:04:50 +00:00
Merge remote-tracking branch 'origin/master' into feat/tui-package
# Conflicts: # docs/architecture.md # docs/config-catalog.md # docs/cordis-catalog/events.md # docs/event-producer-consumer.md # docs/module-graph.md # examples/coding-agent/cordis.yml # examples/echo-agent/cordis.yml # knip.json # packages/core/agent-loop/README.md # packages/core/agent-loop/tests/config-session-id.spec.ts # packages/examples/README.md # packages/examples/stdio-demo/README.md # packages/examples/stdio-demo/tests/stdio-agent.spec.ts
This commit is contained in:
@@ -1,10 +1,10 @@
|
||||
import { mkdtemp, readFile, rm } from 'node:fs/promises'
|
||||
import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { Context } from 'cordis'
|
||||
import LlmService from '@deepseek-ai/dsh-llm'
|
||||
import SessionStore from '@deepseek-ai/dsh-session'
|
||||
import SessionStore, { SessionId } from '@deepseek-ai/dsh-session'
|
||||
import type { SessionEvent } from '@deepseek-ai/dsh-session'
|
||||
import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
|
||||
import ToolRegistry, { RUN_CODE_NAME } from '@deepseek-ai/dsh-tools'
|
||||
@@ -14,6 +14,9 @@ import { LocalBashExecutor } from '@deepseek-ai/dsh-bash-local'
|
||||
import * as ToolBash from '@deepseek-ai/dsh-tool-bash'
|
||||
import * as LlmDeepSeek from '@deepseek-ai/dsh-llm-deepseek'
|
||||
import { WorkerCodeRuntime } from '@deepseek-ai/dsh-code-runtime-worker'
|
||||
import LocalFileSystem from '@deepseek-ai/dsh-fs-local'
|
||||
import * as ToolFs from '@deepseek-ai/dsh-tool-fs'
|
||||
import * as WorkspaceContext from '@deepseek-ai/dsh-workspace-context'
|
||||
|
||||
/**
|
||||
* With-key Code Mode proof: a real model receives only `run_code`, composes two
|
||||
@@ -23,6 +26,7 @@ import { WorkerCodeRuntime } from '@deepseek-ai/dsh-code-runtime-worker'
|
||||
|
||||
const PERSONA = 'You are coding-agent. You work by writing TypeScript programs for run_code: '
|
||||
+ 'batch related tool work into one program and print or return ONLY the findings that matter.'
|
||||
const WORKSPACE_PROBE = 'dragonfruit-8675309'
|
||||
|
||||
let ctx: Context | undefined
|
||||
let workdir: string | undefined
|
||||
@@ -45,13 +49,29 @@ async function codeModeHarness(cwd: string): Promise<Context> {
|
||||
await harness.plugin(ToolRegistry, { mode: 'code' })
|
||||
await harness.plugin(AgentRegistry)
|
||||
await harness.plugin(AgentLoop, { agents: [] })
|
||||
await harness.plugin(LlmDeepSeek, { models: ['deepseek-v4-flash'] })
|
||||
await harness.plugin(LlmDeepSeek)
|
||||
await harness.plugin(LocalBashExecutor, { cwd, timeoutMs: 30_000 })
|
||||
await harness.plugin(ToolBash)
|
||||
await harness.plugin(WorkerCodeRuntime, {})
|
||||
return harness
|
||||
}
|
||||
|
||||
async function workspaceCodeModeHarness(): Promise<Context> {
|
||||
const harness = new Context()
|
||||
await harness.plugin(LlmService)
|
||||
await harness.plugin(SessionStore)
|
||||
await harness.plugin(SystemPrompt, { persona: PERSONA })
|
||||
await harness.plugin(ToolRegistry, { mode: 'code' })
|
||||
await harness.plugin(AgentRegistry)
|
||||
await harness.plugin(LocalFileSystem, { cwd: '/' })
|
||||
await harness.plugin(ToolFs)
|
||||
await harness.plugin(WorkspaceContext, { maxBytes: 65536 })
|
||||
await harness.plugin(AgentLoop, { agents: [] })
|
||||
await harness.plugin(LlmDeepSeek, { models: [{ id: 'deepseek-v4-flash' }] })
|
||||
await harness.plugin(WorkerCodeRuntime, {})
|
||||
return harness
|
||||
}
|
||||
|
||||
function waitForIdle(harness: Context, agent: ReactLoopAgent): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
const dispose = harness.on('agent/status', (subject, status) => {
|
||||
@@ -67,7 +87,7 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY)('Code Mode: real model writes a p
|
||||
it('collapses the wire tool list to [run_code], bridges sub-calls, and returns curated output', async () => {
|
||||
workdir = await mkdtemp(join(tmpdir(), 'dsh-code-mode-e2e-'))
|
||||
ctx = await codeModeHarness(workdir)
|
||||
const agent = ctx.agentLoop.create(AgentId('e2e-code-mode'), { model: 'deepseek-v4-flash' })
|
||||
const agent = ctx.agentLoop.create(AgentId('e2e-code-mode'), { provider: 'deepseek', model: 'deepseek-v4-flash' })
|
||||
|
||||
agent.send([{
|
||||
type: 'text',
|
||||
@@ -107,4 +127,43 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY)('Code Mode: real model writes a p
|
||||
expect(finalText).toContain('alpha-7')
|
||||
expect(finalText).toContain('beta-9')
|
||||
}, 180_000)
|
||||
|
||||
it('delivers nested workspace instructions discovered by an fs sub-call after the outer result', async () => {
|
||||
workdir = await mkdtemp(join(tmpdir(), 'dsh-code-mode-workspace-e2e-'))
|
||||
await mkdir(join(workdir, '.git'), { recursive: true })
|
||||
await mkdir(join(workdir, 'pkg/deep'), { recursive: true })
|
||||
await writeFile(join(workdir, 'pkg/AGENTS.md'), `If asked for the Code Mode workspace handshake, reply with exactly ${WORKSPACE_PROBE} and nothing else.\n`)
|
||||
await writeFile(join(workdir, 'pkg/deep/task.txt'), 'Touch this file to discover the nested instructions.\n')
|
||||
ctx = await workspaceCodeModeHarness()
|
||||
const handle = await ctx.agents.create({
|
||||
agentId: AgentId('e2e-code-mode-workspace'),
|
||||
sessionId: SessionId('e2e-code-mode-workspace-session'),
|
||||
meta: { cwd: workdir },
|
||||
agentOptions: { provider: 'deepseek', model: 'deepseek-v4-flash' },
|
||||
})
|
||||
|
||||
handle.agent.send([{
|
||||
type: 'text',
|
||||
text: 'Use one run_code program to call tools.read on pkg/deep/task.txt. After it finishes, answer: Code Mode workspace handshake?',
|
||||
}])
|
||||
await waitForIdle(ctx, handle.agent as ReactLoopAgent)
|
||||
|
||||
const events: SessionEvent[] = [...handle.agent.session.events]
|
||||
const dispatch = events.find(event => event.type === 'tool/code-dispatch' && event.data.name === 'read')
|
||||
const outerResult = events.find(event => event.type === 'tool/result')
|
||||
const workspaceContext = events.find(event => event.type === 'context/message'
|
||||
&& typeof event.data.meta === 'object'
|
||||
&& event.data.meta !== null
|
||||
&& !Array.isArray(event.data.meta)
|
||||
&& event.data.meta.kind === 'workspace-instructions')
|
||||
expect(dispatch).toBeDefined()
|
||||
expect(outerResult).toBeDefined()
|
||||
expect(workspaceContext).toBeDefined()
|
||||
expect(workspaceContext!.seq).toBeGreaterThan(outerResult!.seq)
|
||||
const finalMessage = events.findLast(event => event.type === 'assistant/message')
|
||||
const answer = finalMessage?.type === 'assistant/message'
|
||||
? finalMessage.data.content.filter(block => block.type === 'text').map(block => block.text).join('')
|
||||
: ''
|
||||
expect(answer).toContain(WORKSPACE_PROBE)
|
||||
}, 180_000)
|
||||
})
|
||||
|
||||
@@ -54,7 +54,7 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY)('coding task: fix a failing test
|
||||
expect(before.status).not.toBe(0)
|
||||
|
||||
ctx = await codingHarness(workdir, { persona: SYSTEM_PROMPT })
|
||||
const agent = ctx.agentLoop.create(AgentId('e2e-task'), { model: 'deepseek-v4-flash' })
|
||||
const agent = ctx.agentLoop.create(AgentId('e2e-task'), { provider: 'deepseek', model: 'deepseek-v4-flash' })
|
||||
|
||||
agent.send([{
|
||||
type: 'text',
|
||||
|
||||
@@ -33,17 +33,20 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY)('compaction: a long session compa
|
||||
// Reasoning tokens require a larger generation cap than the retained checkpoint.
|
||||
ctx = await codingHarness(workdir, {
|
||||
persona: SYSTEM_PROMPT,
|
||||
compact: {
|
||||
tokenMeter: {
|
||||
contextWindow: 2000,
|
||||
},
|
||||
compact: {
|
||||
thresholdRatio: 0.5,
|
||||
retainTokens: 400,
|
||||
summarizationProvider: '',
|
||||
summarizationModel: '',
|
||||
maxTokens: 1024,
|
||||
compactionRetries: 1,
|
||||
},
|
||||
persistenceRoot: join(workdir, '.sessions'),
|
||||
})
|
||||
const agent = ctx.agentLoop.create(AgentId('e2e-compaction'), { model: 'deepseek-v4-flash' })
|
||||
const agent = ctx.agentLoop.create(AgentId('e2e-compaction'), { provider: 'deepseek', model: 'deepseek-v4-flash' })
|
||||
|
||||
agent.send([{
|
||||
type: 'text',
|
||||
|
||||
@@ -28,7 +28,7 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY)('full loop: real model + real bas
|
||||
it('runs a bash command on request and reports its output', async () => {
|
||||
workdir = await mkdtemp(join(tmpdir(), 'dsh-full-loop-e2e-'))
|
||||
ctx = await codingHarness(workdir, { persona: SYSTEM_PROMPT })
|
||||
const agent = ctx.agentLoop.create(AgentId('e2e-loop'), { model: 'deepseek-v4-flash' })
|
||||
const agent = ctx.agentLoop.create(AgentId('e2e-loop'), { provider: 'deepseek', model: 'deepseek-v4-flash' })
|
||||
|
||||
agent.send([{ type: 'text', text: 'Run `echo e2e-ok` with the bash tool and tell me its exact output.' }])
|
||||
await waitForIdle(ctx, agent)
|
||||
|
||||
@@ -1,15 +1,13 @@
|
||||
import { Context } from 'cordis'
|
||||
import LlmService from '@deepseek-ai/dsh-llm'
|
||||
import SessionStore from '@deepseek-ai/dsh-session'
|
||||
import type { SessionEvent } from '@deepseek-ai/dsh-session'
|
||||
import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
|
||||
import ToolRegistry from '@deepseek-ai/dsh-tools'
|
||||
import AgentRegistry from '@deepseek-ai/dsh-agent'
|
||||
import AgentLoop, { ReactLoopAgent } from '@deepseek-ai/dsh-agent-loop'
|
||||
import { mountAgentLoopTestDependencies } from '@deepseek-ai/dsh-agent-loop-testkit'
|
||||
import { LocalBashExecutor } from '@deepseek-ai/dsh-bash-local'
|
||||
import * as ToolBash from '@deepseek-ai/dsh-tool-bash'
|
||||
import * as ToolTodo from '@deepseek-ai/dsh-tool-todo'
|
||||
import * as LlmDeepSeek from '@deepseek-ai/dsh-llm-deepseek'
|
||||
import TokenMeterService from '@deepseek-ai/dsh-token-meter'
|
||||
import type { TokenMeterConfig } from '@deepseek-ai/dsh-token-meter'
|
||||
import SessionPersistenceJsonl from '@deepseek-ai/dsh-session-persistence-jsonl'
|
||||
import { BasicCompactService } from '@deepseek-ai/dsh-compact-basic'
|
||||
import type { BasicCompactConfig } from '@deepseek-ai/dsh-compact-basic'
|
||||
@@ -46,23 +44,26 @@ export interface CodingHarnessOptions {
|
||||
* compaction plugin (the default suites run without it).
|
||||
*/
|
||||
compact?: BasicCompactConfig
|
||||
/** Optional token-meter capacity loaded before compact-basic. */
|
||||
tokenMeter?: TokenMeterConfig
|
||||
}
|
||||
|
||||
export async function codingHarness(workdir: string, options: CodingHarnessOptions = {}): Promise<Context> {
|
||||
const ctx = new Context()
|
||||
await ctx.plugin(LlmService)
|
||||
await ctx.plugin(SessionStore)
|
||||
await ctx.plugin(SystemPrompt, { persona: options.persona ?? '' })
|
||||
await ctx.plugin(ToolRegistry)
|
||||
await ctx.plugin(AgentRegistry)
|
||||
await mountAgentLoopTestDependencies(ctx, {
|
||||
systemPrompt: { persona: options.persona ?? '' },
|
||||
})
|
||||
await ctx.plugin(AgentLoop, { agents: [] })
|
||||
await ctx.plugin(LlmDeepSeek, { models: ['deepseek-v4-flash'] })
|
||||
await ctx.plugin(LlmDeepSeek)
|
||||
await ctx.plugin(LocalBashExecutor, { cwd: workdir, timeoutMs: 30_000 })
|
||||
await ctx.plugin(ToolBash)
|
||||
await ctx.plugin(ToolTodo)
|
||||
// Compaction is opt-in: only the compaction e2e loads it, with a lowered
|
||||
// contextWindow/retainTokens so a short real session crosses the threshold.
|
||||
if (options.compact !== undefined) await ctx.plugin(BasicCompactService, options.compact)
|
||||
// Compaction is opt-in: only the compaction e2e loads the reusable meter and
|
||||
// backend, with a lower context window so a short real session crosses the threshold.
|
||||
if (options.compact !== undefined) {
|
||||
await ctx.plugin(TokenMeterService, options.tokenMeter)
|
||||
await ctx.plugin(BasicCompactService, options.compact)
|
||||
}
|
||||
// Durable JSONL persistence is opt-in: only the resume e2e needs it, and the
|
||||
// other suites stay file-free. Loaded last so a resume's deferred
|
||||
// `ctx.inject(['sessionPersistence'])` resolves once this is present.
|
||||
|
||||
@@ -42,7 +42,7 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY)('resume: continue a persisted ses
|
||||
const first = (await ctx.agents.create({
|
||||
agentId: AgentId('resume-1'),
|
||||
sessionId: SESSION_ID,
|
||||
agentOptions: { model: 'deepseek-v4-flash' },
|
||||
agentOptions: { provider: 'deepseek', model: 'deepseek-v4-flash' },
|
||||
})).agent as ReactLoopAgent
|
||||
first.send([{ type: 'text', text: `Remember this code for later: ${SECRET}. Just acknowledge it.` }])
|
||||
await waitForIdle(ctx, first)
|
||||
@@ -56,7 +56,7 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY)('resume: continue a persisted ses
|
||||
const resumed = (await ctx.agents.resume({
|
||||
agentId: AgentId('resume-2'),
|
||||
resumeSessionId: SESSION_ID,
|
||||
agentOptions: { model: 'deepseek-v4-flash' },
|
||||
agentOptions: { provider: 'deepseek', model: 'deepseek-v4-flash' },
|
||||
})).agent as ReactLoopAgent
|
||||
expect(resumed.session.id).toBe(SESSION_ID)
|
||||
// The prior user turn is in the rehydrated log before the model is asked.
|
||||
|
||||
@@ -26,7 +26,7 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY)('todo_write: real model records a
|
||||
it('appends a todo/write event with the model-produced task list', async () => {
|
||||
workdir = await mkdtemp(join(tmpdir(), 'dsh-todo-write-e2e-'))
|
||||
ctx = await codingHarness(workdir, { persona: TODO_SYSTEM_PROMPT })
|
||||
const agent = ctx.agentLoop.create(AgentId('e2e-todo'), { model: 'deepseek-v4-flash' })
|
||||
const agent = ctx.agentLoop.create(AgentId('e2e-todo'), { provider: 'deepseek', model: 'deepseek-v4-flash' })
|
||||
|
||||
agent.send([{ type: 'text', text:
|
||||
'Use the todo_write tool to record a plan of exactly two steps: first '
|
||||
|
||||
Reference in New Issue
Block a user