mirror of
https://github.com/deepseek-ai/deepseek-harness
synced 2026-08-15 21:04:50 +00:00
test(acp): require a real sandbox denial
This commit is contained in:
@@ -117,7 +117,7 @@ fs/web/todo execute in-process, so their sandbox semantics are policy at their s
|
||||
|
||||
- **Unit:** pin platform selection and profiles, fail-closed runner classification, per-call mode/root resolution, per-process facts, escalation validation and outcomes, permission preset folding and write-through, narrator coalescing, ACP advertisement and validation, and turn-enclosed config writes.
|
||||
- **Keyless real-runner:** exercise bwrap, Landlock, and Seatbelt against real filesystem effects at provider and bash-consumer layers; one real Cordis context concurrently drives two project sessions through shipped bash and fs tools, proving own-root success and sibling-root denial. Packed-install coverage proves the registry launcher remains executable. The real ACP composition pins permission switching and rejects unknown presets. CI rejects a silent all-skip.
|
||||
- **With-key:** drive a real model, runner, bridge answerer, and disk effect through granted and rejected escalation; unavailable credentials or runners self-skip.
|
||||
- **With-key:** start the real ACP composition in read-only mode, let a model-driven bash write hit the runner's denial marker, then drive the bridge answerer and disk effect through granted and rejected workspace-write retries; unavailable credentials or runners self-skip.
|
||||
- **Snapshot:** pin the permission config-option wire, preset and knob events, prompt deltas and notices, and both scripted approval branches. A real ACP example scenario places its session under the user home while the deployment fallback points at `/tmp`, then pins a successful workspace-write mutation; this distinguishes session-root resolution from the process fallback without depending on runner-specific denial text. Other snapshots start unconfined so unrelated fixtures remain platform-independent, and policy scenarios switch explicitly.
|
||||
|
||||
## Deferred phases
|
||||
|
||||
@@ -25,13 +25,11 @@ import { cleanupAcpExampleTest } from './cleanup.ts'
|
||||
* model nor a sandbox runner is ever exercised.
|
||||
*
|
||||
* With-key escalation flow (self-skips without DEEPSEEK_API_KEY or a usable
|
||||
* platform runner): a scripted ACP client plays the human. The prompt asserts
|
||||
* a prior denial (the organic denial→marker path lives on the sandbox e2e
|
||||
* legs and unit tiers), the real model escalates with `sandbox_permissions` +
|
||||
* `justification`, the bridge prompts THIS client over
|
||||
* `session/request_permission`, the client answers `allow-once`, and the
|
||||
* retried write must land ON DISK (world-verified) — under the granted mode,
|
||||
* a temp-dir session cwd is writable either way.
|
||||
* platform runner): a scripted ACP client plays the human. The subprocess
|
||||
* starts read-only, its first real bash write is denied, the model retries with
|
||||
* `sandbox_permissions` + `justification`, and the bridge prompts THIS client
|
||||
* over `session/request_permission`. An approved workspace-write retry must
|
||||
* then land ON DISK (world-verified).
|
||||
*/
|
||||
|
||||
const AGENT: AgentUnderTest = {
|
||||
@@ -58,14 +56,21 @@ interface Spawned extends LaunchedAcpTestAgent {
|
||||
permissionRequests: RequestPermissionRequest[]
|
||||
}
|
||||
|
||||
/** Boot the example as an ACP subprocess; the scripted client answers every permission prompt with `answer`. */
|
||||
function launchExampleAcpAgent(cwd: string, answer: 'allow-once' | 'reject-once'): Spawned {
|
||||
/** Boot the example with an optional sandbox override; the scripted client answers every permission prompt with `answer`. */
|
||||
function launchExampleAcpAgent(
|
||||
cwd: string,
|
||||
answer: 'allow-once' | 'reject-once',
|
||||
sandboxMode?: 'read-only' | 'workspace-write' | 'danger-full-access',
|
||||
): Spawned {
|
||||
const permissionRequests: RequestPermissionRequest[] = []
|
||||
const launched = launchAcpTestAgent({
|
||||
agent: AGENT,
|
||||
cwd,
|
||||
// A dummy key lets the adapter boot keylessly; live tests carry the real key.
|
||||
env: { DEEPSEEK_API_KEY: process.env.DEEPSEEK_API_KEY ?? 'sk-dummy-for-boot' },
|
||||
env: {
|
||||
DEEPSEEK_API_KEY: process.env.DEEPSEEK_API_KEY ?? 'sk-dummy-for-boot',
|
||||
DSH_PERMISSION_MODE: sandboxMode,
|
||||
},
|
||||
requestPermission(params) {
|
||||
permissionRequests.push(params)
|
||||
const option = params.options.find(o => o.optionId === answer)
|
||||
@@ -78,6 +83,17 @@ function launchExampleAcpAgent(cwd: string, answer: 'allow-once' | 'reject-once'
|
||||
return Object.assign(launched, { permissionRequests })
|
||||
}
|
||||
|
||||
function escalationPrompt(path: string, content: string): string {
|
||||
return `Create ${path} containing exactly ${JSON.stringify(content)} using bash, not filesystem tools. `
|
||||
+ 'First try the command without sandbox_permissions. If the sandbox denies it, retry that exact command once '
|
||||
+ 'with sandbox_permissions set to workspace-write and a one-sentence justification.'
|
||||
}
|
||||
|
||||
function includesReadOnlyDenial(updates: LaunchedAcpTestAgent['updates']): boolean {
|
||||
return updates.some(update => update.sessionUpdate === 'tool_call_update'
|
||||
&& JSON.stringify(update.content).includes('[sandbox: file access denied under read-only mode]'))
|
||||
}
|
||||
|
||||
let spawned: Spawned | undefined
|
||||
let workdir: string | undefined
|
||||
|
||||
@@ -137,17 +153,20 @@ describe('default sandbox composition keyless smoke (real cordis.yml via the Loa
|
||||
describe.skipIf(!process.env.DEEPSEEK_API_KEY || !hasRunner)('default sandbox composition e2e: the live approval loop', () => {
|
||||
it('denial → model escalation → editor prompt → allow-once → the retried write lands on disk', async () => {
|
||||
workdir = await mkdtemp(join(tmpdir(), 'sandbox-acp-e2e-'))
|
||||
spawned = launchExampleAcpAgent(workdir, 'allow-once')
|
||||
const { client, permissionRequests } = spawned
|
||||
spawned = launchExampleAcpAgent(workdir, 'allow-once', 'read-only')
|
||||
const { client, permissionRequests, updates } = spawned
|
||||
|
||||
await client.initialize({ protocolVersion: PROTOCOL_VERSION, clientCapabilities: {} })
|
||||
const { sessionId } = await client.newSession({ cwd: workdir, mcpServers: [] })
|
||||
const res = await client.prompt({
|
||||
sessionId,
|
||||
prompt: [{ type: 'text', text: `The sandbox already denied writing ${workdir}/escalated.txt. Create it now containing exactly "ACP_ESCALATION_OK": `
|
||||
+ 'one single bash call with sandbox_permissions set to danger-full-access and a one-sentence justification, then stop.' }],
|
||||
prompt: [{
|
||||
type: 'text',
|
||||
text: `${escalationPrompt(join(workdir, 'escalated.txt'), 'ACP_ESCALATION_OK')} Then stop.`,
|
||||
}],
|
||||
})
|
||||
expect(['end_turn', 'max_tokens']).toContain(res.stopReason)
|
||||
expect(includesReadOnlyDenial(updates)).toBe(true)
|
||||
|
||||
// The WORLD: the approved escalated retry landed the write.
|
||||
const proof = await readFile(join(workdir, 'escalated.txt'), 'utf8')
|
||||
@@ -166,17 +185,20 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY || !hasRunner)('default sandbox co
|
||||
|
||||
it('a rejected escalation stays denied: no write lands, the turn still ends', async () => {
|
||||
workdir = await mkdtemp(join(tmpdir(), 'sandbox-acp-e2e-'))
|
||||
spawned = launchExampleAcpAgent(workdir, 'reject-once')
|
||||
const { client, permissionRequests } = spawned
|
||||
spawned = launchExampleAcpAgent(workdir, 'reject-once', 'read-only')
|
||||
const { client, permissionRequests, updates } = spawned
|
||||
|
||||
await client.initialize({ protocolVersion: PROTOCOL_VERSION, clientCapabilities: {} })
|
||||
const { sessionId } = await client.newSession({ cwd: workdir, mcpServers: [] })
|
||||
const res = await client.prompt({
|
||||
sessionId,
|
||||
prompt: [{ type: 'text', text: `The sandbox already denied writing ${workdir}/refused.txt. Create it now containing "NO": `
|
||||
+ 'one single bash call with sandbox_permissions set to danger-full-access and a one-sentence justification. If that is rejected, stop and say so.' }],
|
||||
prompt: [{
|
||||
type: 'text',
|
||||
text: `${escalationPrompt(join(workdir, 'refused.txt'), 'NO')} If approval is rejected, stop and say so.`,
|
||||
}],
|
||||
})
|
||||
expect(['end_turn', 'max_tokens']).toContain(res.stopReason)
|
||||
expect(includesReadOnlyDenial(updates)).toBe(true)
|
||||
|
||||
// The WORLD: rejected means the file never appeared.
|
||||
await expect(readFile(join(workdir, 'refused.txt'), 'utf8')).rejects.toThrow()
|
||||
|
||||
Reference in New Issue
Block a user