From be62eecc602b137101a0a7c6f79bc057b4f117c0 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Thu, 23 Jul 2026 16:39:02 +0800 Subject: [PATCH] test(acp): require a real sandbox denial --- .../implemented/feature/2026-07-06-sandbox.md | 2 +- examples/acp-agent/tests/escalation.e2e.ts | 58 +++++++++++++------ 2 files changed, 41 insertions(+), 19 deletions(-) diff --git a/.agents/notes/implemented/feature/2026-07-06-sandbox.md b/.agents/notes/implemented/feature/2026-07-06-sandbox.md index d5d0c6d5aa..507142fac9 100644 --- a/.agents/notes/implemented/feature/2026-07-06-sandbox.md +++ b/.agents/notes/implemented/feature/2026-07-06-sandbox.md @@ -117,7 +117,7 @@ fs/web/todo execute in-process, so their sandbox semantics are policy at their s - **Unit:** pin platform selection and profiles, fail-closed runner classification, per-call mode/root resolution, per-process facts, escalation validation and outcomes, permission preset folding and write-through, narrator coalescing, ACP advertisement and validation, and turn-enclosed config writes. - **Keyless real-runner:** exercise bwrap, Landlock, and Seatbelt against real filesystem effects at provider and bash-consumer layers; one real Cordis context concurrently drives two project sessions through shipped bash and fs tools, proving own-root success and sibling-root denial. Packed-install coverage proves the registry launcher remains executable. The real ACP composition pins permission switching and rejects unknown presets. CI rejects a silent all-skip. -- **With-key:** drive a real model, runner, bridge answerer, and disk effect through granted and rejected escalation; unavailable credentials or runners self-skip. +- **With-key:** start the real ACP composition in read-only mode, let a model-driven bash write hit the runner's denial marker, then drive the bridge answerer and disk effect through granted and rejected workspace-write retries; unavailable credentials or runners self-skip. - **Snapshot:** pin the permission config-option wire, preset and knob events, prompt deltas and notices, and both scripted approval branches. A real ACP example scenario places its session under the user home while the deployment fallback points at `/tmp`, then pins a successful workspace-write mutation; this distinguishes session-root resolution from the process fallback without depending on runner-specific denial text. Other snapshots start unconfined so unrelated fixtures remain platform-independent, and policy scenarios switch explicitly. ## Deferred phases diff --git a/examples/acp-agent/tests/escalation.e2e.ts b/examples/acp-agent/tests/escalation.e2e.ts index ae53252743..1bf0a5ac16 100644 --- a/examples/acp-agent/tests/escalation.e2e.ts +++ b/examples/acp-agent/tests/escalation.e2e.ts @@ -25,13 +25,11 @@ import { cleanupAcpExampleTest } from './cleanup.ts' * model nor a sandbox runner is ever exercised. * * With-key escalation flow (self-skips without DEEPSEEK_API_KEY or a usable - * platform runner): a scripted ACP client plays the human. The prompt asserts - * a prior denial (the organic denial→marker path lives on the sandbox e2e - * legs and unit tiers), the real model escalates with `sandbox_permissions` + - * `justification`, the bridge prompts THIS client over - * `session/request_permission`, the client answers `allow-once`, and the - * retried write must land ON DISK (world-verified) — under the granted mode, - * a temp-dir session cwd is writable either way. + * platform runner): a scripted ACP client plays the human. The subprocess + * starts read-only, its first real bash write is denied, the model retries with + * `sandbox_permissions` + `justification`, and the bridge prompts THIS client + * over `session/request_permission`. An approved workspace-write retry must + * then land ON DISK (world-verified). */ const AGENT: AgentUnderTest = { @@ -58,14 +56,21 @@ interface Spawned extends LaunchedAcpTestAgent { permissionRequests: RequestPermissionRequest[] } -/** Boot the example as an ACP subprocess; the scripted client answers every permission prompt with `answer`. */ -function launchExampleAcpAgent(cwd: string, answer: 'allow-once' | 'reject-once'): Spawned { +/** Boot the example with an optional sandbox override; the scripted client answers every permission prompt with `answer`. */ +function launchExampleAcpAgent( + cwd: string, + answer: 'allow-once' | 'reject-once', + sandboxMode?: 'read-only' | 'workspace-write' | 'danger-full-access', +): Spawned { const permissionRequests: RequestPermissionRequest[] = [] const launched = launchAcpTestAgent({ agent: AGENT, cwd, // A dummy key lets the adapter boot keylessly; live tests carry the real key. - env: { DEEPSEEK_API_KEY: process.env.DEEPSEEK_API_KEY ?? 'sk-dummy-for-boot' }, + env: { + DEEPSEEK_API_KEY: process.env.DEEPSEEK_API_KEY ?? 'sk-dummy-for-boot', + DSH_PERMISSION_MODE: sandboxMode, + }, requestPermission(params) { permissionRequests.push(params) const option = params.options.find(o => o.optionId === answer) @@ -78,6 +83,17 @@ function launchExampleAcpAgent(cwd: string, answer: 'allow-once' | 'reject-once' return Object.assign(launched, { permissionRequests }) } +function escalationPrompt(path: string, content: string): string { + return `Create ${path} containing exactly ${JSON.stringify(content)} using bash, not filesystem tools. ` + + 'First try the command without sandbox_permissions. If the sandbox denies it, retry that exact command once ' + + 'with sandbox_permissions set to workspace-write and a one-sentence justification.' +} + +function includesReadOnlyDenial(updates: LaunchedAcpTestAgent['updates']): boolean { + return updates.some(update => update.sessionUpdate === 'tool_call_update' + && JSON.stringify(update.content).includes('[sandbox: file access denied under read-only mode]')) +} + let spawned: Spawned | undefined let workdir: string | undefined @@ -137,17 +153,20 @@ describe('default sandbox composition keyless smoke (real cordis.yml via the Loa describe.skipIf(!process.env.DEEPSEEK_API_KEY || !hasRunner)('default sandbox composition e2e: the live approval loop', () => { it('denial → model escalation → editor prompt → allow-once → the retried write lands on disk', async () => { workdir = await mkdtemp(join(tmpdir(), 'sandbox-acp-e2e-')) - spawned = launchExampleAcpAgent(workdir, 'allow-once') - const { client, permissionRequests } = spawned + spawned = launchExampleAcpAgent(workdir, 'allow-once', 'read-only') + const { client, permissionRequests, updates } = spawned await client.initialize({ protocolVersion: PROTOCOL_VERSION, clientCapabilities: {} }) const { sessionId } = await client.newSession({ cwd: workdir, mcpServers: [] }) const res = await client.prompt({ sessionId, - prompt: [{ type: 'text', text: `The sandbox already denied writing ${workdir}/escalated.txt. Create it now containing exactly "ACP_ESCALATION_OK": ` - + 'one single bash call with sandbox_permissions set to danger-full-access and a one-sentence justification, then stop.' }], + prompt: [{ + type: 'text', + text: `${escalationPrompt(join(workdir, 'escalated.txt'), 'ACP_ESCALATION_OK')} Then stop.`, + }], }) expect(['end_turn', 'max_tokens']).toContain(res.stopReason) + expect(includesReadOnlyDenial(updates)).toBe(true) // The WORLD: the approved escalated retry landed the write. const proof = await readFile(join(workdir, 'escalated.txt'), 'utf8') @@ -166,17 +185,20 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY || !hasRunner)('default sandbox co it('a rejected escalation stays denied: no write lands, the turn still ends', async () => { workdir = await mkdtemp(join(tmpdir(), 'sandbox-acp-e2e-')) - spawned = launchExampleAcpAgent(workdir, 'reject-once') - const { client, permissionRequests } = spawned + spawned = launchExampleAcpAgent(workdir, 'reject-once', 'read-only') + const { client, permissionRequests, updates } = spawned await client.initialize({ protocolVersion: PROTOCOL_VERSION, clientCapabilities: {} }) const { sessionId } = await client.newSession({ cwd: workdir, mcpServers: [] }) const res = await client.prompt({ sessionId, - prompt: [{ type: 'text', text: `The sandbox already denied writing ${workdir}/refused.txt. Create it now containing "NO": ` - + 'one single bash call with sandbox_permissions set to danger-full-access and a one-sentence justification. If that is rejected, stop and say so.' }], + prompt: [{ + type: 'text', + text: `${escalationPrompt(join(workdir, 'refused.txt'), 'NO')} If approval is rejected, stop and say so.`, + }], }) expect(['end_turn', 'max_tokens']).toContain(res.stopReason) + expect(includesReadOnlyDenial(updates)).toBe(true) // The WORLD: rejected means the file never appeared. await expect(readFile(join(workdir, 'refused.txt'), 'utf8')).rejects.toThrow()