test(acp): require a real sandbox denial

This commit is contained in:
Tianyi Cui
2026-07-23 16:39:02 +08:00
parent 7c0c516f60
commit be62eecc60
2 changed files with 41 additions and 19 deletions

View File

@@ -117,7 +117,7 @@ fs/web/todo execute in-process, so their sandbox semantics are policy at their s
- **Unit:** pin platform selection and profiles, fail-closed runner classification, per-call mode/root resolution, per-process facts, escalation validation and outcomes, permission preset folding and write-through, narrator coalescing, ACP advertisement and validation, and turn-enclosed config writes.
- **Keyless real-runner:** exercise bwrap, Landlock, and Seatbelt against real filesystem effects at provider and bash-consumer layers; one real Cordis context concurrently drives two project sessions through shipped bash and fs tools, proving own-root success and sibling-root denial. Packed-install coverage proves the registry launcher remains executable. The real ACP composition pins permission switching and rejects unknown presets. CI rejects a silent all-skip.
- **With-key:** drive a real model, runner, bridge answerer, and disk effect through granted and rejected escalation; unavailable credentials or runners self-skip.
- **With-key:** start the real ACP composition in read-only mode, let a model-driven bash write hit the runner's denial marker, then drive the bridge answerer and disk effect through granted and rejected workspace-write retries; unavailable credentials or runners self-skip.
- **Snapshot:** pin the permission config-option wire, preset and knob events, prompt deltas and notices, and both scripted approval branches. A real ACP example scenario places its session under the user home while the deployment fallback points at `/tmp`, then pins a successful workspace-write mutation; this distinguishes session-root resolution from the process fallback without depending on runner-specific denial text. Other snapshots start unconfined so unrelated fixtures remain platform-independent, and policy scenarios switch explicitly.
## Deferred phases

View File

@@ -25,13 +25,11 @@ import { cleanupAcpExampleTest } from './cleanup.ts'
* model nor a sandbox runner is ever exercised.
*
* With-key escalation flow (self-skips without DEEPSEEK_API_KEY or a usable
* platform runner): a scripted ACP client plays the human. The prompt asserts
* a prior denial (the organic denial→marker path lives on the sandbox e2e
* legs and unit tiers), the real model escalates with `sandbox_permissions` +
* `justification`, the bridge prompts THIS client over
* `session/request_permission`, the client answers `allow-once`, and the
* retried write must land ON DISK (world-verified) — under the granted mode,
* a temp-dir session cwd is writable either way.
* platform runner): a scripted ACP client plays the human. The subprocess
* starts read-only, its first real bash write is denied, the model retries with
* `sandbox_permissions` + `justification`, and the bridge prompts THIS client
* over `session/request_permission`. An approved workspace-write retry must
* then land ON DISK (world-verified).
*/
const AGENT: AgentUnderTest = {
@@ -58,14 +56,21 @@ interface Spawned extends LaunchedAcpTestAgent {
permissionRequests: RequestPermissionRequest[]
}
/** Boot the example as an ACP subprocess; the scripted client answers every permission prompt with `answer`. */
function launchExampleAcpAgent(cwd: string, answer: 'allow-once' | 'reject-once'): Spawned {
/** Boot the example with an optional sandbox override; the scripted client answers every permission prompt with `answer`. */
function launchExampleAcpAgent(
cwd: string,
answer: 'allow-once' | 'reject-once',
sandboxMode?: 'read-only' | 'workspace-write' | 'danger-full-access',
): Spawned {
const permissionRequests: RequestPermissionRequest[] = []
const launched = launchAcpTestAgent({
agent: AGENT,
cwd,
// A dummy key lets the adapter boot keylessly; live tests carry the real key.
env: { DEEPSEEK_API_KEY: process.env.DEEPSEEK_API_KEY ?? 'sk-dummy-for-boot' },
env: {
DEEPSEEK_API_KEY: process.env.DEEPSEEK_API_KEY ?? 'sk-dummy-for-boot',
DSH_PERMISSION_MODE: sandboxMode,
},
requestPermission(params) {
permissionRequests.push(params)
const option = params.options.find(o => o.optionId === answer)
@@ -78,6 +83,17 @@ function launchExampleAcpAgent(cwd: string, answer: 'allow-once' | 'reject-once'
return Object.assign(launched, { permissionRequests })
}
function escalationPrompt(path: string, content: string): string {
return `Create ${path} containing exactly ${JSON.stringify(content)} using bash, not filesystem tools. `
+ 'First try the command without sandbox_permissions. If the sandbox denies it, retry that exact command once '
+ 'with sandbox_permissions set to workspace-write and a one-sentence justification.'
}
function includesReadOnlyDenial(updates: LaunchedAcpTestAgent['updates']): boolean {
return updates.some(update => update.sessionUpdate === 'tool_call_update'
&& JSON.stringify(update.content).includes('[sandbox: file access denied under read-only mode]'))
}
let spawned: Spawned | undefined
let workdir: string | undefined
@@ -137,17 +153,20 @@ describe('default sandbox composition keyless smoke (real cordis.yml via the Loa
describe.skipIf(!process.env.DEEPSEEK_API_KEY || !hasRunner)('default sandbox composition e2e: the live approval loop', () => {
it('denial → model escalation → editor prompt → allow-once → the retried write lands on disk', async () => {
workdir = await mkdtemp(join(tmpdir(), 'sandbox-acp-e2e-'))
spawned = launchExampleAcpAgent(workdir, 'allow-once')
const { client, permissionRequests } = spawned
spawned = launchExampleAcpAgent(workdir, 'allow-once', 'read-only')
const { client, permissionRequests, updates } = spawned
await client.initialize({ protocolVersion: PROTOCOL_VERSION, clientCapabilities: {} })
const { sessionId } = await client.newSession({ cwd: workdir, mcpServers: [] })
const res = await client.prompt({
sessionId,
prompt: [{ type: 'text', text: `The sandbox already denied writing ${workdir}/escalated.txt. Create it now containing exactly "ACP_ESCALATION_OK": `
+ 'one single bash call with sandbox_permissions set to danger-full-access and a one-sentence justification, then stop.' }],
prompt: [{
type: 'text',
text: `${escalationPrompt(join(workdir, 'escalated.txt'), 'ACP_ESCALATION_OK')} Then stop.`,
}],
})
expect(['end_turn', 'max_tokens']).toContain(res.stopReason)
expect(includesReadOnlyDenial(updates)).toBe(true)
// The WORLD: the approved escalated retry landed the write.
const proof = await readFile(join(workdir, 'escalated.txt'), 'utf8')
@@ -166,17 +185,20 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY || !hasRunner)('default sandbox co
it('a rejected escalation stays denied: no write lands, the turn still ends', async () => {
workdir = await mkdtemp(join(tmpdir(), 'sandbox-acp-e2e-'))
spawned = launchExampleAcpAgent(workdir, 'reject-once')
const { client, permissionRequests } = spawned
spawned = launchExampleAcpAgent(workdir, 'reject-once', 'read-only')
const { client, permissionRequests, updates } = spawned
await client.initialize({ protocolVersion: PROTOCOL_VERSION, clientCapabilities: {} })
const { sessionId } = await client.newSession({ cwd: workdir, mcpServers: [] })
const res = await client.prompt({
sessionId,
prompt: [{ type: 'text', text: `The sandbox already denied writing ${workdir}/refused.txt. Create it now containing "NO": `
+ 'one single bash call with sandbox_permissions set to danger-full-access and a one-sentence justification. If that is rejected, stop and say so.' }],
prompt: [{
type: 'text',
text: `${escalationPrompt(join(workdir, 'refused.txt'), 'NO')} If approval is rejected, stop and say so.`,
}],
})
expect(['end_turn', 'max_tokens']).toContain(res.stopReason)
expect(includesReadOnlyDenial(updates)).toBe(true)
// The WORLD: rejected means the file never appeared.
await expect(readFile(join(workdir, 'refused.txt'), 'utf8')).rejects.toThrow()