deepseek-harness/packages/support/acp-snapshot/src/harness.ts

/**
 * Shared subprocess harness for ACP snapshot suites. A library module driven by
 * the suite factory in ./suite.ts (and directly by harness-level specs); each
 * example's `*.snapshot.ts` names its own agent-under-test paths.
 *
 * It boots the REAL agent bin subprocess via the cordis Loader (so the
 * export-shape bug class stays guarded — see docs/postmortem/0001), drives it
 * over real ACP JSON-RPC stdio with a deterministic input script, tees raw
 * stdout (for the expected-output and purity checks) into an SDK `ClientSideConnection`,
 * and — in record mode — harvests the persisted session JSONL after a graceful
 * shutdown flush. The pure normalizers in ./normalize.ts turn the captured
 * stdout frames and the session-log events into stable, snapshot-able text.
 *
 * See .agents/notes/implemented/testing/2026-06-19-acp-snapshot-tests.md.
 *
 * @module @deepseek-ai/dsh-acp-snapshot/harness
 */

import { cp, mkdtemp, readFile, readdir, rm } from 'node:fs/promises'
import { existsSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join, delimiter } from 'node:path'
import {
  ClientSideConnection,
  PROTOCOL_VERSION,
  type RequestPermissionRequest,
  type RequestPermissionResponse,
  type SessionNotification,
} from '@agentclientprotocol/sdk'
import { launchAcpTestAgent, type AgentUnderTest, type LaunchedAcpTestAgent } from './launcher.ts'

export type { AgentUnderTest } from './launcher.ts'

/**
 * One step of a scenario's deterministic input script (`input.json`). The
 * harness interprets these in order. `newSession` captures the server-issued
 * (random) session id into a `{{sessionId}}` variable that later steps
 * reference, since a committed file cannot know the id in advance.
 *
 * `promptAndCancel` starts a prompt without awaiting completion, waits until
 * the client observes the selected update (`agent_message_chunk` by default),
 * then cancels and awaits completion. A named `waitForToolCallUpdate` keeps the
 * step open for a terminal tool update that may follow the prompt response.
 * `promptAndWaitForAgentMessage` arms an exact text-chunk waiter before sending
 * the prompt, then keeps the application live until that later update arrives.
 */
export type InputStep =
  | { op: 'initialize'; terminalOutput?: boolean }
  | { op: 'newSession' }
  | { op: 'newSessionExpectError'; additionalDirectories?: string[] }
  | { op: 'prompt'; text: string }
  | { op: 'promptAndWaitForAgentMessage'; text: string; waitForText: string }
  | { op: 'promptExpectError'; text: string }
  | {
    op: 'promptAndCancel'
    text: string
    afterUpdate?: 'agent_message_chunk' | 'tool_call'
    waitForToolCallUpdate?: string
  }
  | { op: 'cancel' }
  | { op: 'setConfigOption'; configId: string; value: string }
  | { op: 'setConfigOptionExpectError'; configId: string; value: string }

/** A scenario's `input.json`: an ordered list of input steps. */
export interface InputScript {
  steps: InputStep[]
  /**
   * Ordered answers for the agent's `session/request_permission` round-trips,
   * consumed FIFO — the Nth request gets the Nth answer. Each answer selects
   * by option KIND: option ids are agent-issued randoms a committed script
   * cannot know, while kinds are the ACP-stable vocabulary, so the client maps
   * kind → the offered `optionId` at answer time. A request beyond the queue
   * (or with no queue at all) is answered `cancelled` — the stub behavior a
   * scenario without approvals relies on. A scripted kind the request does
   * not offer REJECTS the run: the scenario scripted an impossible click,
   * and {@link runScenario} throws once the in-flight step settles (the
   * agent itself just sees `cancelled`, so it cannot absorb the bug).
   */
  permissionAnswers?: PermissionAnswer[]
}

/** One scripted answer to a permission request: which offered option kind to select. */
export interface PermissionAnswer {
  /** The `PermissionOption.kind` to select (`allow_once`, `reject_always`, …). */
  kind: 'allow_once' | 'allow_always' | 'reject_once' | 'reject_always'
}

/** One harvested session log plus the identifying facts off its header line. */
export interface HarvestedLog {
  /** The recorded session id (header `id`). */
  id: string
  /** Session creation time (header `createdAt`) — the child-ordering key. */
  createdAt: number
  /** The parent session id, if this log is a subagent child (header `parentSession`). */
  parentSession?: string
  /** The full `.jsonl` file content. */
  content: string
}

/** The result of running a scenario: raw stdout + the harvested session log(s). */
export interface RunResult {
  /** Raw stdout bytes (decoded utf8), every newline-delimited JSON-RPC frame. */
  rawStdout: string
  /** stderr (for diagnostics on failure). */
  stderr: string
  /** The session id the server issued (undefined if no session was created). */
  sessionId?: string
  /** The temp cwd the session ran in (the bash workspace). */
  cwd: string
  /**
   * Every persisted session log harvested after the run, ordered primary-first:
   * the top-level (parent) session — the one with no `parentSession` — then each
   * subagent child by ascending `createdAt`. A single-session scenario harvests
   * exactly one; a nested-agent scenario harvests the parent plus one per child.
   */
  sessionLogs: HarvestedLog[]
}

/** How to run one scenario: the agent to boot, the mode, and the fixture wiring. */
export interface RunOptions {
  /** The agent composition to boot. */
  agent: AgentUnderTest
  /** `replay` (default, keyless) or `record` (real API, harvests the log). */
  mode: 'replay' | 'record'
  /** The recorded session JSONL fixture path (replay reads it; record writes near it). */
  fixtureFile: string
  /** Optional sidecar override path (replay). */
  overrideFile?: string
  /**
   * Recorded SUBAGENT child-session fixture paths (replay). A nested-agent
   * scenario ships one per child (`session.1.jsonl`, …); the harness forwards
   * them to `dsh-llm-replay` via `$DSH_SNAPSHOT_CHILD_FILES` so each child
   * session replays from its own recorded script. Empty for single-session
   * scenarios. Ignored in record mode (children are harvested, not replayed).
   */
  childFiles?: string[]
  /**
   * Optional `<scenario>/workspace/` directory whose contents are copied into
   * the temp cwd BEFORE the run — the standard way to seed files the agent
   * operates on (a file to read, edit, or grep). Absent for scenarios that
   * start from an empty workspace.
   */
  workspaceDir?: string
  /**
   * Alternate LIVE config path for the boot (absolute), overriding
   * {@link AgentUnderTest.configPath} for this run. A scenario needing a
   * differently-composed tree (the Code Mode scenarios) ships an overlay
   * whose basename still ends in `cordis.yml`, so the bin's replay swap
   * resolves the sibling `*cordis.snapshot.yml` the same way it does for
   * the default.
   */
  configPath?: string
}

/**
 * Run a scenario end-to-end against a freshly-spawned subprocess. Owns the
 * child and its temp dirs; always tears them down. Returns the captured stdout
 * and (record mode) the harvested session-log path.
 *
 * @param input The scenario's input script (steps + optional permission answers).
 * @param opts The agent to boot, the mode, and the fixture wiring.
 * @returns The captured stdout/stderr, session id, temp cwd, and harvested logs.
 */
export async function runScenario(input: InputScript, opts: RunOptions): Promise<RunResult> {
  const cwd = await mkdtemp(join(tmpdir(), 'acp-snap-cwd-'))
  const sessionsRoot = await mkdtemp(join(tmpdir(), 'acp-snap-sessions-'))
  // Fixed path length: spill-policy budgets the preview against the REAL path
  // before stdout normalization, so tmpdir() length differences churn expected outputs.
  const spillRoot = '/tmp/dsh-acp-snapshot-spill'
  // Everything past the temp-dir creation is followed by failure-safe cleanup,
  // so a failure in workspace seeding, spawn, or any step never leaks resources.
  let launched: LaunchedAcpTestAgent | undefined
  let sessionId: string | undefined
  let sessionLogs: HarvestedLog[] = []
  const outcome = await (async (): Promise<RunResult> => {
    // Seed the workspace if the scenario ships one (a file the agent reads/edits).
    // Copied into the temp cwd so the agent's bash tools see it; the expected outputs
    // normalize the cwd, so the seeded paths stay stable across runs.
    if (opts.workspaceDir !== undefined && existsSync(opts.workspaceDir)) {
      await cp(opts.workspaceDir, cwd, { recursive: true })
    }
    const env: NodeJS.ProcessEnv = {
      DSH_SNAPSHOT: opts.mode,
      DSH_SNAPSHOT_FILE: opts.fixtureFile,
      DSH_SNAPSHOT_SESSIONS_ROOT: sessionsRoot,
      DSH_SNAPSHOT_SPILL_ROOT: spillRoot,
      DSH_HOME: join(cwd, '.dsh'),
      DSH_AGENTS_HOME: join(cwd, '.agents'),
      ...opts.overrideFile !== undefined ? { DSH_SNAPSHOT_OVERRIDE: opts.overrideFile } : {},
      ...opts.childFiles !== undefined && opts.childFiles.length > 0
        ? { DSH_SNAPSHOT_CHILD_FILES: opts.childFiles.join(delimiter) }
        : {},
    }

    // Permission answers are consumed FIFO across the whole run; exhaustion
    // falls back to `cancelled` so approval-free scenarios keep the plain stub.
    const permissionQueue = [...input.permissionAnswers ?? []]
    // A scenario bug detected inside a client callback (a scripted permission
    // kind the agent never offered). It cannot fail the run from in there: a
    // callback throw only becomes a JSON-RPC error RESPONSE to the agent, and
    // a tolerant agent treats that as a denial and carries on — the run (or
    // worse, a record) would absorb the impossible click silently. So the
    // callback answers `cancelled` (a well-defined path for the agent),
    // captures the error here, and the step loop fails the run on it.
    let scriptError: Error | undefined
    launched = launchAcpTestAgent({
      agent: opts.agent,
      cwd,
      ...opts.configPath !== undefined ? { configPath: opts.configPath } : {},
      env,
      requestPermission(params: RequestPermissionRequest): Promise<RequestPermissionResponse> {
        const answer = permissionQueue.shift()
        if (answer === undefined) return Promise.resolve({ outcome: { outcome: 'cancelled' } })
        const option = params.options.find(o => o.kind === answer.kind)
        if (option === undefined) {
          // The scenario scripted a click the agent never offered — a scenario
          // bug. Captured (last one wins; same bug class either way) and
          // answered `cancelled`; the step loop rejects the run on it.
          scriptError = new Error(
            `snapshot-harness: scripted permission answer ${answer.kind} not among `
            + `the offered options [${params.options.map(o => o.kind).join(', ')}]`,
          )
          return Promise.resolve({ outcome: { outcome: 'cancelled' } })
        }
        return Promise.resolve({ outcome: { outcome: 'selected', optionId: option.optionId } })
      },
    })
    const active = launched
    await active.spawned
    const { client } = active

    for (const step of input.steps) {
      await runStep(client, step, cwd, match => active.waitForUpdate(match), () => sessionId, (id) => { sessionId = id })
      // A permission exchange happens while a step's request is in flight, so
      // by the time the step settles any script bug it exposed is captured —
      // fail the run HERE, as a harness error, rather than hoping the agent's
      // reaction to the answer perturbs the transcript.
      if (scriptError !== undefined) throw scriptError
    }
    // Done driving: close stdin so the server disposes gracefully (flushing
    // persistence) and exits. Then await exit so the harvested log is complete.
    await active.close()
    // Harvest EVERY persisted log (parent + any subagent children) while the
    // temp dirs still exist, ordered primary-first.
    sessionLogs = await harvestSessionLogs(sessionsRoot)
    return {
      rawStdout: launched.rawStdout(),
      stderr: launched.stderr(),
      cwd,
      ...sessionId !== undefined ? { sessionId } : {},
      sessionLogs,
    }
  })().then(
    value => ({ status: 'fulfilled', value } as const),
    (error: unknown) => {
      const stderr = launched?.stderr() ?? ''
      return {
        status: 'rejected',
        error: stderr === ''
          ? error
          : new Error(`snapshot-harness: scenario failed: ${String(error)}\nagent stderr:\n${stderr}`, { cause: error }),
      } as const
    },
  )

  // Failure-safe teardown: wait for a still-running child, then attempt every
  // owned-path removal even when an earlier cleanup rejects. Report every
  // teardown failure alongside a scenario failure so neither orthogonal
  // outcome hides the other.
  const cleanupResults: PromiseSettledResult<unknown>[] = []
  const cleanup = async (action: () => Promise<unknown>): Promise<void> => {
    cleanupResults.push(...await Promise.allSettled([action()]))
  }
  /* v8 ignore next 1 -- launch itself can only throw on a defensive synchronous spawn API failure */
  await cleanup(() => launched?.close('SIGKILL') ?? Promise.resolve())
  await cleanup(() => rm(cwd, { recursive: true, force: true }))
  await cleanup(() => rm(sessionsRoot, { recursive: true, force: true }))
  await cleanup(() => rm(spillRoot, { recursive: true, force: true }))

  const cleanupFailures = cleanupResults
    .filter((result): result is PromiseRejectedResult => result.status === 'rejected')
    .map(result => result.reason as unknown)
  if (cleanupFailures.length > 0) {
    throw new AggregateError(
      outcome.status === 'rejected' ? [outcome.error, ...cleanupFailures] : cleanupFailures,
      outcome.status === 'rejected'
        ? 'snapshot scenario and cleanup failed'
        : 'snapshot cleanup failed',
    )
  }
  if (outcome.status === 'rejected') throw outcome.error
  return outcome.value
}

/** Drive one input step over the client connection. */
async function runStep(
  client: ClientSideConnection,
  step: InputStep,
  cwd: string,
  waitForUpdate: (match: (u: SessionNotification['update']) => boolean) => Promise<SessionNotification['update']>,
  getSessionId: () => string | undefined,
  setSessionId: (id: string) => void,
): Promise<void> {
  switch (step.op) {
    case 'initialize':
      await client.initialize({
        protocolVersion: PROTOCOL_VERSION,
        clientCapabilities: step.terminalOutput === true ? { _meta: { terminal_output: true } } : {},
      })
      return
    case 'newSession': {
      const { sessionId } = await client.newSession({ cwd, mcpServers: [] })
      setSessionId(sessionId)
      return
    }
    case 'newSessionExpectError': {
      // The bridge rejects a session/new that widens the workspace scope
      // (non-empty additionalDirectories / mcpServers — unimplemented). The SDK
      // surfaces that as a rejected RPC; swallow it so the run completes and the
      // error frame is captured in the transcript.
      await client.newSession({
        cwd,
        mcpServers: [],
        ...step.additionalDirectories !== undefined ? { additionalDirectories: step.additionalDirectories } : {},
      }).then(
        () => { throw new Error('snapshot-harness: expected session/new to be rejected but it succeeded') },
        () => { /* expected: the bridge rejected the unsupported workspace scope */ },
      )
      return
    }
    case 'prompt': {
      const sessionId = getSessionId()
      if (sessionId === undefined) throw new Error('snapshot-harness: prompt before newSession')
      await client.prompt({ sessionId, prompt: [{ type: 'text', text: step.text }] })
      return
    }
    case 'promptAndWaitForAgentMessage': {
      const sessionId = getSessionId()
      if (sessionId === undefined) throw new Error('snapshot-harness: promptAndWaitForAgentMessage before newSession')
      const updateDone = waitForUpdate(update => update.sessionUpdate === 'agent_message_chunk'
        && update.content.type === 'text' && update.content.text === step.waitForText)
      await client.prompt({ sessionId, prompt: [{ type: 'text', text: step.text }] })
      await updateDone
      return
    }
    case 'promptExpectError': {
      const sessionId = getSessionId()
      if (sessionId === undefined) throw new Error('snapshot-harness: promptExpectError before newSession')
      // The model fails this turn (a recorded provider error), so the bridge
      // answers the prompt with a JSON-RPC error and the SDK rejects. That
      // rejection IS the expected editor experience — swallow it so the run
      // completes and the stdout transcript (the error frame) is captured.
      await client.prompt({ sessionId, prompt: [{ type: 'text', text: step.text }] })
        .then(() => { throw new Error('snapshot-harness: expected the prompt to fail but it succeeded') },
          () => { /* expected: the turn failed and the bridge returned an error */ })
      return
    }
    case 'promptAndCancel': {
      const sessionId = getSessionId()
      if (sessionId === undefined) throw new Error('snapshot-harness: promptAndCancel before newSession')
      // Dispatch without awaiting because the fixture does not settle on its
      // own. Waiting for the selected update pins it before cancellation and
      // the cancelled prompt response in the transcript.
      const promptDone = client.prompt({ sessionId, prompt: [{ type: 'text', text: step.text }] })
      const afterUpdate = step.afterUpdate ?? 'agent_message_chunk'
      await waitForUpdate(u => u.sessionUpdate === afterUpdate)
      // Arm this before cancellation so a fast tool drain cannot outrun the waiter.
      const toolCallUpdateDone = step.waitForToolCallUpdate === undefined
        ? undefined
        : waitForUpdate(u => u.sessionUpdate === 'tool_call_update' && u.toolCallId === step.waitForToolCallUpdate)
      await client.cancel({ sessionId })
      await promptDone
      if (toolCallUpdateDone !== undefined) await toolCallUpdateDone
      return
    }
    case 'cancel': {
      const sessionId = getSessionId()
      if (sessionId === undefined) throw new Error('snapshot-harness: cancel before newSession')
      await client.cancel({ sessionId })
      return
    }
    case 'setConfigOption': {
      const sessionId = getSessionId()
      if (sessionId === undefined) throw new Error('snapshot-harness: setConfigOption before newSession')
      await client.setSessionConfigOption({ sessionId, configId: step.configId, value: step.value })
      return
    }
    case 'setConfigOptionExpectError': {
      const sessionId = getSessionId()
      if (sessionId === undefined) throw new Error('snapshot-harness: setConfigOptionExpectError before newSession')
      // The bridge rejects an unknown id / out-of-vocabulary value; the SDK
      // surfaces that as a rejected RPC — swallow it so the run completes and
      // the error frame is captured in the transcript.
      await client.setSessionConfigOption({ sessionId, configId: step.configId, value: step.value }).then(
        () => { throw new Error('snapshot-harness: expected set_config_option to be rejected but it succeeded') },
        () => { /* expected: the bridge rejected the id or value */ },
      )
      return
    }
    default:
      throw new Error(`snapshot-harness: unknown input op ${JSON.stringify(step)}`)
  }
}

/**
 * Harvest EVERY persisted `.jsonl` session log under a sessions root, parse each
 * header line, and return them ordered primary-first: the top-level session (no
 * `parentSession`) leads, then each subagent child by ascending `createdAt`.
 *
 * Snapshot configs select the JSONL backend's raw mode, which lays sessions
 * out as `<root>/<cwd-bucket>/<encoded-id>.jsonl` (one bucket per cwd). A
 * parent and its same-cwd in-process child land in the SAME bucket, so
 * collecting all files across all buckets catches both. Returns `[]` if no log
 * was produced (a no-session scenario).
 */
async function harvestSessionLogs(root: string): Promise<HarvestedLog[]> {
  let cwdDirs: string[]
  try {
    cwdDirs = await readdir(root)
  } catch {
    return []
  }
  const logs: HarvestedLog[] = []
  for (const dir of cwdDirs) {
    const sub = join(root, dir)
    let files: string[]
    try {
      files = await readdir(sub)
    } catch {
      continue
    }
    for (const f of files) {
      if (!f.endsWith('.jsonl')) continue
      const content = await readFile(join(sub, f), 'utf8')
      const firstLine = content.split('\n').find(line => line.trim().length > 0) ?? '{}'
      const header = JSON.parse(firstLine) as { id?: unknown; createdAt?: unknown; parentSession?: unknown }
      logs.push({
        id: typeof header.id === 'string' ? header.id : '',
        createdAt: typeof header.createdAt === 'number' ? header.createdAt : 0,
        ...typeof header.parentSession === 'string' ? { parentSession: header.parentSession } : {},
        content,
      })
    }
  }
  // Primary (no parentSession) first, then children by ascending createdAt. A
  // scenario has exactly one top-level session. In the synchronous cut sibling
  // children are created strictly sequentially, so their createdAt values are
  // strictly ordered; the recordedId tiebreak only keeps a degenerate
  // same-millisecond collision (unreachable here) deterministic. This harvest
  // order must match the replay load order in dsh-llm-replay's loadSessionScripts
  // so session.<n>.jsonl maps to the same child on record and replay — replay
  // re-sorts childFiles by the same key, so the two stay consistent.
  logs.sort((a, b) => {
    const ap = a.parentSession === undefined ? 0 : 1
    const bp = b.parentSession === undefined ? 0 : 1
    return ap - bp || a.createdAt - b.createdAt || a.id.localeCompare(b.id)
  })
  return logs
}