mirror of
https://github.com/deepseek-ai/deepseek-harness
synced 2026-08-15 21:04:50 +00:00
04b7f517ae
Five browser e2e scenarios over the existing keyless lane, one recorded base fixture per spec family: - live-interactions: one tool-free recorded turn + per-run override sidecars authored in the spec (content single-sourced from the fixture via deriveReplayScript, minted into a spec-owned temp dir). Cancel uses a hang patch with a readyFile marker — the marker proves the stream is parked mid-turn before the Stop click, so mid-stream cancellation is deterministic by construction (turn/end 'aborted', composer re-enabled). AUTH pins the non-retryable path: turn/end 'error', zero llm/retry events, composer recovers; FIXME(web-error-surface) marks the found product gap (no error copy renders — the client consumes no agent/error frames and a pre-chunk failure freezes no partial). SERVER retry appends the fixture's own success after an injected throw and proves llm-retry end-to-end in the browser via the durable llm/retry record. - question-composer: the shipped ask_user_question takeover blocks the turn mid-step on the real userInteraction seam; the test answers through the composer (the one sanctioned model-content-reactive drive step: the turn cannot complete without it) and the tool result carries the answer. Adds the composer waiting-state aria golden. - steering: steers mid-turn while the composer blocks the step (the deterministic mid-turn window). The steer rides the real wire (session.prompt mode:'steer' POSTed from the page; the locked composer has no steering gesture yet — TODO(web-steer-composer)); downstream is all product: gateway -> Agent.steer -> step-boundary drain -> durable steering/message -> SSE -> badged interjection bubble. Record mode rejects a fixture whose live reply ignored the steer. Scaffold gains the replayOverride passthrough; specs register in both tsconfig planes (client exclude, host include).
100 lines
5.0 KiB
TypeScript
100 lines
5.0 KiB
TypeScript
// Web e2e scenario: the resident question composer. The shipped composition
|
|
// already exposes ask_user_question (the ui-question row's node half mounts
|
|
// the tool), so a recorded turn where the model asks blocks mid-turn on the
|
|
// real userInteraction seam: the composer renders in the browser, the test
|
|
// answers through it, and the turn completes with the answer in the log.
|
|
// Replay is fully deterministic — the question content arrives from replayed
|
|
// chunks, the composer wait is real, and the answer click is the test's own
|
|
// gesture (the ONE place a drive step legitimately reacts to model content:
|
|
// the turn cannot complete without it, in record and replay alike).
|
|
import { readFile } from 'node:fs/promises'
|
|
import { fileURLToPath } from 'node:url'
|
|
import { join } from 'node:path'
|
|
import type { Browser, Page } from 'playwright'
|
|
import { chromium } from 'playwright'
|
|
import { afterAll, beforeAll, describe, expect, it, onTestFailed } from 'vitest'
|
|
import type { SessionEvent } from '@deepseek-ai/dsh-session'
|
|
import {
|
|
assertFixtureInventory, captureStableAria, compareOrRefreshGolden, fixtureUserPrompts,
|
|
launchWebScaffold, recordFixture, watchConsole, webSnapshotMode, type WebScaffold,
|
|
} from './scaffold.ts'
|
|
import { saveFailureShot } from './support.ts'
|
|
|
|
const SNAPSHOT_DIR = fileURLToPath(new URL('./snapshots/question-composer', import.meta.url))
|
|
const FIXTURE = join(SNAPSHOT_DIR, 'session.jsonl')
|
|
const UI_EXPECTED = join(SNAPSHOT_DIR, 'ui.expected.md')
|
|
const MODE = webSnapshotMode()
|
|
|
|
const PROMPT = 'Use the ask_user_question tool to ask me exactly one question with id "color", question "Which color do you prefer?", header "Pick one", and options labeled "Blue" and "Green". After I answer, reply with the single word DONE and stop.'
|
|
|
|
describe('web e2e: resident question composer round trip', () => {
|
|
let scaffold: WebScaffold
|
|
let browser: Browser
|
|
let page: Page
|
|
let tripwire: ReturnType<typeof watchConsole>
|
|
const sessionEvents: SessionEvent[] = []
|
|
|
|
beforeAll(async () => {
|
|
scaffold = await launchWebScaffold(MODE === 'record' ? {} : { replayFixture: FIXTURE, paceMs: 15 })
|
|
scaffold.ctx.on('session/event', (_session, event: SessionEvent) => { sessionEvents.push(event) })
|
|
browser = await chromium.launch()
|
|
page = await browser.newPage({ viewport: { width: 1680, height: 1000 } })
|
|
tripwire = watchConsole(page)
|
|
await page.goto(scaffold.baseUrl, { waitUntil: 'load' })
|
|
await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
|
|
}, 120_000)
|
|
|
|
afterAll(async () => {
|
|
await browser?.close()
|
|
await scaffold?.close()
|
|
})
|
|
|
|
it('asks through the composer, answers, and completes with the answer logged', async () => {
|
|
onTestFailed(() => saveFailureShot(page, 'web-e2e-question'))
|
|
if (MODE !== 'record') {
|
|
expect(fixtureUserPrompts(await readFile(FIXTURE, 'utf8'))).toEqual([PROMPT])
|
|
}
|
|
const input = page.locator('textarea').first()
|
|
await input.waitFor({ timeout: 10_000 })
|
|
const settled = scaffold.whenTurnSettled(MODE === 'record' ? 180_000 : 30_000)
|
|
await input.fill(PROMPT)
|
|
await input.press('Enter')
|
|
|
|
// The composer takes over the input area while the tool blocks. Its
|
|
// presence is a STABLE waiting state (not a transient): it stays until
|
|
// answered, so a plain waitFor is race-free.
|
|
const composer = page.locator('[data-question-key]')
|
|
await composer.waitFor({ timeout: MODE === 'record' ? 120_000 : 30_000 })
|
|
await expect.poll(() => composer.getByText('Which color do you prefer?').count(), { timeout: 10_000 }).toBeGreaterThan(0)
|
|
|
|
if (MODE !== 'record') {
|
|
// Golden of the composer's waiting state (the transcript region golden
|
|
// is #612's job; this pins the question surface).
|
|
const snapshot = await captureStableAria(page, '[data-question-key]', scaffold.workspaceCwd)
|
|
await compareOrRefreshGolden(UI_EXPECTED, snapshot, MODE)
|
|
}
|
|
|
|
await composer.getByRole('radio', { name: 'Blue' }).click()
|
|
// Submit: Enter on the focused option (the composer's documented submit).
|
|
await composer.getByRole('radio', { name: 'Blue' }).press('Enter')
|
|
|
|
const sessionId = await settled
|
|
if (MODE === 'record') {
|
|
await recordFixture(scaffold, sessionId, FIXTURE)
|
|
return
|
|
}
|
|
// World state: the tool result carries the chosen answer, and DONE lands.
|
|
const results = sessionEvents.filter(e => e.type === 'tool/result')
|
|
expect(JSON.stringify(results.at(-1))).toContain('Blue')
|
|
await expect.poll(() => page.getByText('DONE', { exact: true }).count(), { timeout: 15_000 }).toBeGreaterThanOrEqual(1)
|
|
// Composer gone; regular input restored.
|
|
expect(await page.locator('[data-question-key]').count()).toBe(0)
|
|
await expect.poll(() => page.locator('textarea').first().isEnabled(), { timeout: 10_000 }).toBe(true)
|
|
expect(tripwire.pageErrors).toEqual([])
|
|
}, 200_000)
|
|
|
|
it.skipIf(MODE === 'record')('keeps the fixture inventory closed', async () => {
|
|
await assertFixtureInventory(SNAPSHOT_DIR, ['session.jsonl', 'ui.expected.md'])
|
|
})
|
|
})
|