mirror of
https://github.com/deepseek-ai/deepseek-harness
synced 2026-08-15 21:04:50 +00:00
cb2f01f48c
Steer, inject, and followup now land as durable user/message events on the session surface; the steering/message event type and its ConversationNode kind are removed from the client projection. Update tests, docs, generated catalogs, and agent notes to match, and align the steering e2e fixture and prompt inventory assertions with the durable user/message landing.
281 lines
14 KiB
TypeScript
281 lines
14 KiB
TypeScript
// Web e2e scenarios for both steering entry points: QueueDock strictly
|
|
// transfers one queued occurrence, while the complementary composer gestures
|
|
// choose Queue or Steer. The question tool supplies a deterministic pending-
|
|
// steering snapshot before the step can drain.
|
|
import { readFile } from 'node:fs/promises'
|
|
import { fileURLToPath } from 'node:url'
|
|
import { join } from 'node:path'
|
|
import type { Browser, Page } from 'playwright'
|
|
import { chromium } from 'playwright'
|
|
import { afterAll, beforeAll, describe, expect, it, onTestFailed } from 'vitest'
|
|
import { parseSessionLog } from '@deepseek-ai/dsh-llm-replay'
|
|
import type { SessionEvent } from '@deepseek-ai/dsh-session'
|
|
import {
|
|
assertFixtureInventory, captureStableAria, compareOrRefreshGolden, fixtureUserPrompts,
|
|
launchWebScaffold, recordFixture, watchConsole, webSnapshotMode, type WebScaffold,
|
|
} from './scaffold.ts'
|
|
import { connectFreshWorkspace, newEnglishPage, saveFailureShot } from './support.ts'
|
|
|
|
const SNAPSHOT_DIR = fileURLToPath(new URL('./snapshots/steering', import.meta.url))
|
|
const FIXTURE = join(SNAPSHOT_DIR, 'session.jsonl')
|
|
// Two goldens pin the transient Host projection and its durable handoff: the
|
|
// mid-turn state renders accepted steering from session/queue while the
|
|
// question blocks admission, then the settled state renders the same message
|
|
// from user/message beside the reply that obeys it.
|
|
const MID_EXPECTED = join(SNAPSHOT_DIR, 'mid-steer.expected.md')
|
|
const SETTLED_EXPECTED = join(SNAPSHOT_DIR, 'settled.expected.md')
|
|
const MODE = webSnapshotMode()
|
|
// The question composer replaces the textarea, so fill → Queue row → Steer
|
|
// must finish inside the first replay chunk window. At 15 ms that window is
|
|
// shorter than Playwright's round trips; 100 ms supplies test-only headroom,
|
|
// while larger values lengthen all three replay scenarios linearly.
|
|
const REPLAY_PACE_MS = 100
|
|
|
|
const PROMPT = 'Use the ask_user_question tool to ask me exactly one question with id "checkpoint", question "Ready to continue?", header "Checkpoint", and options labeled "Yes" and "No". After I answer, reply with one short sentence acknowledging my answer and stop.'
|
|
const STEER = 'Interjection: include the word BANANA in your final reply.'
|
|
|
|
/** Concatenated assistant text deltas — the model-visible reply body. */
|
|
function assistantText(events: SessionEvent[]): string {
|
|
return events
|
|
.filter(e => e.type === 'assistant/chunk')
|
|
.map((e) => {
|
|
const chunk = (e as SessionEvent & { data: { chunk: { type: string; text?: string } } }).data.chunk
|
|
return chunk.type === 'text-delta' ? chunk.text ?? '' : ''
|
|
})
|
|
.join('')
|
|
}
|
|
|
|
/** Claimed user messages whose payload contains the exact scenario text. */
|
|
function claimedMessages(events: readonly SessionEvent[], text: string): SessionEvent<'user/message'>[] {
|
|
return events.filter((event): event is SessionEvent<'user/message'> =>
|
|
event.type === 'user/message' && JSON.stringify(event.data.content).includes(text))
|
|
}
|
|
|
|
describe('web e2e: mid-turn steering lands durably and visibly', () => {
|
|
let scaffold: WebScaffold
|
|
let browser: Browser
|
|
let page: Page
|
|
let tripwire: ReturnType<typeof watchConsole>
|
|
const sessionEvents: SessionEvent[] = []
|
|
|
|
beforeAll(async () => {
|
|
scaffold = await launchWebScaffold(MODE === 'record'
|
|
? {}
|
|
: { replayFixture: FIXTURE, paceMs: REPLAY_PACE_MS })
|
|
scaffold.ctx.on('session/event', (_session, event) => { sessionEvents.push(event) })
|
|
browser = await chromium.launch()
|
|
page = await newEnglishPage(browser)
|
|
tripwire = watchConsole(page)
|
|
await page.goto(scaffold.baseUrl, { waitUntil: 'load' })
|
|
await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
|
|
// Fresh world: connect a Workspace so the composer scenarios start live.
|
|
await connectFreshWorkspace(page, scaffold.workspaceCwd)
|
|
}, 120_000)
|
|
|
|
afterAll(async () => {
|
|
await browser?.close()
|
|
await scaffold?.close()
|
|
})
|
|
|
|
it('strictly steers one queued row; the interjection is logged, rendered, and obeyed', async () => {
|
|
onTestFailed(() => saveFailureShot(page, 'web-e2e-steering'))
|
|
if (MODE !== 'record') {
|
|
// The steer lands as a durable user/message, so the inventory holds
|
|
// both the opening prompt and the later same-turn steer.
|
|
expect(fixtureUserPrompts(await readFile(FIXTURE, 'utf8'))).toEqual([PROMPT, STEER])
|
|
}
|
|
const input = page.locator('textarea').first()
|
|
await input.waitFor({ timeout: 10_000 })
|
|
const settled = scaffold.whenTurnSettled(MODE === 'record' ? 180_000 : 30_000)
|
|
await input.fill(PROMPT)
|
|
await input.press('Enter')
|
|
|
|
// Enter remains the Queue gesture. The row action then atomically moves
|
|
// this exact occurrence into the current turn's steering outbox.
|
|
await input.fill(STEER)
|
|
await input.press('Enter')
|
|
const queued = page.getByText(STEER, { exact: true })
|
|
await queued.waitFor({ timeout: 10_000 })
|
|
const queuedRow = page.getByRole('listitem').filter({ hasText: STEER })
|
|
const steerButton = queuedRow.getByRole('button', { name: 'Steer queued message' })
|
|
await expect.poll(() => steerButton.isEnabled(), { timeout: 10_000 }).toBe(true)
|
|
await steerButton.click({ timeout: 10_000 })
|
|
const pendingSteering = page.locator('[data-pending-steering]').filter({ hasText: STEER })
|
|
// A timeout while the Queue row remains means strict steer lost to a
|
|
// closing window (`steer-unavailable`); inspect replay pacing first.
|
|
await pendingSteering.waitFor({ timeout: 10_000 })
|
|
|
|
// The blocked composer keeps steering pending long enough to observe the
|
|
// Host-authoritative mirror before the loop admits it durably.
|
|
const composer = page.locator('[data-question-key]')
|
|
await composer.waitFor({ timeout: MODE === 'record' ? 120_000 : 30_000 })
|
|
|
|
if (MODE !== 'record') {
|
|
expect(await page.getByText(STEER, { exact: true }).count()).toBe(1)
|
|
expect(await pendingSteering.count()).toBe(1)
|
|
expect(await page.getByRole('button', { name: 'Edit queued message' }).count()).toBe(0)
|
|
const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold.workspaceCwd)
|
|
await compareOrRefreshGolden(MID_EXPECTED, snapshot, MODE)
|
|
}
|
|
|
|
// Answer the composer; the tool result closes the step, the loop drains
|
|
// the steer as user/message, and the steered continuation runs the
|
|
// final model call.
|
|
await composer.getByRole('radio', { name: 'Yes' }).click()
|
|
await composer.getByRole('radio', { name: 'Yes' }).press('Enter')
|
|
await settled
|
|
|
|
if (MODE === 'record') {
|
|
const sessionId = await settled
|
|
await recordFixture(scaffold, sessionId, FIXTURE)
|
|
// Fixture honesty: a recording where the live model ignored the steer
|
|
// would replay as a vacuous scenario — reject it and re-record instead.
|
|
const recorded = parseSessionLog(await readFile(FIXTURE, 'utf8'))
|
|
expect(claimedMessages(recorded, STEER)).toHaveLength(1)
|
|
expect(assistantText(recorded)).toContain('BANANA')
|
|
return
|
|
}
|
|
|
|
// Durable: exactly one claimed user/message carrying the steering text.
|
|
const steerEvents = claimedMessages(sessionEvents, STEER)
|
|
expect(steerEvents).toHaveLength(1)
|
|
expect(JSON.stringify(steerEvents[0])).toContain('BANANA')
|
|
const turnEnds = sessionEvents.filter(e => e.type === 'turn/end')
|
|
expect(turnEnds).toHaveLength(1)
|
|
expect((turnEnds[0] as SessionEvent & { data: { reason: { kind: string } } }).data.reason.kind).toBe('completed')
|
|
|
|
// Visible: the plain steering bubble plus the reply that obeys it
|
|
// (steer text + final reply each contain the marker word).
|
|
await expect.poll(() => page.getByText(STEER, { exact: true }).count(), { timeout: 15_000 }).toBe(1)
|
|
expect(await pendingSteering.count()).toBe(0)
|
|
await expect.poll(() => page.getByText('BANANA', { exact: false }).count(), { timeout: 10_000 }).toBeGreaterThanOrEqual(2)
|
|
expect(await page.locator('[data-question-key]').count()).toBe(0)
|
|
// Settled golden: steer text between the question round trip and the
|
|
// obeying reply, composer takeover gone.
|
|
const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold.workspaceCwd)
|
|
await compareOrRefreshGolden(SETTLED_EXPECTED, snapshot, MODE)
|
|
expect(tripwire.pageErrors).toEqual([])
|
|
expect(tripwire.warnings).toEqual([])
|
|
}, 200_000)
|
|
|
|
it.skipIf(MODE === 'record')('keeps the fixture inventory closed', async () => {
|
|
await assertFixtureInventory(SNAPSHOT_DIR, ['session.jsonl', 'mid-steer.expected.md', 'settled.expected.md'])
|
|
})
|
|
})
|
|
|
|
describe('web e2e: composer shortcut steers directly', () => {
|
|
let scaffold: WebScaffold
|
|
let browser: Browser
|
|
let page: Page
|
|
let tripwire: ReturnType<typeof watchConsole>
|
|
const sessionEvents: SessionEvent[] = []
|
|
|
|
beforeAll(async () => {
|
|
scaffold = await launchWebScaffold({ replayFixture: FIXTURE, paceMs: REPLAY_PACE_MS })
|
|
scaffold.ctx.on('session/event', (_session, event) => { sessionEvents.push(event) })
|
|
browser = await chromium.launch()
|
|
page = await newEnglishPage(browser)
|
|
tripwire = watchConsole(page)
|
|
await page.goto(scaffold.baseUrl, { waitUntil: 'load' })
|
|
await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
|
|
await connectFreshWorkspace(page, scaffold.workspaceCwd)
|
|
}, 120_000)
|
|
|
|
afterAll(async () => {
|
|
await browser?.close()
|
|
await scaffold?.close()
|
|
})
|
|
|
|
it.skipIf(MODE === 'record')('uses Cmd+Enter without creating a Queue row', async () => {
|
|
onTestFailed(() => saveFailureShot(page, 'web-e2e-composer-steering'))
|
|
expect(fixtureUserPrompts(await readFile(FIXTURE, 'utf8'))).toEqual([PROMPT, STEER])
|
|
const input = page.locator('textarea').first()
|
|
await input.waitFor({ timeout: 10_000 })
|
|
const settled = scaffold.whenTurnSettled(30_000)
|
|
await input.fill(PROMPT)
|
|
await input.press('Enter')
|
|
await page.getByRole('button', { name: 'Stop generating' }).waitFor({ timeout: 10_000 })
|
|
|
|
await input.fill(STEER)
|
|
await input.press('Meta+Enter')
|
|
await expect.poll(() => input.inputValue(), { timeout: 5_000 }).toBe('')
|
|
expect(await page.locator('[data-queue-dock]').count()).toBe(0)
|
|
|
|
const composer = page.locator('[data-question-key]')
|
|
await composer.waitFor({ timeout: 30_000 })
|
|
const pendingSteering = page.locator('[data-pending-steering]').filter({ hasText: STEER })
|
|
await pendingSteering.waitFor({ timeout: 10_000 })
|
|
await composer.getByRole('radio', { name: 'Yes' }).click()
|
|
await composer.getByRole('radio', { name: 'Yes' }).press('Enter')
|
|
await settled
|
|
|
|
const steerEvents = claimedMessages(sessionEvents, STEER)
|
|
expect(steerEvents).toHaveLength(1)
|
|
await expect.poll(() => page.getByText(STEER, { exact: true }).count(), { timeout: 15_000 }).toBe(1)
|
|
expect(await pendingSteering.count()).toBe(0)
|
|
await expect.poll(() => page.getByText('BANANA', { exact: false }).count(), { timeout: 10_000 })
|
|
.toBeGreaterThanOrEqual(2)
|
|
expect(tripwire.pageErrors).toEqual([])
|
|
expect(tripwire.warnings).toEqual([])
|
|
}, 90_000)
|
|
})
|
|
|
|
describe('web e2e: composer shortcut follows the swapped busy behavior', () => {
|
|
let scaffold: WebScaffold
|
|
let browser: Browser
|
|
let page: Page
|
|
let tripwire: ReturnType<typeof watchConsole>
|
|
const sessionEvents: SessionEvent[] = []
|
|
|
|
beforeAll(async () => {
|
|
scaffold = await launchWebScaffold({ replayFixture: FIXTURE, paceMs: REPLAY_PACE_MS })
|
|
scaffold.ctx.on('session/event', (_session, event) => { sessionEvents.push(event) })
|
|
browser = await chromium.launch()
|
|
page = await newEnglishPage(browser)
|
|
tripwire = watchConsole(page)
|
|
await page.goto(scaffold.baseUrl, { waitUntil: 'load' })
|
|
await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
|
|
await connectFreshWorkspace(page, scaffold.workspaceCwd)
|
|
}, 120_000)
|
|
|
|
afterAll(async () => {
|
|
await browser?.close()
|
|
await scaffold?.close()
|
|
})
|
|
|
|
it.skipIf(MODE === 'record')('queues Cmd+Enter when plain Enter is configured to Steer', async () => {
|
|
onTestFailed(() => saveFailureShot(page, 'web-e2e-composer-swapped-shortcut'))
|
|
await page.getByRole('button', { name: 'Settings', exact: true }).click()
|
|
const dialog = page.getByRole('dialog', { name: 'Settings' })
|
|
await dialog.getByRole('button', { name: 'Queue' }).click()
|
|
await page.getByRole('menuitem', { name: 'Steer' }).click()
|
|
await dialog.getByRole('button', { name: 'Steer' }).waitFor({ timeout: 10_000 })
|
|
await page.keyboard.press('Escape')
|
|
|
|
const input = page.locator('textarea').first()
|
|
const settled = scaffold.whenTurnSettled(30_000)
|
|
await input.fill(PROMPT)
|
|
await input.press('Enter')
|
|
await page.getByRole('button', { name: 'Stop generating' }).waitFor({ timeout: 10_000 })
|
|
|
|
const queuedText = 'Queued by the complementary Cmd+Enter shortcut.'
|
|
await input.fill(queuedText)
|
|
await input.press('Meta+Enter')
|
|
const queuedRow = page.locator('[data-queue-dock]').getByRole('listitem').filter({ hasText: queuedText })
|
|
await queuedRow.getByText(queuedText, { exact: true }).waitFor({ timeout: 10_000 })
|
|
expect(await page.locator('[data-pending-steering]').filter({ hasText: queuedText }).count()).toBe(0)
|
|
expect(claimedMessages(sessionEvents, queuedText)).toHaveLength(0)
|
|
|
|
// Remove the asserted Queue row, then finish the recorded question turn
|
|
// so replay teardown still proves that every fixture call was consumed.
|
|
await queuedRow.getByRole('button', { name: 'Remove queued message' }).click()
|
|
const composer = page.locator('[data-question-key]')
|
|
await composer.waitFor({ timeout: 30_000 })
|
|
await composer.getByRole('radio', { name: 'Yes' }).click()
|
|
await composer.getByRole('radio', { name: 'Yes' }).press('Enter')
|
|
await settled
|
|
expect(tripwire.pageErrors).toEqual([])
|
|
expect(tripwire.warnings).toEqual([])
|
|
}, 90_000)
|
|
})
|