mirror of
https://github.com/deepseek-ai/deepseek-harness
synced 2026-08-15 21:04:50 +00:00
test(web): cover empty-draft queue flush with a keyless replay scenario
Also pin the queue-item-not-found convergence arm through the hub and document the repeated-trigger contract on steerQueue.
This commit is contained in:
36
apps/web/tests/snapshots/steer-all/mid-steer.expected.md
Normal file
36
apps/web/tests/snapshots/steer-all/mid-steer.expected.md
Normal file
@@ -0,0 +1,36 @@
|
||||
- banner:
|
||||
- navigation "Session hierarchy":
|
||||
- button "Use the ask_user_question tool to" [disabled]
|
||||
- tablist:
|
||||
- tab "Chat" [selected]
|
||||
- tab "Trajectory"
|
||||
- text: Use the ask_user_question tool to ask me exactly one question with id "checkpoint", question "Ready to continue?", header "Checkpoint", and options labeled "Yes" and "No". After I answer, reply with one short sentence acknowledging my answer and stop. {{clock}}
|
||||
- button "Copy":
|
||||
- img
|
||||
- button "Branch into a new conversation" [disabled]:
|
||||
- img
|
||||
- text: Available only on the last message of a completed turn
|
||||
- button "Context injection @deepseek-ai/dsh-system-prompt":
|
||||
- img
|
||||
- img
|
||||
- text: Context injection @deepseek-ai/dsh-system-prompt
|
||||
- text: Running
|
||||
- button "Think":
|
||||
- img
|
||||
- img
|
||||
- text: Think
|
||||
- status: Deep diving...
|
||||
- text: "Interjection Interjection: include the word BANANA in your final reply."
|
||||
- button "Copy":
|
||||
- img
|
||||
- text: "Interjection Interjection: include the word ORANGE in your final reply."
|
||||
- button "Copy":
|
||||
- img
|
||||
- textbox "Message the agent"
|
||||
- button "Commands":
|
||||
- img
|
||||
- 'button "Access mode, current: Workspace Write"': Workspace Write
|
||||
- button "Select model, current DeepSeek-V4-Flash":
|
||||
- text: DeepSeek-V4-Flash
|
||||
- img
|
||||
- button "Stop generating"
|
||||
47
apps/web/tests/snapshots/steer-all/replay.override.json
Normal file
47
apps/web/tests/snapshots/steer-all/replay.override.json
Normal file
@@ -0,0 +1,47 @@
|
||||
[
|
||||
{
|
||||
"kind": "chunks",
|
||||
"chunks": [
|
||||
{ "type": "block-start", "index": 0, "blockType": "reasoning" },
|
||||
{ "type": "reasoning-delta", "index": 0, "text": "The user wants me to ask them a checkpoint question first, then continue with whatever they interject. Let me do exactly that." },
|
||||
{ "type": "block-start", "index": 1, "blockType": "tool-call" },
|
||||
{
|
||||
"type": "tool-call-delta",
|
||||
"index": 1,
|
||||
"id": "call_00_steer_all",
|
||||
"name": "ask_user_question",
|
||||
"argumentsDelta": "{\"questions\": [{\"id\": \"checkpoint\", \"question\": \"Ready to continue?\", \"header\": \"Checkpoint\", \"options\": [{\"label\": \"Yes\"}, {\"label\": \"No\"}]}]}"
|
||||
},
|
||||
{
|
||||
"type": "block-end",
|
||||
"index": 0,
|
||||
"block": {
|
||||
"type": "reasoning",
|
||||
"text": "The user wants me to ask them a checkpoint question first, then continue with whatever they interject. Let me do exactly that."
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "block-end",
|
||||
"index": 1,
|
||||
"block": {
|
||||
"type": "tool-call",
|
||||
"id": "call_00_steer_all",
|
||||
"name": "ask_user_question",
|
||||
"arguments": "{\"questions\": [{\"id\": \"checkpoint\", \"question\": \"Ready to continue?\", \"header\": \"Checkpoint\", \"options\": [{\"label\": \"Yes\"}, {\"label\": \"No\"}]}]}"
|
||||
}
|
||||
},
|
||||
{ "type": "usage", "usage": { "inputTokens": 10, "outputTokens": 10, "cacheReadTokens": 0, "reasoningTokens": 0 } },
|
||||
{ "type": "finish", "reason": { "kind": "tool-calls" } }
|
||||
]
|
||||
},
|
||||
{
|
||||
"kind": "chunks",
|
||||
"chunks": [
|
||||
{ "type": "block-start", "index": 0, "blockType": "text" },
|
||||
{ "type": "text-delta", "index": 0, "text": "Got it: BANANA and ORANGE." },
|
||||
{ "type": "block-end", "index": 0, "block": { "type": "text", "text": "Got it: BANANA and ORANGE." } },
|
||||
{ "type": "usage", "usage": { "inputTokens": 10, "outputTokens": 10, "cacheReadTokens": 0, "reasoningTokens": 0 } },
|
||||
{ "type": "finish", "reason": { "kind": "stop" } }
|
||||
]
|
||||
}
|
||||
]
|
||||
51
apps/web/tests/snapshots/steer-all/settled.expected.md
Normal file
51
apps/web/tests/snapshots/steer-all/settled.expected.md
Normal file
@@ -0,0 +1,51 @@
|
||||
- banner:
|
||||
- navigation "Session hierarchy":
|
||||
- button "Use the ask_user_question tool to" [disabled]
|
||||
- tablist:
|
||||
- tab "Chat" [selected]
|
||||
- tab "Trajectory"
|
||||
- text: Use the ask_user_question tool to ask me exactly one question with id "checkpoint", question "Ready to continue?", header "Checkpoint", and options labeled "Yes" and "No". After I answer, reply with one short sentence acknowledging my answer and stop. {{clock}}
|
||||
- button "Copy":
|
||||
- img
|
||||
- button "Branch into a new conversation" [disabled]:
|
||||
- img
|
||||
- text: Available only on the last message of a completed turn
|
||||
- button "Context injection @deepseek-ai/dsh-system-prompt":
|
||||
- img
|
||||
- img
|
||||
- text: Context injection @deepseek-ai/dsh-system-prompt
|
||||
- button "Think The user wants me to ask them a checkpoint question first, then continue with whatever they interject. Let me do exactly that.":
|
||||
- img
|
||||
- img
|
||||
- text: Think The user wants me to ask them a checkpoint question first, then continue with whatever they interject. Let me do exactly that.
|
||||
- button "Ask question 1/1 answered":
|
||||
- img
|
||||
- img
|
||||
- text: Ask question 1/1 answered
|
||||
- text: "Interjection Interjection: include the word BANANA in your final reply. {{clock}}"
|
||||
- button "Copy":
|
||||
- img
|
||||
- button "Branch into a new conversation" [disabled]:
|
||||
- img
|
||||
- text: "Available only on the last message of a completed turn Interjection Interjection: include the word ORANGE in your final reply. {{clock}}"
|
||||
- button "Copy":
|
||||
- img
|
||||
- button "Branch into a new conversation" [disabled]:
|
||||
- img
|
||||
- text: Available only on the last message of a completed turn
|
||||
- paragraph: "Got it: BANANA and ORANGE."
|
||||
- button "Copy":
|
||||
- img
|
||||
- button "Branch into a new conversation":
|
||||
- img
|
||||
- text: {{clock}} Ran for {{duration}} TTFT {{duration}} {{throughput}} tok/s
|
||||
- textbox "Message the agent"
|
||||
- button "Commands":
|
||||
- img
|
||||
- 'button "Access mode, current: Workspace Write"': Workspace Write
|
||||
- button "Select model, current DeepSeek-V4-Flash":
|
||||
- text: DeepSeek-V4-Flash
|
||||
- img
|
||||
- button "0% of context used"
|
||||
- button "Send message" [disabled]
|
||||
- text: 1 turns · 2 steps LLM {{duration}} · Tool call {{duration}} TTFT avg {{duration}} · {{throughput}} tok/s Cache hit 0% Input 20 tok · Output 20 tok
|
||||
@@ -34,6 +34,18 @@ const REPLAY_PACE_MS = 100
|
||||
const PROMPT = 'Use the ask_user_question tool to ask me exactly one question with id "checkpoint", question "Ready to continue?", header "Checkpoint", and options labeled "Yes" and "No". After I answer, reply with one short sentence acknowledging my answer and stop.'
|
||||
const STEER = 'Interjection: include the word BANANA in your final reply.'
|
||||
|
||||
// Empty-draft flush scenario: an override-only fixture. The whole-script
|
||||
// replacement answers both model calls of a FRESH session (no recorded
|
||||
// session.jsonl exists — call 0 keeps the turn open with a question-tool
|
||||
// call, call 1 is the reply after both steerings drain).
|
||||
const STEER_ALL_DIR = fileURLToPath(new URL('./snapshots/steer-all', import.meta.url))
|
||||
const STEER_ALL_FIXTURE = join(STEER_ALL_DIR, 'session.jsonl')
|
||||
const STEER_ALL_OVERRIDE = join(STEER_ALL_DIR, 'replay.override.json')
|
||||
const STEER_ALL_MID = join(STEER_ALL_DIR, 'mid-steer.expected.md')
|
||||
const STEER_ALL_SETTLED = join(STEER_ALL_DIR, 'settled.expected.md')
|
||||
const STEER_ONE = 'Interjection: include the word BANANA in your final reply.'
|
||||
const STEER_TWO = 'Interjection: include the word ORANGE in your final reply.'
|
||||
|
||||
/** Concatenated assistant text deltas — the model-visible reply body. */
|
||||
function assistantText(events: SessionEvent[]): string {
|
||||
return events
|
||||
@@ -278,3 +290,98 @@ describe('web e2e: composer shortcut follows the swapped busy behavior', () => {
|
||||
expect(tripwire.warnings).toEqual([])
|
||||
}, 90_000)
|
||||
})
|
||||
|
||||
describe('web e2e: empty-draft Cmd+Enter steers the whole queue', () => {
|
||||
let scaffold: WebScaffold
|
||||
let browser: Browser
|
||||
let page: Page
|
||||
let tripwire: ReturnType<typeof watchConsole>
|
||||
const sessionEvents: SessionEvent[] = []
|
||||
|
||||
beforeAll(async () => {
|
||||
// The scenario boots a fresh session against the override-only fixture;
|
||||
// the replay.override.json sidecar replaces the derived script, so the
|
||||
// (deliberately absent) session.jsonl is never read.
|
||||
scaffold = await launchWebScaffold({
|
||||
replayFixture: STEER_ALL_FIXTURE,
|
||||
replayOverride: STEER_ALL_OVERRIDE,
|
||||
paceMs: REPLAY_PACE_MS,
|
||||
})
|
||||
scaffold.ctx.on('session/event', (_session, event) => { sessionEvents.push(event) })
|
||||
browser = await chromium.launch()
|
||||
page = await newEnglishPage(browser)
|
||||
tripwire = watchConsole(page)
|
||||
await page.goto(scaffold.baseUrl, { waitUntil: 'load' })
|
||||
await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
|
||||
await connectFreshWorkspace(page, scaffold.workspaceCwd)
|
||||
}, 120_000)
|
||||
|
||||
afterAll(async () => {
|
||||
await browser?.close()
|
||||
await scaffold?.close()
|
||||
})
|
||||
|
||||
it.skipIf(MODE === 'record')('queues two messages, then flushes both with an empty-draft Cmd+Enter', async () => {
|
||||
onTestFailed(() => saveFailureShot(page, 'web-e2e-steer-all'))
|
||||
const input = page.locator('textarea').first()
|
||||
await input.waitFor({ timeout: 10_000 })
|
||||
const settled = scaffold.whenTurnSettled(30_000)
|
||||
|
||||
// Call 0 streams a question-tool call; the fills must land inside the
|
||||
// first replay window, before the question composer replaces the textarea.
|
||||
await input.fill(PROMPT)
|
||||
await input.press('Enter')
|
||||
await input.fill(STEER_ONE)
|
||||
await input.press('Enter')
|
||||
await input.fill(STEER_TWO)
|
||||
await input.press('Enter')
|
||||
const dock = page.locator('[data-queue-dock]')
|
||||
// Both messages queued: the two-row dock shows a collapsed count header,
|
||||
// and Playwright text matching skips the hidden rows — expand the list,
|
||||
// then assert each row's content.
|
||||
await dock.getByText('2 queued messages').waitFor({ timeout: 10_000 })
|
||||
await dock.getByRole('button').click()
|
||||
await dock.getByText(STEER_ONE, { exact: true }).waitFor({ timeout: 10_000 })
|
||||
await dock.getByText(STEER_TWO, { exact: true }).waitFor({ timeout: 10_000 })
|
||||
expect(await page.locator('[data-pending-steering]').count()).toBe(0)
|
||||
|
||||
// Empty draft + Cmd+Enter: both queued rows steer in FIFO order, the dock
|
||||
// empties, and the pending steering renders at the conversation tail.
|
||||
await input.press('Meta+Enter')
|
||||
await expect.poll(
|
||||
() => page.locator('[data-pending-steering]').filter({ hasText: /BANANA|ORANGE/ }).count(),
|
||||
{ timeout: 10_000 },
|
||||
).toBe(2)
|
||||
expect(await page.locator('[data-queue-dock]').count()).toBe(0)
|
||||
const mid = await captureStableAria(page, '[class*="centerCol"]', scaffold.workspaceCwd)
|
||||
await compareOrRefreshGolden(STEER_ALL_MID, mid, MODE)
|
||||
|
||||
// Answer the question; the step closes, the loop drains both steerings
|
||||
// into one next-step request, and the final reply obeys both markers.
|
||||
const composer = page.locator('[data-question-key]')
|
||||
await composer.waitFor({ timeout: 30_000 })
|
||||
await composer.getByRole('radio', { name: 'Yes' }).click()
|
||||
await composer.getByRole('radio', { name: 'Yes' }).press('Enter')
|
||||
await settled
|
||||
|
||||
const first = claimedMessages(sessionEvents, STEER_ONE)
|
||||
const second = claimedMessages(sessionEvents, STEER_TWO)
|
||||
expect(first).toHaveLength(1)
|
||||
expect(second).toHaveLength(1)
|
||||
expect(assistantText(sessionEvents)).toContain('BANANA')
|
||||
expect(assistantText(sessionEvents)).toContain('ORANGE')
|
||||
await expect.poll(() => page.getByText(STEER_ONE, { exact: true }).count(), { timeout: 15_000 }).toBe(1)
|
||||
await expect.poll(() => page.getByText(STEER_TWO, { exact: true }).count(), { timeout: 15_000 }).toBe(1)
|
||||
expect(await page.locator('[data-pending-steering]').count()).toBe(0)
|
||||
const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold.workspaceCwd)
|
||||
await compareOrRefreshGolden(STEER_ALL_SETTLED, snapshot, MODE)
|
||||
expect(tripwire.pageErrors).toEqual([])
|
||||
expect(tripwire.warnings).toEqual([])
|
||||
}, 200_000)
|
||||
|
||||
it.skipIf(MODE === 'record')('keeps the fixture inventory closed', async () => {
|
||||
await assertFixtureInventory(STEER_ALL_DIR, [
|
||||
'replay.override.json', 'mid-steer.expected.md', 'settled.expected.md',
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
@@ -152,7 +152,10 @@ export class InputHub implements InputService {
|
||||
* order — the same strict-steer operation as the queue dock's per-row
|
||||
* button. A turn closing mid-way (`steer-unavailable`) or a row already
|
||||
* claimed by the agent (`queue-item-not-found`) converges silently, while a
|
||||
* genuine failure surfaces as one composer notice.
|
||||
* genuine failure surfaces as one composer notice. Repeated triggers
|
||||
* (e.g. two rapid empty-draft chords) rely on that `queue-item-not-found`
|
||||
* convergence: the snapshot may still list a row the host already steered,
|
||||
* and the duplicate strict steer is a silent no-op.
|
||||
* @param session - the addressed host session.
|
||||
* @param shell - the resident shell (notice outlet).
|
||||
*/
|
||||
|
||||
@@ -125,12 +125,25 @@ describe('InputHub queue steering (empty-draft accelerated Enter)', () => {
|
||||
await b.runtime.sessions.updateSnapshot('s1', (draft) => {
|
||||
draft.queue = [row('q-1'), row('q-2')]
|
||||
})
|
||||
// The turn closes before the second row: the flush stops, silently.
|
||||
b.updateQueue.mockResolvedValueOnce({
|
||||
ok: false, error: { code: 'steer-unavailable', message: 'closed', details: {} },
|
||||
} as never)
|
||||
b.shell.steerQueue()
|
||||
await vi.waitFor(() => { expect(b.updateQueue).toHaveBeenCalledTimes(1) })
|
||||
expect(b.shell.notices.getSnapshot()).toBeNull()
|
||||
|
||||
// A row the host already claimed (e.g. a repeated empty-draft chord):
|
||||
// the duplicate strict steer is a silent no-op.
|
||||
await b.runtime.sessions.updateSnapshot('s1', (draft) => {
|
||||
draft.queue = [row('q-3')]
|
||||
})
|
||||
b.updateQueue.mockResolvedValueOnce({
|
||||
ok: false, error: { code: 'queue-item-not-found', message: 'claimed', details: {} },
|
||||
} as never)
|
||||
b.shell.steerQueue()
|
||||
await vi.waitFor(() => { expect(b.updateQueue).toHaveBeenCalledTimes(2) })
|
||||
expect(b.shell.notices.getSnapshot()).toBeNull()
|
||||
await b.runtime.dispose()
|
||||
})
|
||||
|
||||
|
||||
Reference in New Issue
Block a user