mirror of
https://github.com/deepseek-ai/deepseek-harness
synced 2026-08-15 21:04:50 +00:00
Merge remote-tracking branch 'origin/master' into feat/read-image-context
This commit is contained in:
@@ -176,8 +176,10 @@ describe('web e2e: agent-preset authoring is a host-side copy', () => {
|
||||
|
||||
await expect.poll(async () => dialog.getByText('我的模式').count(), { timeout: 10_000 }).toBe(0)
|
||||
expect(existsSync(join(userRoot, 'my-agent'))).toBe(false)
|
||||
// Custom group gone with its only member; the shipped set stands.
|
||||
expect(await dialog.getByRole('heading', { name: '自定义' }).count()).toBe(0)
|
||||
// The custom group outlives its only member: the heading stays with the
|
||||
// creator entry so the place to author a preset never disappears.
|
||||
expect(await dialog.getByRole('heading', { name: '自定义' }).count()).toBe(1)
|
||||
expect(await dialog.getByRole('button', { name: '用「创造模式」创作自定义预设' }).count()).toBe(1)
|
||||
expect(await dialog.getByText('标准模式').count()).toBeGreaterThan(0)
|
||||
}, 60_000)
|
||||
|
||||
|
||||
101
apps/web/tests/feedback-command.e2e.ts
Normal file
101
apps/web/tests/feedback-command.e2e.ts
Normal file
@@ -0,0 +1,101 @@
|
||||
// Keyless assembled-browser coverage for the /feedback command over the
|
||||
// shipped Web bundles and the real host wire. The command plane settles
|
||||
// without a model turn: the host appends the log-only command/run +
|
||||
// feedback/record + command/done lifecycle, and the transcript renders the
|
||||
// acknowledgement — the recorded session id plus the session-sharing
|
||||
// disclosure — as a persistent command row. The scaffold mounts the shipped
|
||||
// telemetry row in FULL mode against a local dead endpoint (no record leaves
|
||||
// the process), so the golden pins the shipped default sentence
|
||||
// `Session sharing is enabled.`; the per-status sentences are pinned by the
|
||||
// package and OTel unit tests.
|
||||
import { readFile } from 'node:fs/promises'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { join } from 'node:path'
|
||||
import type { Browser, Page } from 'playwright'
|
||||
import { chromium } from 'playwright'
|
||||
import { afterAll, beforeAll, describe, expect, it, onTestFailed } from 'vitest'
|
||||
import {
|
||||
assertFixtureInventory, captureStableAria, compareOrRefreshGolden, fixtureUserPrompts,
|
||||
launchWebScaffold, recordFixture, watchConsole, webSnapshotMode, type WebScaffold,
|
||||
} from './scaffold.ts'
|
||||
import { connectFreshWorkspace, newEnglishPage, saveFailureShot } from './support.ts'
|
||||
|
||||
const SNAPSHOT_DIR = fileURLToPath(new URL('./snapshots/feedback-command', import.meta.url))
|
||||
const FIXTURE = join(SNAPSHOT_DIR, 'session.jsonl')
|
||||
const ACK_EXPECTED = join(SNAPSHOT_DIR, 'ack.expected.md')
|
||||
const MODE = webSnapshotMode()
|
||||
// Discard port: loopback listener never binds, so FULL telemetry discloses
|
||||
// the shipped default policy without any record reaching a collector.
|
||||
const TELEMETRY_URL = 'http://127.0.0.1:9/v1/logs'
|
||||
|
||||
const PROMPT = 'Reply with the single word LIGHTHOUSE and stop.'
|
||||
|
||||
describe('web e2e: /feedback command acknowledgement', () => {
|
||||
let scaffold: WebScaffold
|
||||
let browser: Browser
|
||||
let page: Page
|
||||
let tripwire: ReturnType<typeof watchConsole>
|
||||
|
||||
beforeAll(async () => {
|
||||
scaffold = await launchWebScaffold({
|
||||
telemetryUrl: TELEMETRY_URL,
|
||||
...(MODE === 'record' ? {} : { replayFixture: FIXTURE }),
|
||||
})
|
||||
browser = await chromium.launch()
|
||||
page = await newEnglishPage(browser)
|
||||
tripwire = watchConsole(page)
|
||||
await page.goto(scaffold.baseUrl, { waitUntil: 'load' })
|
||||
await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
|
||||
// Fresh world: connecting a workspace births the blank session whose
|
||||
// live composer accepts the slash line.
|
||||
await connectFreshWorkspace(page, scaffold.workspaceCwd)
|
||||
}, 120_000)
|
||||
|
||||
afterAll(async () => {
|
||||
await browser?.close()
|
||||
await scaffold?.close()
|
||||
})
|
||||
|
||||
it('drives the recorded prompt to a settled turn (all modes)', async () => {
|
||||
onTestFailed(() => saveFailureShot(page, 'web-e2e-feedback-drive'))
|
||||
if (MODE !== 'record') {
|
||||
// Drift guard: the committed fixture must carry exactly the drive prompt.
|
||||
expect(fixtureUserPrompts(await readFile(FIXTURE, 'utf8'))).toEqual([PROMPT])
|
||||
}
|
||||
const input = page.locator('textarea').first()
|
||||
await input.waitFor({ timeout: 10_000 })
|
||||
// Arm the turn-boundary waiter BEFORE sending, so a burst replay cannot
|
||||
// miss the turn/end that settles the recorded turn.
|
||||
const settled = scaffold.whenTurnSettled()
|
||||
await input.fill(PROMPT)
|
||||
await input.press('Enter')
|
||||
const sessionId = await settled
|
||||
if (MODE === 'record') {
|
||||
await recordFixture(scaffold, sessionId, FIXTURE)
|
||||
}
|
||||
}, 60_000)
|
||||
|
||||
it.skipIf(MODE === 'record')('records feedback and renders the acknowledgement with session id and sharing status', async () => {
|
||||
onTestFailed(() => saveFailureShot(page, 'web-e2e-feedback-command'))
|
||||
// The drive test settled the recorded turn: the transcript is active (a
|
||||
// command row does not render while a fresh session is still blank) and
|
||||
// the replayed reply is on screen.
|
||||
await page.getByText('LIGHTHOUSE', { exact: true }).waitFor({ timeout: 15_000 })
|
||||
const input = page.locator('textarea').first()
|
||||
await input.fill('/feedback the diff view is unreadable')
|
||||
await input.press('Enter')
|
||||
// The command plane settles without a model turn: the ack row names the
|
||||
// recorded session and the mounted FULL backend's disclosure.
|
||||
await page.getByText(/Feedback recorded for session/).waitFor({ timeout: 10_000 })
|
||||
expect(await page.getByText(/Session sharing is enabled/).count()).toBe(1)
|
||||
const snapshot = await captureStableAria(page, '[class*="centerCol"]', scaffold.workspaceCwd)
|
||||
await compareOrRefreshGolden(ACK_EXPECTED, snapshot, MODE)
|
||||
|
||||
expect(tripwire.pageErrors).toEqual([])
|
||||
expect(tripwire.warnings).toEqual([])
|
||||
}, 60_000)
|
||||
|
||||
it.skipIf(MODE === 'record')('keeps the fixture inventory closed', async () => {
|
||||
await assertFixtureInventory(SNAPSHOT_DIR, ['session.jsonl', 'ack.expected.md'])
|
||||
})
|
||||
})
|
||||
@@ -128,6 +128,7 @@ describe('web e2e: message IconActions and clocks on settled history', () => {
|
||||
onTestFailed(() => saveFailureShot(page, 'web-e2e-message-actions-aria'))
|
||||
await page.getByRole('button', { name: /^Select model, current/ })
|
||||
.waitFor({ timeout: 10_000 })
|
||||
await page.getByText(/Cache hit \d+%/u).first().waitFor({ timeout: 10_000 })
|
||||
// Keep a footer focused so opacity-hidden actions stay in the a11y tree
|
||||
// as an active/focused control during the capture.
|
||||
await page.getByRole('button', { name: 'Copy' }).first().focus()
|
||||
|
||||
@@ -245,6 +245,13 @@ export interface LaunchOptions {
|
||||
}
|
||||
/** Leave the current welcome notice unacknowledged; ordinary scenarios publish it as complete before browser boot. */
|
||||
welcomeNoticePending?: boolean
|
||||
/**
|
||||
* Mount the shipped telemetry row in FULL mode against this exporter URL
|
||||
* instead of disabling it. Used to pin a real backend disclosure in
|
||||
* assembled coverage; point the URL at a local dead endpoint so no record
|
||||
* leaves the process.
|
||||
*/
|
||||
telemetryUrl?: string
|
||||
/**
|
||||
* Browse through a trusted non-loopback hostname that the browser resolves
|
||||
* to loopback (for example `*.localhost`). The test server stays bound to
|
||||
@@ -334,6 +341,7 @@ export async function launchWebScaffold(options: LaunchOptions = {}): Promise<We
|
||||
} catch (error) {
|
||||
const failures: unknown[] = [error]
|
||||
await rm(workspaceCwd, { recursive: true, force: true }).catch((cleanupError: unknown) => failures.push(cleanupError))
|
||||
restoreSkillRootEnvironment()
|
||||
if (failures.length > 1) throw new AggregateError(failures, 'web scaffold temp-root setup failed')
|
||||
throw error
|
||||
}
|
||||
@@ -395,8 +403,11 @@ export async function launchWebScaffold(options: LaunchOptions = {}): Promise<We
|
||||
{ id: 'session-title-llm', disabled: true },
|
||||
// Fixture sessions must never leave the process: the shipped row defaults
|
||||
// to the production OTLP endpoint (or whatever DSH_TELEMETRY_OTLP_URL
|
||||
// names in the ambient environment).
|
||||
{ id: 'telemetry-otel', disabled: true },
|
||||
// names in the ambient environment). A scenario that pins a real backend
|
||||
// disclosure passes a local dead endpoint instead of disabling the row.
|
||||
options.telemetryUrl === undefined
|
||||
? { id: 'telemetry-otel', disabled: true }
|
||||
: { id: 'telemetry-otel', config: { exporter: { url: options.telemetryUrl }, shutdownTimeoutMillis: 1_000 } },
|
||||
{
|
||||
id: 'webserver',
|
||||
config: { host: '127.0.0.1', port: 0 },
|
||||
|
||||
@@ -468,9 +468,9 @@ describe('web e2e: seeded history renders through cold resume', () => {
|
||||
if (done?.type !== 'command/done') throw new Error('feedback command did not settle')
|
||||
const [sessionLine, userLine, extraLine] = done.data.text?.split('\n') ?? []
|
||||
expect(sessionLine).toBe(`Feedback recorded for session ${SEED_ID}`)
|
||||
expect(userLine).toMatch(/^User: [0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i)
|
||||
expect(userLine).toMatch(/^User: [0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\./i)
|
||||
expect(extraLine).toBeUndefined()
|
||||
const userId = userLine?.slice('User: '.length)
|
||||
const userId = userLine?.match(/^User: ([0-9a-f-]+)/i)?.[1]
|
||||
if (userId === undefined) throw new Error('feedback command omitted the user id')
|
||||
|
||||
const snapshot = (await captureStableAria(page, '[class*="centerCol"]', scaffold.workspaceCwd))
|
||||
|
||||
@@ -58,6 +58,7 @@
|
||||
- 'button "复制: 创造模式"':
|
||||
- img
|
||||
- text: 复制
|
||||
- heading "自定义" [level=3]
|
||||
- button "用「创造模式」创作自定义预设":
|
||||
- img
|
||||
- text: 用「创造模式」创作自定义预设
|
||||
|
||||
39
apps/web/tests/snapshots/feedback-command/ack.expected.md
Normal file
39
apps/web/tests/snapshots/feedback-command/ack.expected.md
Normal file
@@ -0,0 +1,39 @@
|
||||
- banner:
|
||||
- navigation "Session hierarchy":
|
||||
- button "Reply with the single word" [disabled]
|
||||
- img
|
||||
- text: Standard mode
|
||||
- tablist:
|
||||
- tab "Chat" [selected]
|
||||
- tab "Trajectory"
|
||||
- text: Reply with the single word LIGHTHOUSE and stop. {{clock}}
|
||||
- button "Copy":
|
||||
- img
|
||||
- button "Context injection @deepseek-ai/dsh-system-prompt":
|
||||
- img
|
||||
- img
|
||||
- text: Context injection @deepseek-ai/dsh-system-prompt
|
||||
- button "Think The user wants me to reply with a single word. Let me comply.":
|
||||
- img
|
||||
- img
|
||||
- text: Think The user wants me to reply with a single word. Let me comply.
|
||||
- paragraph: LIGHTHOUSE
|
||||
- button "Copy":
|
||||
- img
|
||||
- button "Branch into a new conversation":
|
||||
- img
|
||||
- text: {{clock}} Ran for {{duration}} TTFT {{duration}} {{throughput}} tok/s
|
||||
- 'button "feedback Feedback recorded for session session-{{uuid}} User: {{uuid}}. Session sharing is enabled."':
|
||||
- img
|
||||
- img
|
||||
- text: "feedback Feedback recorded for session session-{{uuid}} User: {{uuid}}. Session sharing is enabled."
|
||||
- textbox "Message the agent"
|
||||
- button "Commands":
|
||||
- img
|
||||
- 'button "Access mode, current: Workspace Write"': Workspace Write
|
||||
- button "Select model, current DeepSeek-V4-Flash":
|
||||
- text: DeepSeek-V4-Flash
|
||||
- img
|
||||
- button "6% of context used"
|
||||
- button "Send message" [disabled]
|
||||
- text: 1 turns · 1 steps LLM {{duration}} TTFT avg {{duration}} · {{throughput}} tok/s Cache hit 99% Input 7.8K tok · Output 21 tok
|
||||
17
apps/web/tests/snapshots/feedback-command/session.jsonl
Normal file
17
apps/web/tests/snapshots/feedback-command/session.jsonl
Normal file
@@ -0,0 +1,17 @@
|
||||
{"type":"session","version":0,"id":"{{sessionId}}","createdAt":1785015039278,"cwd":"{{cwd}}/workspace"}
|
||||
{"type":"turn/start","seq":0,"time":1785015039291,"data":{"turn":1,"trigger":{"kind":"message","source":{"kind":"user","rpcId":"{{rpcId}}"}}}}
|
||||
{"type":"user/message","seq":1,"time":1785015039292,"data":{"content":[{"type":"text","text":"Reply with the single word LIGHTHOUSE and stop."}],"source":{"kind":"user","rpcId":"{{rpcId}}"}},"surfaceOp":"append"}
|
||||
{"type":"session/title","seq":2,"time":1785015039294,"data":{"title":"Reply with the single word","messageSeqs":[1],"source":{"kind":"fallback"}}}
|
||||
{"type":"step/start","seq":3,"time":1785015039362,"data":{"turn":1,"step":1}}
|
||||
{"type":"request/header","seq":4,"time":1785015039363,"data":{"header":{"config":{"provider":"deepseek-official","model":"deepseek-v4-flash"},"system":"{{system}}","tools":"{{tools}}","messagePrefix":["{{messagePrefix}}"]},"reason":"initial"}}
|
||||
{"type":"assistant/chunk","seq":5,"time":1785015039930,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":0,"blockType":"reasoning"}}}
|
||||
{"type":"reasoning-chunks","seq0":6,"time0":1785015039930,"data":{"turn":1,"step":1,"index":0,"dt":[162,28,1,0,0,46,1,0,0,0,11,0,0,30],"texts":["The"," user"," wants"," me"," to"," reply"," with"," a"," single"," word","."," Let"," me"," comply","."]}}
|
||||
{"type":"assistant/chunk","seq":21,"time":1785015040209,"data":{"turn":1,"step":1,"chunk":{"type":"block-start","index":1,"blockType":"text"}}}
|
||||
{"type":"text-chunks","seq0":22,"time0":1785015040209,"data":{"turn":1,"step":1,"index":1,"dt":[1,0,30,1],"texts":["L","IGH","TH","O","USE"]}}
|
||||
{"type":"assistant/chunk","seq":27,"time":1785015040241,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":0,"block":{"type":"reasoning","text":"The user wants me to reply with a single word. Let me comply."}}}}
|
||||
{"type":"assistant/chunk","seq":28,"time":1785015040242,"data":{"turn":1,"step":1,"chunk":{"type":"block-end","index":1,"block":{"type":"text","text":"LIGHTHOUSE"}}}}
|
||||
{"type":"assistant/chunk","seq":29,"time":1785015040242,"data":{"turn":1,"step":1,"chunk":{"type":"usage","usage":{"inputTokens":109,"outputTokens":21,"cacheReadTokens":7680,"reasoningTokens":15}}}}
|
||||
{"type":"assistant/chunk","seq":30,"time":1785015040242,"data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"stop"}}}}
|
||||
{"type":"assistant/message","seq":31,"time":1785015040244,"data":{"turn":1,"step":1,"content":[{"type":"reasoning","text":"The user wants me to reply with a single word. Let me comply."},{"type":"text","text":"LIGHTHOUSE"}],"provenance":{"provider":"deepseek-official","model":"deepseek-v4-flash"},"usage":{"inputTokens":109,"outputTokens":21,"cacheReadTokens":7680,"reasoningTokens":15}},"sourceEventSeqs":[5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30],"surfaceOp":"append"}
|
||||
{"type":"step/end","seq":32,"time":1785015040246,"data":{"turn":1,"step":1}}
|
||||
{"type":"turn/end","seq":33,"time":1785015040247,"data":{"turn":1,"reason":{"kind":"completed"}}}
|
||||
@@ -38,10 +38,10 @@
|
||||
- text: Context injection AGENTS.md
|
||||
- img
|
||||
- text: permission preset read-only
|
||||
- 'button "feedback Feedback recorded for session {{seededId}} User: {{uuid}}" [expanded]':
|
||||
- 'button "feedback Feedback recorded for session {{seededId}} User: {{uuid}}. Session sharing is not configured." [expanded]':
|
||||
- img
|
||||
- text: "feedback Feedback recorded for session {{seededId}} User: {{uuid}}"
|
||||
- text: "Feedback recorded for session {{seededId}} User: {{uuid}}"
|
||||
- text: "feedback Feedback recorded for session {{seededId}} User: {{uuid}}. Session sharing is not configured."
|
||||
- text: "Feedback recorded for session {{seededId}} User: {{uuid}}. Session sharing is not configured."
|
||||
- textbox "Message the agent"
|
||||
- button "Commands":
|
||||
- img
|
||||
|
||||
@@ -395,7 +395,8 @@ describe('web e2e: persisted subagent conversation and human continuation', () =
|
||||
expect([
|
||||
Math.round(clickAreaBox!.x - treeBox!.x),
|
||||
Math.round(treeBox!.x + treeBox!.width - clickAreaBox!.x - clickAreaBox!.width),
|
||||
]).toEqual([5, 5])
|
||||
// Menu padding alone insets the rows now that the border is gone.
|
||||
]).toEqual([4, 4])
|
||||
await compareOrRefreshGolden(
|
||||
BRANCHLESS_EXPECTED,
|
||||
await captureStableAria(page, '[role="tree"][aria-label="Subagent sessions"]', scaffold.workspaceCwd),
|
||||
|
||||
Reference in New Issue
Block a user