mirror of
https://github.com/deepseek-ai/deepseek-harness
synced 2026-08-15 21:04:50 +00:00
test(web): pin the remaining markdown fixture's event times
Master stabilized two of the three hand-built markdown fixtures by spacing their event times, so the stats line's LLM segment stops depending on how fast the replay ran. `markdown-images` was left on the old shape and kept failing for exactly that reason — a different test each run, always the same one-line difference. Pin it the same way and record the segment its golden now always shows. This supersedes the normalizer that dropped those segments outright, reverted here: pinning the fixture keeps a real part of the page in the goldens instead of hiding it from every one of them, and master's two goldens already record it.
This commit is contained in:
@@ -83,6 +83,7 @@ async function stopServer(server: Server): Promise<void> {
|
||||
/** Build one closed, invariant-checked session fixture with remote and local image Markdown. */
|
||||
function markdownImageFixture(remoteUrl: string): string {
|
||||
const session = Session.create(SessionId('markdown-image-source'))
|
||||
const eventTimeOrigin = new Date().setHours(12, 0, 0, 0)
|
||||
session.append('turn/start', { turn: 1 })
|
||||
const user = session.append('user/message', createUserMessage({
|
||||
content: [{ type: 'text', text: 'Show the Markdown image policy.' }],
|
||||
@@ -126,7 +127,14 @@ function markdownImageFixture(remoteUrl: string): string {
|
||||
}
|
||||
return [
|
||||
JSON.stringify(header),
|
||||
...session.events.map(event => JSON.stringify(event)),
|
||||
// Spaced event times, exactly as the sibling markdown fixtures pin them:
|
||||
// the stats line renders its LLM segment only while the step's measured
|
||||
// milliseconds exceed zero, so a fixture that leaves the times unset lets
|
||||
// the replay's own speed decide whether the golden matches.
|
||||
...session.events.map(event => JSON.stringify({
|
||||
...event,
|
||||
time: eventTimeOrigin + event.seq * 1_000,
|
||||
})),
|
||||
'',
|
||||
].join('\n')
|
||||
}
|
||||
|
||||
@@ -533,24 +533,13 @@ export async function seedSession(scaffold: WebScaffold, fixtureText: string, id
|
||||
|
||||
/**
|
||||
* Normalize an aria snapshot: uuid, cwd, workspace-basename, duration, and
|
||||
* decode-throughput volatility collapse to stable tokens, and the stats line's
|
||||
* wall-clock-gated segments drop out entirely.
|
||||
* decode-throughput volatility collapse to stable tokens.
|
||||
*
|
||||
* Throughput needs a token for the same reason durations do, and no fixture
|
||||
* can supply one: the figure divides a replayed step's output tokens by the
|
||||
* wall time the local run took to stream them, so it moves between two runs
|
||||
* on one machine (measured 69 → 70 tok/s) and swings wildly on a fast replay
|
||||
* (26333 tok/s for a 3 ms stream).
|
||||
*
|
||||
* Tokenizing those values is not enough, because `StatsLine` renders each such
|
||||
* segment only while its measurement exceeds zero (`llmMs`, `toolMs`, and
|
||||
* `decodeMs` all gate on `> 0`). A replay that finishes a step inside one
|
||||
* millisecond therefore omits the segment a slower machine keeps, and the
|
||||
* golden would record how fast the recording machine was rather than what the
|
||||
* page shows: goldens recorded across this suite disagree on the `LLM` segment
|
||||
* for exactly that reason, and CI failed on whichever test happened to run on
|
||||
* a slow enough runner. Dropping the segments makes presence stop deciding.
|
||||
* `TTFT avg` stays: it gates on a step count the fixture determines.
|
||||
*/
|
||||
function normalizeAria(snapshot: string, workspaceCwd: string): string {
|
||||
// The session heading renders the workspace's basename, not the full
|
||||
@@ -571,13 +560,6 @@ function normalizeAria(snapshot: string, workspaceCwd: string): string {
|
||||
duration => duration.startsWith('约') ? duration : '{{duration}}',
|
||||
)
|
||||
.replace(/\d+(?:\.\d+)?(?= tok\/s(?!\w))/g, '{{throughput}}')
|
||||
// Each removal takes one adjacent separator with it, so nothing is left
|
||||
// holding a dangling `·` or a doubled space: a segment followed by its
|
||||
// intra-group separator loses that, and one ending its group loses the
|
||||
// space in front of it instead.
|
||||
.replace(/(?:LLM|Tool call|工具调用) \{\{duration\}\} · /g, '')
|
||||
.replace(/ (?:LLM|Tool call|工具调用) \{\{duration\}\}/g, '')
|
||||
.replace(/ (?:· )?\{\{throughput\}\} tok\/s/g, '')
|
||||
// Message IconActions clocks widen by calendar day/year; collapse every
|
||||
// shape so goldens stay stable across midnight and year boundaries.
|
||||
.replace(/\d{4}年\d{1,2}月\d{1,2}日 \d{2}:\d{2}/g, '{{clock}}')
|
||||
|
||||
@@ -30,4 +30,4 @@
|
||||
- text: Select model
|
||||
- img
|
||||
- button "Send message" [disabled]
|
||||
- text: 1 turns · 1 steps TTFT avg {{duration}} Cache hit 0% Input 10 tok · Output 10 tok
|
||||
- text: 1 turns · 1 steps LLM {{duration}} · Tool call {{duration}} TTFT avg {{duration}} · {{throughput}} tok/s Cache hit 0% Input 10 tok · Output 10 tok
|
||||
|
||||
@@ -36,7 +36,7 @@
|
||||
- img
|
||||
- button "Branch into a new conversation":
|
||||
- img
|
||||
- text: {{clock}} Ran for {{duration}} TTFT {{duration}}
|
||||
- text: {{clock}} Ran for {{duration}} TTFT {{duration}} {{throughput}} tok/s
|
||||
- textbox "Message the agent"
|
||||
- button "Commands":
|
||||
- img
|
||||
@@ -46,4 +46,4 @@
|
||||
- img
|
||||
- button "7% of context used"
|
||||
- button "Send message" [disabled]
|
||||
- text: 1 turns · 2 steps TTFT avg {{duration}} Cache hit 52% Input 17.2K tok · Output 252 tok
|
||||
- text: 1 turns · 2 steps LLM {{duration}} · Tool call {{duration}} TTFT avg {{duration}} · {{throughput}} tok/s Cache hit 52% Input 17.2K tok · Output 252 tok
|
||||
|
||||
@@ -51,7 +51,7 @@
|
||||
- img
|
||||
- button "Branch into a new conversation":
|
||||
- img
|
||||
- text: {{clock}} Ran for {{duration}} TTFT {{duration}}
|
||||
- text: {{clock}} Ran for {{duration}} TTFT {{duration}} {{throughput}} tok/s
|
||||
- textbox "Message the agent"
|
||||
- button "Commands":
|
||||
- img
|
||||
@@ -61,4 +61,4 @@
|
||||
- img
|
||||
- button "13% of context used"
|
||||
- button "Send message" [disabled]
|
||||
- text: 1 turns · 4 steps TTFT avg {{duration}} Cache hit 77% Input 66.5K tok · Output 312 tok
|
||||
- text: 1 turns · 4 steps LLM {{duration}} · Tool call {{duration}} TTFT avg {{duration}} · {{throughput}} tok/s Cache hit 77% Input 66.5K tok · Output 312 tok
|
||||
|
||||
@@ -31,7 +31,7 @@
|
||||
- img
|
||||
- button "Branch into a new conversation":
|
||||
- img
|
||||
- text: {{clock}} Ran for {{duration}} TTFT {{duration}}
|
||||
- text: {{clock}} Ran for {{duration}} TTFT {{duration}} {{throughput}} tok/s
|
||||
- textbox "Message the agent"
|
||||
- button "Commands":
|
||||
- img
|
||||
@@ -41,4 +41,4 @@
|
||||
- img
|
||||
- button "6% of context used"
|
||||
- button "Send message" [disabled]
|
||||
- text: 1 turns · 2 steps TTFT avg {{duration}} Cache hit 99% Input 15.7K tok · Output 111 tok
|
||||
- text: 1 turns · 2 steps LLM {{duration}} · Tool call {{duration}} TTFT avg {{duration}} · {{throughput}} tok/s Cache hit 99% Input 15.7K tok · Output 111 tok
|
||||
|
||||
@@ -23,7 +23,7 @@
|
||||
- img
|
||||
- button "Branch into a new conversation":
|
||||
- img
|
||||
- text: {{clock}} Ran for {{duration}} TTFT {{duration}}
|
||||
- text: {{clock}} Ran for {{duration}} TTFT {{duration}} {{throughput}} tok/s
|
||||
- textbox "Message the agent"
|
||||
- button "Commands":
|
||||
- img
|
||||
@@ -33,4 +33,4 @@
|
||||
- img
|
||||
- button "6% of context used"
|
||||
- button "Send message" [disabled]
|
||||
- text: 1 turns · 1 steps TTFT avg {{duration}} Cache hit 99% Input 7.8K tok · Output 21 tok
|
||||
- text: 1 turns · 1 steps LLM {{duration}} TTFT avg {{duration}} · {{throughput}} tok/s Cache hit 99% Input 7.8K tok · Output 21 tok
|
||||
|
||||
@@ -25,7 +25,7 @@
|
||||
- img
|
||||
- button "Branch into a new conversation":
|
||||
- img
|
||||
- text: {{clock}} Ran for {{duration}} TTFT {{duration}}
|
||||
- text: {{clock}} Ran for {{duration}} TTFT {{duration}} {{throughput}} tok/s
|
||||
- textbox "Message the agent"
|
||||
- button "Commands":
|
||||
- img
|
||||
@@ -35,4 +35,4 @@
|
||||
- img
|
||||
- button "6% of context used"
|
||||
- button "Send message" [disabled]
|
||||
- text: 1 turns · 1 steps TTFT avg {{duration}} Cache hit 99% Input 7.8K tok · Output 79 tok
|
||||
- text: 1 turns · 1 steps LLM {{duration}} TTFT avg {{duration}} · {{throughput}} tok/s Cache hit 99% Input 7.8K tok · Output 79 tok
|
||||
|
||||
@@ -28,4 +28,4 @@
|
||||
- text: Select model
|
||||
- img
|
||||
- button "Send message" [disabled]
|
||||
- text: 1 turns · 1 steps Input 0 tok · Output 0 tok
|
||||
- text: 1 turns · 1 steps LLM {{duration}} Input 0 tok · Output 0 tok
|
||||
|
||||
@@ -44,4 +44,4 @@
|
||||
- text: Select model
|
||||
- img
|
||||
- button "Send message" [disabled]
|
||||
- text: 1 turns · 1 steps Input 0 tok · Output 0 tok
|
||||
- text: 1 turns · 1 steps LLM {{duration}} Input 0 tok · Output 0 tok
|
||||
|
||||
@@ -20,7 +20,7 @@
|
||||
- img
|
||||
- button "Branch into a new conversation" [disabled]:
|
||||
- img
|
||||
- text: Available only on the last message of a completed turn 7/25 {{clock}} Ran for {{duration}} TTFT {{duration}}
|
||||
- text: Available only on the last message of a completed turn 7/25 {{clock}} Ran for {{duration}} TTFT {{duration}} {{throughput}} tok/s
|
||||
- button "Read a.txt":
|
||||
- img
|
||||
- img
|
||||
@@ -55,4 +55,4 @@
|
||||
- text: Select model
|
||||
- img
|
||||
- button "Send message" [disabled]
|
||||
- text: 2 turns · 3 steps TTFT avg {{duration}} Cache hit 98% Input 7.8K tok · Output 103 tok
|
||||
- text: 2 turns · 3 steps LLM {{duration}} · Tool call {{duration}} TTFT avg {{duration}} · {{throughput}} tok/s Cache hit 98% Input 7.8K tok · Output 103 tok
|
||||
|
||||
@@ -36,7 +36,7 @@
|
||||
- img
|
||||
- button "Branch into a new conversation":
|
||||
- img
|
||||
- text: {{clock}} Ran for {{duration}} TTFT {{duration}}
|
||||
- text: {{clock}} Ran for {{duration}} TTFT {{duration}} {{throughput}} tok/s
|
||||
- textbox "Message the agent"
|
||||
- button "Commands":
|
||||
- img
|
||||
@@ -46,4 +46,4 @@
|
||||
- img
|
||||
- button "4% of context used"
|
||||
- button "Send message" [disabled]
|
||||
- text: 1 turns · 2 steps TTFT avg {{duration}} Cache hit 51% Input 10.2K tok · Output 346 tok
|
||||
- text: 1 turns · 2 steps LLM {{duration}} · Tool call {{duration}} TTFT avg {{duration}} · {{throughput}} tok/s Cache hit 51% Input 10.2K tok · Output 346 tok
|
||||
|
||||
@@ -31,7 +31,7 @@
|
||||
- img
|
||||
- button "Branch into a new conversation":
|
||||
- img
|
||||
- text: {{clock}} Ran for {{duration}} TTFT {{duration}}
|
||||
- text: {{clock}} Ran for {{duration}} TTFT {{duration}} {{throughput}} tok/s
|
||||
- textbox "Message the agent"
|
||||
- button "Commands":
|
||||
- img
|
||||
@@ -41,4 +41,4 @@
|
||||
- img
|
||||
- button "3% of context used"
|
||||
- button "Send message" [disabled]
|
||||
- text: 1 turns · 2 steps TTFT avg {{duration}} Cache hit 95% Input 8.6K tok · Output 180 tok
|
||||
- text: 1 turns · 2 steps LLM {{duration}} · Tool call {{duration}} TTFT avg {{duration}} · {{throughput}} tok/s Cache hit 95% Input 8.6K tok · Output 180 tok
|
||||
|
||||
@@ -33,7 +33,7 @@
|
||||
- img
|
||||
- button "Branch into a new conversation":
|
||||
- img
|
||||
- text: 7/25 {{clock}} Ran for {{duration}} TTFT {{duration}}
|
||||
- text: 7/25 {{clock}} Ran for {{duration}} TTFT {{duration}} {{throughput}} tok/s
|
||||
- button "Context compacted View compaction summary":
|
||||
- img
|
||||
- text: Context compacted View compaction summary
|
||||
@@ -51,4 +51,4 @@
|
||||
- text: Select model
|
||||
- img
|
||||
- button "Send message" [disabled]
|
||||
- text: 1 turns · 2 steps TTFT avg {{duration}} Cache hit 98% Input 15.8K tok · Output 135 tok
|
||||
- text: 1 turns · 2 steps LLM {{duration}} · Tool call {{duration}} TTFT avg {{duration}} · {{throughput}} tok/s Cache hit 98% Input 15.8K tok · Output 135 tok
|
||||
|
||||
@@ -33,7 +33,7 @@
|
||||
- img
|
||||
- button "Branch into a new conversation":
|
||||
- img
|
||||
- text: 7/25 {{clock}} Ran for {{duration}} TTFT {{duration}}
|
||||
- text: 7/25 {{clock}} Ran for {{duration}} TTFT {{duration}} {{throughput}} tok/s
|
||||
- button "Context compacted View compaction summary":
|
||||
- img
|
||||
- text: Context compacted View compaction summary
|
||||
@@ -49,4 +49,4 @@
|
||||
- text: Select model
|
||||
- img
|
||||
- button "Send message" [disabled]
|
||||
- text: 1 turns · 2 steps TTFT avg {{duration}} Cache hit 98% Input 15.8K tok · Output 135 tok
|
||||
- text: 1 turns · 2 steps LLM {{duration}} · Tool call {{duration}} TTFT avg {{duration}} · {{throughput}} tok/s Cache hit 98% Input 15.8K tok · Output 135 tok
|
||||
|
||||
@@ -37,7 +37,7 @@
|
||||
- img
|
||||
- button "Branch into a new conversation":
|
||||
- img
|
||||
- text: {{clock}} Ran for {{duration}} TTFT {{duration}}
|
||||
- text: {{clock}} Ran for {{duration}} TTFT {{duration}} {{throughput}} tok/s
|
||||
- textbox "Message the agent"
|
||||
- button "Commands":
|
||||
- img
|
||||
@@ -47,4 +47,4 @@
|
||||
- img
|
||||
- button "6% of context used"
|
||||
- button "Send message" [disabled]
|
||||
- text: 1 turns · 2 steps TTFT avg {{duration}} Cache hit 98% Input 15.8K tok · Output 156 tok
|
||||
- text: 1 turns · 2 steps LLM {{duration}} · Tool call {{duration}} TTFT avg {{duration}} · {{throughput}} tok/s Cache hit 98% Input 15.8K tok · Output 156 tok
|
||||
|
||||
@@ -28,7 +28,7 @@
|
||||
- img
|
||||
- button "Branch into a new conversation":
|
||||
- img
|
||||
- text: {{clock}} Ran for {{duration}} TTFT {{duration}} Now give the same explanation to a human reader. {{clock}}
|
||||
- text: {{clock}} Ran for {{duration}} TTFT {{duration}} {{throughput}} tok/s Now give the same explanation to a human reader. {{clock}}
|
||||
- button "Copy":
|
||||
- img
|
||||
- button "Branch into a new conversation" [disabled]:
|
||||
@@ -43,11 +43,11 @@
|
||||
- img
|
||||
- button "Branch into a new conversation":
|
||||
- img
|
||||
- text: {{clock}} Ran for {{duration}} TTFT {{duration}}
|
||||
- text: {{clock}} Ran for {{duration}} TTFT {{duration}} {{throughput}} tok/s
|
||||
- textbox "Message the agent"
|
||||
- button "Commands":
|
||||
- img
|
||||
- 'button "Access mode, current: Workspace Write"': Workspace Write
|
||||
- button "6% of context used"
|
||||
- button "Send message" [disabled]
|
||||
- text: 2 turns · 2 steps TTFT avg {{duration}} Cache hit 99% Input 15.6K tok · Output 158 tok
|
||||
- text: 2 turns · 2 steps LLM {{duration}} TTFT avg {{duration}} · {{throughput}} tok/s Cache hit 99% Input 15.6K tok · Output 158 tok
|
||||
|
||||
@@ -23,7 +23,7 @@
|
||||
- img
|
||||
- button "Branch into a new conversation":
|
||||
- img
|
||||
- text: {{clock}} Ran for {{duration}} TTFT {{duration}}
|
||||
- text: {{clock}} Ran for {{duration}} TTFT {{duration}} {{throughput}} tok/s
|
||||
- textbox "Message the agent"
|
||||
- button "Commands":
|
||||
- img
|
||||
@@ -33,4 +33,4 @@
|
||||
- img
|
||||
- button "0% of context used"
|
||||
- button "Send message" [disabled]
|
||||
- text: 1 turns · 2 steps TTFT avg {{duration}} Cache hit 0% Input 22 tok · Output 7 tok
|
||||
- text: 1 turns · 2 steps LLM {{duration}} · Tool call {{duration}} TTFT avg {{duration}} · {{throughput}} tok/s Cache hit 0% Input 22 tok · Output 7 tok
|
||||
|
||||
Reference in New Issue
Block a user