diff --git a/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.i18n.yaml b/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.i18n.yaml index 3b0096576d..6b2d7b05e3 100644 --- a/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.i18n.yaml +++ b/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.md -2026-07-24-web-gui-browser-e2e-lane.md: 51277846f7e796430081501a06c3164fc3e2cdd0 -2026-07-24-web-gui-browser-e2e-lane.zh.md: 0a625ca98257df813e3413c90cc47c3be563f3be +2026-07-24-web-gui-browser-e2e-lane.md: e9559aa1f6fba89a99d91e79026552511b337793 +2026-07-24-web-gui-browser-e2e-lane.zh.md: 5f2047125e38f6e1eecf4b8b81d6f5da16686afa diff --git a/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.md b/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.md index 51277846f7..e9559aa1f6 100644 --- a/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.md +++ b/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.md @@ -48,7 +48,7 @@ The lane covers three behavior families. Live-turn scenarios pin ordinary tool e The lane is a required compare-only gate for Linux pull requests under the [browser snapshot CI decision](2026-07-30-web-browser-snapshot-ci-gate.md). The `node 24 / snapshots and artifacts` consumer job owns the [single Linux build](../process/2026-07-30-independent-ci-consumer-build.md), installs the lockfile-selected Chromium, restores its OS-and-lockfile-keyed cache, and runs the lane with `DSH_SNAPSHOT=replay`. This is an intentional plane split: the host and specs use the [tsx source-launch contract](../architecture/2026-07-29-dsh-source-launch-tsx-esm.md), while the browser consumes `apps/web/dist` and package `lib/client.js` artifacts, so the gate depends on `built-package-invariants` for those client artifacts. The hosted and self-hosted default-branch Linux serial jobs run the same gate; the hosted job produces the browser cache consumed by pull requests, while the persistent self-hosted pool needs no hosted cache. CI never records or refreshes goldens. Scenarios remain POSIX-oriented and stay outside the Windows and macOS matrices. -High-cardinality performance diagnostics use the separate opt-in `apps/web/tests/**/*.perf.ts` inventory selected only by `vitest.web.perf.config.ts`. The `complex-history.perf.ts` case reuses the real scaffold, seeds 1,000 compact sessions plus one 500-turn history containing 500 tool calls, and reports Chromium main-thread, DOM, listener, heap, paging, search, and Trajectory measurements. It then opens a blank session, fills a deterministic mixed-language prompt through the real composer, clicks Send, and replays 120 paced text deltas through the real loop and SSE wire; the report includes click-to-user-echo, click-to-first-chunk, stream completion, browser mutation, and persisted-chunk measurements. It carries structural assertions for the intended load and stream shapes but no timing thresholds because machine speed is not a correctness contract. The required `vitest.web.config.ts` inventory remains limited to `*.e2e.ts` and `*.snapshot.ts`, so neither `test:web:built` nor its CI gate collects performance cases. +High-cardinality performance diagnostics use the separate opt-in `apps/web/tests/**/*.perf.ts` inventory selected only by `vitest.web.perf.config.ts`. The isolated `complex-history.perf.ts` cases reuse the real scaffold: the workspace case seeds 1,000 compact sessions plus one 500-turn history containing 500 tool calls, exhausts and remounts that history in Chat, and reports Chromium main-thread, DOM, listener, heap, paging, search, and Trajectory measurements. Two continuation cases seed the same long history but compare the default 24-turn Chat window with all 500 turns expanded before each continues eight identical turns through the real composer, agent loop, SSE wire, tools, and persistence; two turns execute a real `bash` call and assert its durable result, while the final turn fills an 8,232-character mixed-language prompt and replays 120 paced text deltas. A separate soak case starts from a blank session, drives 100 consecutive real composer turns with a `bash` call and result every tenth turn, forces GC every ten turns, and reports ten-turn latency windows plus retained browser state. Per-turn diagnostics cover composer fill, click-to-user-echo, click-to-first-chunk, completion, browser mutations, persisted chunks, and tool events; the synthetic replay model has enough context capacity to keep fixture cardinality stable instead of consuming scripted calls through compaction. Structural assertions pin the intended load, stream, and tool shapes, but timing remains threshold-free because machine speed is not a correctness contract. The required `vitest.web.config.ts` inventory remains limited to `*.e2e.ts` and `*.snapshot.ts`, so neither `test:web:built` nor its CI gate collects performance cases. ## Prior art diff --git a/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.zh.md b/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.zh.md index 0a625ca982..5f2047125e 100644 --- a/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.zh.md +++ b/.agents/notes/implemented/testing/2026-07-24-web-gui-browser-e2e-lane.zh.md @@ -48,7 +48,7 @@ Web GUI 以一条真实组装链交付——chromium 页面 → client 插件 bu 根据[浏览器快照 CI 决策](2026-07-30-web-browser-snapshot-ci-gate.md)，该车道是 Linux 拉取请求必需的只比较门禁。`node 24 / snapshots and artifacts` 消费方任务在[消费方独立构建](../process/2026-07-30-independent-ci-consumer-build.md)中负责唯一一次 Linux 构建，安装锁文件选定的 Chromium，恢复以操作系统和锁文件为键的缓存，并用 `DSH_SNAPSHOT=replay` 运行该车道。这是有意的平面切分：host 与 spec 使用 [tsx 源码启动契约](../architecture/2026-07-29-dsh-source-launch-tsx-esm.md)，浏览器则消费 `apps/web/dist` 和包的 `lib/client.js` 产物，因此门禁依赖 `built-package-invariants` 提供这些客户端产物。托管和自托管的默认分支 Linux 串行任务运行同一门禁；托管任务生成供 PR 消费的浏览器缓存，持久化自托管池则不需要托管侧缓存。CI 从不录制或刷新预期输出。场景仍面向 POSIX，并继续置于 Windows 和 macOS 矩阵之外。 -高基数性能诊断使用单独按需启用的 `apps/web/tests/**/*.perf.ts` 清单，并且只由 `vitest.web.perf.config.ts` 选中。`complex-history.perf.ts` 用例复用真实 scaffold，播种 1,000 个紧凑会话以及一份包含 500 次工具调用的 500 轮次历史，并报告 Chromium 主线程、DOM、监听器、堆内存、分页、搜索和 Trajectory 测量结果。随后它打开一个空白会话，通过真实输入框填入一条确定性的混合语言提示词，点击 Send，并通过真实循环和 SSE wire 回放 120 个带节奏的文本增量；报告包括点击到用户消息回显、点击到首个分片、流完成、浏览器变更和持久化分片测量结果。它对预期负载和流形状设有结构性断言，但不设时间阈值，因为机器速度不属于正确性契约。必需的 `vitest.web.config.ts` 清单仍仅限 `*.e2e.ts` 和 `*.snapshot.ts`，因此 `test:web:built` 及其 CI 门禁都不会收集性能用例。 +高基数性能诊断使用单独按需启用的 `apps/web/tests/**/*.perf.ts` 清单，并且只由 `vitest.web.perf.config.ts` 选中。`complex-history.perf.ts` 的隔离用例复用真实 scaffold：工作区用例播种 1,000 个紧凑会话以及一份包含 500 次工具调用的 500 轮次历史，在 Chat 中穷尽并重新挂载该历史，并报告 Chromium 主线程、DOM、监听器、堆内存、分页、搜索和 Trajectory 测量结果。两个续聊用例播种同一份长历史，但比较默认的 24 轮次 Chat 窗口与展开全部 500 轮次的状态，然后各自通过真实输入框、agent loop、SSE wire、工具和持久化继续进行 8 个相同轮次；其中两轮执行真实 `bash` 调用并断言其持久化结果，最后一轮则填入一条包含 8,232 个字符的混合语言提示词，并回放 120 个带节奏的文本增量。一个单独的 soak 用例从空白会话开始，通过真实输入框连续驱动 100 轮，每第 10 轮执行一次 `bash` 调用并产生结果，每 10 轮强制执行一次 GC，并报告每 10 轮的延迟窗口及保留的浏览器状态。逐轮诊断涵盖输入框填入、点击到用户消息回显、点击到首个分片、完成、浏览器变更、持久化分片和工具事件；合成回放模型拥有足够的上下文容量，可使 fixture 基数保持稳定，而不会因压缩（compaction）消耗脚本化调用。结构性断言钉住预期的负载、流和工具形状，但时间仍不设阈值，因为机器速度不属于正确性契约。必需的 `vitest.web.config.ts` 清单仍仅限 `*.e2e.ts` 和 `*.snapshot.ts`，因此 `test:web:built` 及其 CI 门禁都不会收集性能用例。 ## 业界先例 diff --git a/apps/web/tests/complex-history.perf.ts b/apps/web/tests/complex-history.perf.ts index 8c677663fa..eecc32be51 100644 --- a/apps/web/tests/complex-history.perf.ts +++ b/apps/web/tests/complex-history.perf.ts @@ -16,7 +16,7 @@ import { createToolResultMessage, createUserMessage, } from '@deepseek-ai/dsh-llm' -import type { ReplayOverrideDoc } from '@deepseek-ai/dsh-llm-replay' +import type { ReplayEntry, ReplayOverrideDoc } from '@deepseek-ai/dsh-llm-replay' import type { SessionEvent } from '@deepseek-ai/dsh-session' import { SESSION_FORMAT_VERSION, @@ -42,8 +42,23 @@ const TOOL_TURN_INTERVAL = 10 const TOOLS_PER_TOOL_TURN = 10 const EXPECTED_TOOL_CALLS = LONG_HISTORY_TURNS / TOOL_TURN_INTERVAL * TOOLS_PER_TOOL_TURN const EXPECTED_TRAJECTORY_ROWS = 2_100 +const DEFAULT_HISTORY_TURNS = 24 +const PERF_REPLAY_CONTEXT_WINDOW = 10_000_000 const STREAM_PACE_MS = 8 const STREAM_DELTA_COUNT = 120 +const COMPARISON_TURNS = 8 +const COMPARISON_DELTA_COUNT = 24 +const COMPARISON_TOOL_INTERVAL = 3 +const SOAK_TURNS = 100 +const SOAK_DELTA_COUNT = 8 +const SOAK_TOOL_INTERVAL = 10 +const SOAK_CHECKPOINT_INTERVAL = 10 +const LONG_CONTINUATION_USER_PREFIX = 'LONG_CONTINUATION_USER' +const LONG_CONTINUATION_FIRST_PREFIX = 'LONG_CONTINUATION_FIRST' +const LONG_CONTINUATION_DONE_PREFIX = 'LONG_CONTINUATION_DONE' +const SOAK_USER_PREFIX = 'SOAK_CONVERSATION_USER' +const SOAK_FIRST_PREFIX = 'SOAK_CONVERSATION_FIRST' +const SOAK_DONE_PREFIX = 'SOAK_CONVERSATION_DONE' const LIVE_PROMPT_MARKER = 'STREAM_PERF_USER_INPUT' const STREAM_FIRST_MARKER = 'STREAM_PERF_FIRST' const STREAM_DONE_MARKER = 'STREAM_PERF_DONE' @@ -68,7 +83,6 @@ const STREAM_DELTAS = Array.from({ length: STREAM_DELTA_COUNT }, (_, index) => { if (index === STREAM_DELTA_COUNT - 1) return `${STREAM_DONE_MARKER}.` return `chunk-${String(index).padStart(3, '0')} ${'response'.repeat(3)} ` }) -const STREAM_RESPONSE = STREAM_DELTAS.join('') interface ChromiumMetrics { readonly [name: string]: number @@ -93,6 +107,72 @@ interface MutationProbeResult { readonly records: number } +interface RetainedBrowserState { + readonly domElements: number + readonly nodes: number + readonly listeners: number + readonly heapMb: number +} + +interface ContinuedTurnReport { + readonly ordinal: number + readonly resultingTurns: number + readonly kind: 'text' | 'tool' + readonly promptChars: number + readonly composerFill: Measurement + readonly stream: MutationProbeResult & Measurement & { + readonly paceMs: number + readonly deltaChunks: number + readonly persistedChunks: number + readonly toolCalls: number + readonly toolResults: number + readonly clickToUserEchoMs: number + readonly clickToFirstChunkMs: number + readonly firstChunkToSettledMs: number + } +} + +interface ConversationTurnSpec { + readonly prompt: string + readonly deltas: readonly string[] + readonly userMarker: string + readonly firstMarker: string + readonly doneMarker: string + readonly toolResultMarker?: string +} + +interface RetainedCheckpoint { + readonly turns: number + readonly state: RetainedBrowserState +} + +interface ConversationReport { + readonly startingTurns: number + readonly turnsAdded: number + readonly toolTurns: number + readonly retainedBefore: RetainedBrowserState + readonly retainedAfter: RetainedBrowserState + readonly retainedDelta: RetainedBrowserState + readonly checkpoints: readonly RetainedCheckpoint[] + readonly turns: readonly ContinuedTurnReport[] +} + +interface PerformanceWorld { + readonly scaffold: WebScaffold + readonly page: Page + readonly tripwire: ReturnType + readonly sessionEvents: SessionEvent[] + readonly setupMs: number + readonly replayDir?: string +} + +interface PerformanceWorldOptions { + readonly browser: Browser + readonly replay?: ReplayOverrideDoc + readonly sidebarSessions?: number + readonly seedLongHistory?: boolean +} + function text(value: string): { type: 'text'; text: string }[] { return [{ type: 'text', text: value }] } @@ -108,7 +188,7 @@ function appendTitle(session: Session, title: string, messageSeq: number): void function appendRequestHeader(session: Session, turn: number, step: number): void { session.append('request/header', { header: { - config: { provider: 'synthetic-perf', model: 'synthetic-perf' }, + config: { provider: 'deepseek-official', model: 'deepseek-v4-flash' }, system: `Synthetic performance system prompt for turn ${String(turn)}, step ${String(step)}.`, }, reason: turn === 1 && step === 1 ? 'initial' : 'change', @@ -126,7 +206,7 @@ function appendAssistant( step, message: createAssistantMessage({ content: text(body), - source: { provider: 'synthetic-perf', model: 'synthetic-perf' }, + source: { provider: 'deepseek-official', model: 'deepseek-v4-flash' }, }), usage: { inputTokens: 4_000 + turn * 10, @@ -168,7 +248,7 @@ function appendToolStep( arguments: args, })), ], - source: { provider: 'synthetic-perf', model: 'synthetic-perf' }, + source: { provider: 'deepseek-official', model: 'deepseek-v4-flash' }, }), usage: { inputTokens: 6_000 + turn * 10, @@ -294,10 +374,11 @@ function longHistoryFixture(): string { return fixtureLog(session) } -function streamingReplayOverride(): ReplayOverrideDoc { - const chunks: StreamChunk[] = [ +function textStream(deltas: readonly string[], inputTokens: number): StreamChunk[] { + const response = deltas.join('') + return [ { type: 'block-start', index: 0, blockType: 'text' }, - ...STREAM_DELTAS.map(text => ({ + ...deltas.map(text => ({ type: 'text-delta' as const, index: 0, text, @@ -305,18 +386,122 @@ function streamingReplayOverride(): ReplayOverrideDoc { { type: 'block-end', index: 0, - block: { type: 'text', text: STREAM_RESPONSE }, + block: { type: 'text', text: response }, }, { type: 'usage', usage: { - inputTokens: Math.ceil(LIVE_PROMPT.length / 4), - outputTokens: Math.ceil(STREAM_RESPONSE.length / 4), + inputTokens, + outputTokens: Math.ceil(response.length / 4), }, }, { type: 'finish', reason: { kind: 'stop' } }, ] - return [{ kind: 'chunks', chunks }] +} + +function comparisonPrompt(index: number): string { + if (index === COMPARISON_TURNS) return LIVE_PROMPT + return (`${LONG_CONTINUATION_USER_PREFIX}_${String(index)} ` + + `继续分析这个长会话的第 ${String(index)} 个增量问题，并保留当前滚动和输入响应。 ` + + 'context '.repeat(80)).trimEnd() +} + +function comparisonDeltas(index: number): string[] { + if (index === COMPARISON_TURNS) return STREAM_DELTAS + return Array.from({ length: COMPARISON_DELTA_COUNT }, (_, chunkIndex) => { + if (chunkIndex === 0) return `${LONG_CONTINUATION_FIRST_PREFIX}_${String(index)} ` + if (chunkIndex === COMPARISON_DELTA_COUNT - 1) { + return `${LONG_CONTINUATION_DONE_PREFIX}_${String(index)}.` + } + return `turn-${String(index)}-chunk-${String(chunkIndex).padStart(2, '0')} ${'response'.repeat(2)} ` + }) +} + +function comparisonTurn(index: number): ConversationTurnSpec { + const toolResultMarker = index % COMPARISON_TOOL_INTERVAL === 0 + ? `LONG_CONTINUATION_TOOL_RESULT_${String(index)}` + : undefined + return { + prompt: comparisonPrompt(index), + deltas: comparisonDeltas(index), + userMarker: index === COMPARISON_TURNS + ? LIVE_PROMPT_MARKER + : `${LONG_CONTINUATION_USER_PREFIX}_${String(index)}`, + firstMarker: index === COMPARISON_TURNS + ? STREAM_FIRST_MARKER + : `${LONG_CONTINUATION_FIRST_PREFIX}_${String(index)}`, + doneMarker: index === COMPARISON_TURNS + ? STREAM_DONE_MARKER + : `${LONG_CONTINUATION_DONE_PREFIX}_${String(index)}`, + ...toolResultMarker === undefined ? {} : { toolResultMarker }, + } +} + +function soakTurn(index: number): ConversationTurnSpec { + const suffix = String(index).padStart(3, '0') + const toolResultMarker = index % SOAK_TOOL_INTERVAL === 0 + ? `SOAK_CONVERSATION_TOOL_RESULT_${suffix}` + : undefined + return { + prompt: (`${SOAK_USER_PREFIX}_${suffix} ` + + `持续对话第 ${String(index)} 轮，检查增量渲染与保留状态。 ` + + 'context '.repeat(20)).trimEnd(), + deltas: Array.from({ length: SOAK_DELTA_COUNT }, (_, chunkIndex) => { + if (chunkIndex === 0) return `${SOAK_FIRST_PREFIX}_${suffix} ` + if (chunkIndex === SOAK_DELTA_COUNT - 1) return `${SOAK_DONE_PREFIX}_${suffix}.` + return `soak-${suffix}-${String(chunkIndex).padStart(2, '0')} response ` + }), + userMarker: `${SOAK_USER_PREFIX}_${suffix}`, + firstMarker: `${SOAK_FIRST_PREFIX}_${suffix}`, + doneMarker: `${SOAK_DONE_PREFIX}_${suffix}`, + ...toolResultMarker === undefined ? {} : { toolResultMarker }, + } +} + +function toolStream(index: number, marker: string): StreamChunk[] { + const callId = CallId(`performance-tool-${marker.toLowerCase()}-${String(index)}`) + const args = JSON.stringify({ + command: `printf '${marker}\\n'`, + description: `Emit performance marker ${String(index)}`, + }) + return [ + { type: 'block-start', index: 0, blockType: 'tool-call' }, + { + type: 'tool-call-delta', + index: 0, + id: callId, + name: 'bash', + argumentsDelta: args, + }, + { + type: 'block-end', + index: 0, + block: { type: 'tool-call', id: callId, name: 'bash', arguments: args }, + }, + { type: 'usage', usage: { inputTokens: 256, outputTokens: 32 } }, + { type: 'finish', reason: { kind: 'tool-calls' } }, + ] +} + +function performanceReplayOverride( + turnCount: number, + turnSpec: (index: number) => ConversationTurnSpec, +): ReplayOverrideDoc { + const continuationEntries = Array.from( + { length: turnCount }, + (_, offset): ReplayEntry[] => { + const index = offset + 1 + const spec = turnSpec(index) + const finalResponse: ReplayEntry = { + kind: 'chunks', + chunks: textStream(spec.deltas, Math.ceil(spec.prompt.length / 4)), + } + return spec.toolResultMarker === undefined + ? [finalResponse] + : [{ kind: 'chunks', chunks: toolStream(index, spec.toolResultMarker) }, finalResponse] + }, + ).flat() + return continuationEntries } function rounded(value: number): number { @@ -328,6 +513,20 @@ async function chromiumMetrics(cdp: CDPSession): Promise { return Object.fromEntries(payload.metrics.map(metric => [metric.name, metric.value])) } +async function retainedBrowserState( + cdp: CDPSession, + page: Page, +): Promise { + await cdp.send('HeapProfiler.collectGarbage') + const metrics = await chromiumMetrics(cdp) + return { + domElements: await page.evaluate(() => document.querySelectorAll('*').length), + nodes: requiredMetric(metrics, 'Nodes'), + listeners: requiredMetric(metrics, 'JSEventListeners'), + heapMb: rounded(requiredMetric(metrics, 'JSHeapUsedSize') / 1_048_576), + } +} + function requiredMetric(metrics: ChromiumMetrics, name: string): number { const value = metrics[name] if (value === undefined) throw new Error(`Chromium performance metric ${name} is unavailable`) @@ -436,256 +635,543 @@ async function conversationTurns(page: Page): Promise { return Number(match[1]) } -describe('manual web performance: complex workspace and history', () => { - let scaffold: WebScaffold - let browser: Browser - let page: Page - let setupMs = 0 - let tripwire: ReturnType +function retainedDelta( + before: RetainedBrowserState, + after: RetainedBrowserState, +): RetainedBrowserState { + return { + domElements: after.domElements - before.domElements, + nodes: after.nodes - before.nodes, + listeners: after.listeners - before.listeners, + heapMb: rounded(after.heapMb - before.heapMb), + } +} + +async function launchPerformanceWorld( + options: PerformanceWorldOptions, +): Promise { + const setupStarted = performance.now() let replayDir: string | undefined - const sessionEvents: SessionEvent[] = [] + let scaffold: WebScaffold | undefined + let page: Page | undefined + try { + if (options.replay === undefined) { + scaffold = await launchWebScaffold() + } else { + replayDir = await mkdtemp(join(tmpdir(), 'dsh-web-perf-replay-')) + const replayOverride = join(replayDir, 'replay.override.json') + await writeFile(replayOverride, JSON.stringify(options.replay)) + scaffold = await launchWebScaffold({ + // The override-only fixture supplies one positional script for this + // world's single live session. + replayFixture: join(replayDir, 'override-only.jsonl'), + replayOverride, + paceMs: STREAM_PACE_MS, + replayContextWindow: PERF_REPLAY_CONTEXT_WINDOW, + }) + } + + const sessionEvents: SessionEvent[] = [] + scaffold.ctx.on('session/event', (_session, event: SessionEvent) => { + sessionEvents.push(event) + }) + if ((options.sidebarSessions ?? 0) > 0) { + const small = smallSidebarFixture() + for (let index = 0; index < (options.sidebarSessions ?? 0); index += 1) { + await seedSession(scaffold, small, `perf-sidebar-${String(index).padStart(4, '0')}`) + } + } + if (options.seedLongHistory === true) { + await seedSession(scaffold, longHistoryFixture(), LONG_SESSION_ID) + } + const setupMs = performance.now() - setupStarted + page = await newEnglishPage(options.browser) + return { + scaffold, + page, + tripwire: watchConsole(page), + sessionEvents, + setupMs, + ...replayDir === undefined ? {} : { replayDir }, + } + } catch (error) { + const failures: unknown[] = [error] + if (page !== undefined) { + try { + await page.close() + } catch (cleanupError) { + failures.push(cleanupError) + } + } + if (scaffold !== undefined) { + try { + await scaffold.close() + } catch (cleanupError) { + failures.push(cleanupError) + } + } + if (replayDir !== undefined) { + try { + await rm(replayDir, { recursive: true, force: true }) + } catch (cleanupError) { + failures.push(cleanupError) + } + } + if (failures.length === 1) throw error + throw new AggregateError(failures, 'web performance setup and cleanup failed') + } +} + +async function closePerformanceWorld(world: PerformanceWorld): Promise { + const failures: unknown[] = [] + await world.page.close().catch((error: unknown) => failures.push(error)) + await world.scaffold.close().catch((error: unknown) => failures.push(error)) + if (world.replayDir !== undefined) { + await rm(world.replayDir, { recursive: true, force: true }) + .catch((error: unknown) => failures.push(error)) + } + if (failures.length === 1) throw failures[0] + if (failures.length > 1) throw new AggregateError(failures, 'web performance teardown failed') +} + +async function openPerformancePage( + world: PerformanceWorld, + expectedSessions: number, +): Promise { + await world.page.goto(world.scaffold.baseUrl, { waitUntil: 'load' }) + await world.page.waitForSelector('[class*="frame"]', { timeout: 30_000 }) + const group = world.page.getByRole('treeitem').first() + await expect.poll(() => group.textContent(), { timeout: 30_000 }) + .toContain(`${String(expectedSessions)} ${expectedSessions === 1 ? 'session' : 'sessions'}`) + return group +} + +async function openLongHistory(page: Page): Promise { + await page.getByRole('textbox', { name: 'Search name, keywords...', exact: true }) + .fill('LONG_PERF_SENTINEL') + const results = page.getByRole('tree', { name: 'Search results' }).getByRole('treeitem') + await expect.poll(() => results.count(), { timeout: 60_000 }).toBe(1) + await results.first().click() + await page.getByRole('tab', { name: 'Chat', exact: true }).waitFor({ timeout: 30_000 }) + return conversationTurns(page) +} + +async function continueConversation( + world: PerformanceWorld, + cdp: CDPSession, + options: { + readonly startingTurns: number + readonly turnCount: number + readonly turnSpec: (index: number) => ConversationTurnSpec + readonly expectedSessionId?: SessionId + readonly checkpointInterval?: number + }, +): Promise { + const composer = world.page.locator('textarea:enabled').last() + await composer.waitFor({ timeout: 15_000 }) + const retainedBefore = await retainedBrowserState(cdp, world.page) + const checkpoints: RetainedCheckpoint[] = [{ turns: options.startingTurns, state: retainedBefore }] + const turns: ContinuedTurnReport[] = [] + let liveSessionId = options.expectedSessionId + + for (let index = 1; index <= options.turnCount; index += 1) { + const spec = options.turnSpec(index) + const composerFill = await measure(cdp, async () => { + await composer.fill(spec.prompt) + await expect.poll(() => composer.inputValue()).toBe(spec.prompt) + return (await composer.inputValue()).length + }) + expect(composerFill.value).toBe(spec.prompt.length) + + const eventStart = world.sessionEvents.length + await startMutationProbe(world.page) + const streamBefore = await chromiumMetrics(cdp) + const streamStarted = performance.now() + const settled = world.scaffold.whenTurnSettled(60_000) + await world.page.getByRole('button', { name: 'Send message', exact: true }).click() + await world.page.getByText(spec.userMarker, { exact: false }).last().waitFor({ timeout: 15_000 }) + const clickToUserEchoMs = performance.now() - streamStarted + await world.page.getByText(spec.firstMarker, { exact: false }).last().waitFor({ timeout: 15_000 }) + const clickToFirstChunkMs = performance.now() - streamStarted + const settledSessionId = await settled + if (liveSessionId === undefined) { + liveSessionId = settledSessionId + } else { + expect(settledSessionId).toBe(liveSessionId) + } + await expect.poll( + () => world.page.locator('[data-streaming="true"]').count(), + { timeout: 15_000 }, + ).toBe(0) + await world.page.getByText(spec.doneMarker, { exact: false }).last().waitFor({ timeout: 15_000 }) + const clickToSettledMs = performance.now() - streamStarted + const streamAfter = await chromiumMetrics(cdp) + const mutations = await stopMutationProbe(world.page) + const turnEvents = world.sessionEvents.slice(eventStart) + const chunks = turnEvents.filter(event => event.type === 'assistant/chunk') + const toolCalls = turnEvents.filter(event => event.type === 'tool/call') + const toolResults = turnEvents.filter(event => event.type === 'tool/result') + const toolTurn = spec.toolResultMarker !== undefined + expect(chunks).toHaveLength(spec.deltas.length + (toolTurn ? 9 : 4)) + expect(toolCalls).toHaveLength(toolTurn ? 1 : 0) + expect(toolResults).toHaveLength(toolTurn ? 1 : 0) + if (spec.toolResultMarker !== undefined) { + expect(toolCalls[0]?.data.name).toBe('bash') + const toolResult = toolResults[0] + if (toolResult?.type !== 'tool/result') { + throw new Error(`continued turn ${String(index)} did not log its tool result`) + } + const resultBlock = toolResult.data.message.content.find(block => block.type === 'tool-result') + expect(resultBlock?.isError).toBe(false) + expect(resultBlock?.content + .filter(block => block.type === 'text') + .map(block => block.text) + .join('')).toContain(spec.toolResultMarker) + } + const user = turnEvents.find( + event => event.type === 'user/message' && event.data.source.kind === 'user', + ) + if (user?.type !== 'user/message') { + throw new Error(`continued turn ${String(index)} did not log its user message`) + } + expect(user.data.content + .filter(block => block.type === 'text') + .map(block => block.text) + .join('')).toBe(spec.prompt) + const resultingTurns = await conversationTurns(world.page) + expect(resultingTurns).toBe(options.startingTurns + index) + turns.push({ + ordinal: index, + resultingTurns, + kind: toolTurn ? 'tool' : 'text', + promptChars: spec.prompt.length, + composerFill: composerFill.measurement, + stream: { + paceMs: STREAM_PACE_MS, + deltaChunks: spec.deltas.length, + persistedChunks: chunks.length, + toolCalls: toolCalls.length, + toolResults: toolResults.length, + clickToUserEchoMs: rounded(clickToUserEchoMs), + clickToFirstChunkMs: rounded(clickToFirstChunkMs), + firstChunkToSettledMs: rounded(clickToSettledMs - clickToFirstChunkMs), + ...mutations, + ...metricDelta(streamBefore, streamAfter, clickToSettledMs), + }, + }) + + if (options.checkpointInterval !== undefined && index % options.checkpointInterval === 0) { + checkpoints.push({ + turns: options.startingTurns + index, + state: await retainedBrowserState(cdp, world.page), + }) + } + } + + const lastCheckpoint = checkpoints.at(-1) + const retainedAfter = lastCheckpoint?.turns === options.startingTurns + options.turnCount + ? lastCheckpoint.state + : await retainedBrowserState(cdp, world.page) + if (lastCheckpoint?.turns !== options.startingTurns + options.turnCount) { + checkpoints.push({ turns: options.startingTurns + options.turnCount, state: retainedAfter }) + } + return { + startingTurns: options.startingTurns, + turnsAdded: options.turnCount, + toolTurns: turns.filter(turn => turn.kind === 'tool').length, + retainedBefore, + retainedAfter, + retainedDelta: retainedDelta(retainedBefore, retainedAfter), + checkpoints, + turns, + } +} + +function average(values: readonly number[]): number { + return rounded(values.reduce((sum, value) => sum + value, 0) / values.length) +} + +function p95(values: readonly number[]): number { + const sorted = [...values].sort((left, right) => left - right) + return sorted[Math.max(0, Math.ceil(sorted.length * 0.95) - 1)] ?? 0 +} + +function summarizeTurnWindows( + turns: readonly ContinuedTurnReport[], + windowSize: number, +): object[] { + const windows: object[] = [] + for (let start = 0; start < turns.length; start += windowSize) { + const window = turns.slice(start, start + windowSize) + const streamWall = window.map(turn => turn.stream.wallMs) + const userEcho = window.map(turn => turn.stream.clickToUserEchoMs) + const firstChunk = window.map(turn => turn.stream.clickToFirstChunkMs) + windows.push({ + turns: `${String(start + 1)}-${String(start + window.length)}`, + toolTurns: window.filter(turn => turn.kind === 'tool').length, + average: { + composerFillWallMs: average(window.map(turn => turn.composerFill.wallMs)), + streamWallMs: average(streamWall), + clickToUserEchoMs: average(userEcho), + clickToFirstChunkMs: average(firstChunk), + taskMs: average(window.map(turn => turn.stream.taskMs)), + scriptMs: average(window.map(turn => turn.stream.scriptMs)), + recalcStyleMs: average(window.map(turn => turn.stream.recalcStyleMs)), + mutationBatches: average(window.map(turn => turn.stream.batches)), + }, + p95: { + streamWallMs: rounded(p95(streamWall)), + clickToUserEchoMs: rounded(p95(userEcho)), + clickToFirstChunkMs: rounded(p95(firstChunk)), + }, + }) + } + return windows +} + +describe('manual web performance: complex workspace and history', () => { + let browser: Browser beforeAll(async () => { if (webSnapshotMode() === 'record') { throw new Error('manual web performance runs only with deterministic replay') } - const setupStarted = performance.now() - replayDir = await mkdtemp(join(tmpdir(), 'dsh-web-perf-replay-')) - const replayOverride = join(replayDir, 'replay.override.json') - await writeFile(replayOverride, JSON.stringify(streamingReplayOverride())) - scaffold = await launchWebScaffold({ - // A whole-script override needs no committed JSONL, but replayFixture - // selects the replay adapter and supplies its diagnostic identity. - replayFixture: join(replayDir, 'override-only.jsonl'), - replayOverride, - paceMs: STREAM_PACE_MS, - }) - scaffold.ctx.on('session/event', (_session, event: SessionEvent) => { - sessionEvents.push(event) - }) - const small = smallSidebarFixture() - for (let index = 0; index < SIDEBAR_SESSION_COUNT; index += 1) { - await seedSession(scaffold, small, `perf-sidebar-${String(index).padStart(4, '0')}`) - } - await seedSession(scaffold, longHistoryFixture(), LONG_SESSION_ID) - setupMs = performance.now() - setupStarted - browser = await chromium.launch() - page = await newEnglishPage(browser) - tripwire = watchConsole(page) }) afterAll(async () => { - const failures: unknown[] = [] - await browser?.close().catch((error: unknown) => failures.push(error)) - await scaffold?.close().catch((error: unknown) => failures.push(error)) - if (replayDir !== undefined) { - await rm(replayDir, { recursive: true, force: true }) - .catch((error: unknown) => failures.push(error)) - } - if (failures.length === 1) throw failures[0] - if (failures.length > 1) { - throw new AggregateError(failures, 'web performance teardown failed') - } + await browser?.close() }) - it('reports sidebar, paging, and trajectory rendering costs', async () => { - const bootStarted = performance.now() - await page.goto(scaffold.baseUrl, { waitUntil: 'load' }) - await page.waitForSelector('[class*="frame"]', { timeout: 30_000 }) - const group = page.getByRole('treeitem').first() - await expect.poll(() => group.textContent(), { timeout: 30_000 }) - .toContain(`${String(SIDEBAR_SESSION_COUNT + 1)} sessions`) - const bootReadyMs = performance.now() - bootStarted - - const cdp = await page.context().newCDPSession(page) - await cdp.send('Performance.enable') - const firstContentfulPaintMs = await page.evaluate( - () => globalThis.performance.getEntriesByName('first-contentful-paint')[0]?.startTime, - ) - - const sidebar = await measure(cdp, async () => { - await group.click() - return stableCount( - page.getByRole('treeitem'), - count => count === SIDEBAR_SESSION_COUNT + 2, + it('reports workspace, history, and trajectory cardinality costs', async () => { + const world = await launchPerformanceWorld({ + browser, + sidebarSessions: SIDEBAR_SESSION_COUNT, + seedLongHistory: true, + }) + try { + const bootStarted = performance.now() + const group = await openPerformancePage(world, SIDEBAR_SESSION_COUNT + 1) + const bootReadyMs = performance.now() - bootStarted + const page = world.page + const cdp = await page.context().newCDPSession(page) + await cdp.send('Performance.enable') + const firstContentfulPaintMs = await page.evaluate( + () => globalThis.performance.getEntriesByName('first-contentful-paint')[0]?.startTime, ) - }) - expect(sidebar.value).toBe(SIDEBAR_SESSION_COUNT + 2) - await group.click() - await expect.poll(() => page.getByRole('treeitem').count()).toBe(1) + const sidebar = await measure(cdp, async () => { + await group.click() + return stableCount( + page.getByRole('treeitem'), + count => count === SIDEBAR_SESSION_COUNT + 2, + ) + }) + expect(sidebar.value).toBe(SIDEBAR_SESSION_COUNT + 2) + await group.click() + await expect.poll(() => page.getByRole('treeitem').count()).toBe(1) - const contentSearch = await measure(cdp, async () => { - await page.getByRole('textbox', { name: 'Search name, keywords...', exact: true }) - .fill('LONG_PERF_SENTINEL') - const results = page.getByRole('tree', { name: 'Search results' }) - .getByRole('treeitem') - await expect.poll(() => results.count(), { timeout: 60_000 }).toBe(1) - const result = results.first() - await result.waitFor({ timeout: 60_000 }) - return result - }) - - const openLongHistory = await measure(cdp, async () => { - await contentSearch.value.click() - await page.getByRole('tab', { name: 'Trajectory', exact: true }).waitFor({ timeout: 30_000 }) - return conversationTurns(page) - }) - expect(openLongHistory.value).toBeGreaterThan(0) - - const trajectoryRows = page.getByRole('row') - const coldTrajectory = await measure(cdp, async () => { - await page.getByRole('tab', { name: 'Trajectory', exact: true }).click() - return stableCount(trajectoryRows, count => count === EXPECTED_TRAJECTORY_ROWS) - }) - expect(coldTrajectory.value).toBe(EXPECTED_TRAJECTORY_ROWS) - - const collapseTurns = await measure(cdp, async () => { - await page.getByRole('button', { name: 'Collapse turns', exact: true }).click() - return stableCount(trajectoryRows, count => count > 0 && count < EXPECTED_TRAJECTORY_ROWS) - }) - expect(collapseTurns.value).toBeLessThan(EXPECTED_TRAJECTORY_ROWS) - - const trajectorySearch = await measure(cdp, async () => { - await page.getByRole('searchbox', { name: 'Search trajectory', exact: true }).fill('turn 499') - return stableCount(trajectoryRows, count => count > 0 && count < 20) - }) - expect(trajectorySearch.value).toBeLessThan(20) - - await page.getByRole('tab', { name: 'Chat', exact: true }).click() - const historyPages: { turns: number; measurement: Measurement }[] = [] - let turns = await conversationTurns(page) - for (let pageIndex = 0; pageIndex < 5; pageIndex += 1) { - const previousTurns = turns - const older = await measure(cdp, async () => { - await page.getByRole('button', { name: 'Load earlier', exact: true }).click() - await expect.poll(() => conversationTurns(page), { timeout: 30_000 }).toBeGreaterThan(previousTurns) + const contentSearch = await measure(cdp, async () => { + await page.getByRole('textbox', { name: 'Search name, keywords...', exact: true }) + .fill('LONG_PERF_SENTINEL') + const results = page.getByRole('tree', { name: 'Search results' }).getByRole('treeitem') + await expect.poll(() => results.count(), { timeout: 60_000 }).toBe(1) + await results.first().waitFor({ timeout: 60_000 }) + return results.first() + }) + const opened = await measure(cdp, async () => { + await contentSearch.value.click() + await page.getByRole('tab', { name: 'Trajectory', exact: true }).waitFor({ timeout: 30_000 }) return conversationTurns(page) }) - turns = older.value - historyPages.push({ turns, measurement: older.measurement }) - } + expect(opened.value).toBe(DEFAULT_HISTORY_TURNS) - const warmTrajectory = await measure(cdp, async () => { - await page.getByRole('tab', { name: 'Trajectory', exact: true }).click() - return stableCount(trajectoryRows, count => count === EXPECTED_TRAJECTORY_ROWS) - }) - expect(warmTrajectory.value).toBe(EXPECTED_TRAJECTORY_ROWS) + const trajectoryRows = page.getByRole('row') + const coldTrajectory = await measure(cdp, async () => { + await page.getByRole('tab', { name: 'Trajectory', exact: true }).click() + return stableCount(trajectoryRows, count => count === EXPECTED_TRAJECTORY_ROWS) + }) + expect(coldTrajectory.value).toBe(EXPECTED_TRAJECTORY_ROWS) - const newSession = await measure(cdp, async () => { - await page.getByRole('button', { name: 'New session', exact: true }).last().click() - await connectFreshWorkspace(page, 'stream-perf') - const input = page.locator( - 'textarea:enabled[placeholder="Describe what you want to build"]', - ) - await input.waitFor({ timeout: 15_000 }) - return input - }) + const collapseTurns = await measure(cdp, async () => { + await page.getByRole('button', { name: 'Collapse turns', exact: true }).click() + return stableCount(trajectoryRows, count => count > 0 && count < EXPECTED_TRAJECTORY_ROWS) + }) + expect(collapseTurns.value).toBeLessThan(EXPECTED_TRAJECTORY_ROWS) + const trajectorySearch = await measure(cdp, async () => { + await page.getByRole('searchbox', { name: 'Search trajectory', exact: true }).fill('turn 499') + return stableCount(trajectoryRows, count => count > 0 && count < 20) + }) + expect(trajectorySearch.value).toBeLessThan(20) - const composerInput = await measure(cdp, async () => { - await newSession.value.fill(LIVE_PROMPT) - await expect.poll(() => newSession.value.inputValue()).toBe(LIVE_PROMPT) - return (await newSession.value.inputValue()).length - }) - expect(composerInput.value).toBe(LIVE_PROMPT.length) + await page.getByRole('tab', { name: 'Chat', exact: true }).click() + const historyPages: { turns: number; measurement: Measurement }[] = [] + let turns = await conversationTurns(page) + while (turns < LONG_HISTORY_TURNS) { + const previousTurns = turns + const older = await measure(cdp, async () => { + await page.getByRole('button', { name: 'Load earlier', exact: true }).click() + await expect.poll(() => conversationTurns(page), { timeout: 30_000 }) + .toBeGreaterThan(previousTurns) + return conversationTurns(page) + }) + turns = older.value + historyPages.push({ turns, measurement: older.measurement }) + } - await startMutationProbe(page) - const streamBefore = await chromiumMetrics(cdp) - const streamStarted = performance.now() - const settled = scaffold.whenTurnSettled(60_000) - await page.getByRole('button', { name: 'Send message', exact: true }).click() - await page.getByText(LIVE_PROMPT_MARKER, { exact: false }).last() - .waitFor({ timeout: 15_000 }) - const clickToUserEchoMs = performance.now() - streamStarted - await page.getByText(STREAM_FIRST_MARKER, { exact: false }).last() - .waitFor({ timeout: 15_000 }) - const clickToFirstChunkMs = performance.now() - streamStarted - await settled - await expect.poll( - () => page.locator('[data-streaming="true"]').count(), - { timeout: 15_000 }, - ).toBe(0) - await page.getByText(STREAM_DONE_MARKER, { exact: false }).last() - .waitFor({ timeout: 15_000 }) - const clickToSettledMs = performance.now() - streamStarted - const streamAfter = await chromiumMetrics(cdp) - const streamMutations = await stopMutationProbe(page) - const streamMeasurement = metricDelta( - streamBefore, - streamAfter, - clickToSettledMs, - ) + const warmTrajectory = await measure(cdp, async () => { + await page.getByRole('tab', { name: 'Trajectory', exact: true }).click() + return stableCount(trajectoryRows, count => count === EXPECTED_TRAJECTORY_ROWS) + }) + expect(warmTrajectory.value).toBe(EXPECTED_TRAJECTORY_ROWS) + const warmConversation = await measure(cdp, async () => { + await page.getByRole('tab', { name: 'Chat', exact: true }).click() + return conversationTurns(page) + }) + expect(warmConversation.value).toBe(LONG_HISTORY_TURNS) - const assistantChunks = sessionEvents.filter( - event => event.type === 'assistant/chunk', - ) - expect(assistantChunks).toHaveLength(STREAM_DELTA_COUNT + 4) - const liveUser = sessionEvents.filter(event => event.type === 'user/message').at(-1) - if (liveUser?.type !== 'user/message') throw new Error('live performance prompt was not logged') - const liveUserText = liveUser.data.content - .filter(block => block.type === 'text') - .map(block => block.text) - .join('') - expect(liveUserText).toBe(LIVE_PROMPT) - - const report = { - fixture: { - sidebarSessions: SIDEBAR_SESSION_COUNT, - totalSessions: SIDEBAR_SESSION_COUNT + 1, - longHistoryTurns: LONG_HISTORY_TURNS, - toolCalls: EXPECTED_TOOL_CALLS, - trajectoryRows: EXPECTED_TRAJECTORY_ROWS, - }, - setupMs: rounded(setupMs), - boot: { - readyMs: rounded(bootReadyMs), - firstContentfulPaintMs: firstContentfulPaintMs === undefined - ? null - : rounded(firstContentfulPaintMs), - }, - sidebarExpand: sidebar.measurement, - contentSearch: contentSearch.measurement, - openLongHistory: { - initialTurns: openLongHistory.value, - ...openLongHistory.measurement, - }, - coldTrajectory: { - rows: coldTrajectory.value, - ...coldTrajectory.measurement, - }, - collapseTurns: { - rows: collapseTurns.value, - ...collapseTurns.measurement, - }, - trajectorySearch: { - rows: trajectorySearch.value, - ...trajectorySearch.measurement, - }, - historyPages, - warmTrajectory: { - rows: warmTrajectory.value, - ...warmTrajectory.measurement, - }, - liveInput: { - promptChars: LIVE_PROMPT.length, - newSession: newSession.measurement, - composerFill: composerInput.measurement, - stream: { - paceMs: STREAM_PACE_MS, - deltaChunks: STREAM_DELTA_COUNT, - persistedChunks: assistantChunks.length, - clickToUserEchoMs: rounded(clickToUserEchoMs), - clickToFirstChunkMs: rounded(clickToFirstChunkMs), - firstChunkToSettledMs: rounded(clickToSettledMs - clickToFirstChunkMs), - mutationBatches: streamMutations.batches, - mutationRecords: streamMutations.records, - ...streamMeasurement, + console.info(`WEB_PERF_RESULT ${JSON.stringify({ + scenario: 'workspace-history-trajectory', + fixture: { + sidebarSessions: SIDEBAR_SESSION_COUNT, + totalSessions: SIDEBAR_SESSION_COUNT + 1, + longHistoryTurns: LONG_HISTORY_TURNS, + toolCalls: EXPECTED_TOOL_CALLS, + trajectoryRows: EXPECTED_TRAJECTORY_ROWS, }, - }, + setupMs: rounded(world.setupMs), + boot: { + readyMs: rounded(bootReadyMs), + firstContentfulPaintMs: firstContentfulPaintMs === undefined + ? null + : rounded(firstContentfulPaintMs), + }, + sidebarExpand: sidebar.measurement, + contentSearch: contentSearch.measurement, + openLongHistory: { initialTurns: opened.value, ...opened.measurement }, + coldTrajectory: { rows: coldTrajectory.value, ...coldTrajectory.measurement }, + collapseTurns: { rows: collapseTurns.value, ...collapseTurns.measurement }, + trajectorySearch: { rows: trajectorySearch.value, ...trajectorySearch.measurement }, + historyPages, + warmTrajectory: { rows: warmTrajectory.value, ...warmTrajectory.measurement }, + warmConversation: { turns: warmConversation.value, ...warmConversation.measurement }, + }, null, 2)}`) + expect(world.tripwire.warnings).toEqual([]) + expect(world.tripwire.pageErrors).toEqual([]) + } finally { + await closePerformanceWorld(world) + } + }) + + it('reports default 24-turn history plus eight continued turns', async () => { + const world = await launchPerformanceWorld({ + browser, + replay: performanceReplayOverride(COMPARISON_TURNS, comparisonTurn), + seedLongHistory: true, + }) + try { + await openPerformancePage(world, 1) + const cdp = await world.page.context().newCDPSession(world.page) + await cdp.send('Performance.enable') + const opened = await measure(cdp, () => openLongHistory(world.page)) + expect(opened.value).toBe(DEFAULT_HISTORY_TURNS) + const conversation = await continueConversation(world, cdp, { + startingTurns: DEFAULT_HISTORY_TURNS, + turnCount: COMPARISON_TURNS, + turnSpec: comparisonTurn, + expectedSessionId: SessionId(LONG_SESSION_ID), + }) + expect(conversation.toolTurns).toBe(2) + console.info(`WEB_PERF_RESULT ${JSON.stringify({ + scenario: 'default-resume-24-plus-8', + setupMs: rounded(world.setupMs), + replayContextWindow: PERF_REPLAY_CONTEXT_WINDOW, + openLongHistory: { turns: opened.value, ...opened.measurement }, + conversation, + }, null, 2)}`) + expect(world.tripwire.warnings).toEqual([]) + expect(world.tripwire.pageErrors).toEqual([]) + } finally { + await closePerformanceWorld(world) + } + }) + + it('reports fully expanded 500-turn history plus eight continued turns', async () => { + const world = await launchPerformanceWorld({ + browser, + replay: performanceReplayOverride(COMPARISON_TURNS, comparisonTurn), + seedLongHistory: true, + }) + try { + await openPerformancePage(world, 1) + const cdp = await world.page.context().newCDPSession(world.page) + await cdp.send('Performance.enable') + expect(await openLongHistory(world.page)).toBe(DEFAULT_HISTORY_TURNS) + const historyPages: { turns: number; measurement: Measurement }[] = [] + let turns = DEFAULT_HISTORY_TURNS + while (turns < LONG_HISTORY_TURNS) { + const previousTurns = turns + const older = await measure(cdp, async () => { + await world.page.getByRole('button', { name: 'Load earlier', exact: true }).click() + await expect.poll(() => conversationTurns(world.page), { timeout: 30_000 }) + .toBeGreaterThan(previousTurns) + return conversationTurns(world.page) + }) + turns = older.value + historyPages.push({ turns, measurement: older.measurement }) + } + expect(turns).toBe(LONG_HISTORY_TURNS) + const conversation = await continueConversation(world, cdp, { + startingTurns: LONG_HISTORY_TURNS, + turnCount: COMPARISON_TURNS, + turnSpec: comparisonTurn, + expectedSessionId: SessionId(LONG_SESSION_ID), + }) + expect(conversation.toolTurns).toBe(2) + console.info(`WEB_PERF_RESULT ${JSON.stringify({ + scenario: 'expanded-history-500-plus-8', + setupMs: rounded(world.setupMs), + replayContextWindow: PERF_REPLAY_CONTEXT_WINDOW, + historyPages, + conversation, + }, null, 2)}`) + expect(world.tripwire.warnings).toEqual([]) + expect(world.tripwire.pageErrors).toEqual([]) + } finally { + await closePerformanceWorld(world) + } + }) + + it('reports one hundred continuously generated turns with retained checkpoints', async () => { + const world = await launchPerformanceWorld({ + browser, + replay: performanceReplayOverride(SOAK_TURNS, soakTurn), + }) + try { + await world.page.goto(world.scaffold.baseUrl, { waitUntil: 'load' }) + await world.page.waitForSelector('[class*="frame"]', { timeout: 30_000 }) + await connectFreshWorkspace(world.page, 'continuous-conversation-perf') + const cdp = await world.page.context().newCDPSession(world.page) + await cdp.send('Performance.enable') + const conversation = await continueConversation(world, cdp, { + startingTurns: 0, + turnCount: SOAK_TURNS, + turnSpec: soakTurn, + checkpointInterval: SOAK_CHECKPOINT_INTERVAL, + }) + expect(conversation.toolTurns).toBe(SOAK_TURNS / SOAK_TOOL_INTERVAL) + const { turns, ...retained } = conversation + console.info(`WEB_PERF_RESULT ${JSON.stringify({ + scenario: 'continuous-100-turn-soak', + setupMs: rounded(world.setupMs), + replayContextWindow: PERF_REPLAY_CONTEXT_WINDOW, + ...retained, + windows: summarizeTurnWindows(turns, SOAK_CHECKPOINT_INTERVAL), + }, null, 2)}`) + expect(world.tripwire.warnings).toEqual([]) + expect(world.tripwire.pageErrors).toEqual([]) + } finally { + await closePerformanceWorld(world) } - console.info(`WEB_PERF_RESULT ${JSON.stringify(report, null, 2)}`) - expect(tripwire.warnings).toEqual([]) - expect(tripwire.pageErrors).toEqual([]) }) }) diff --git a/apps/web/tests/scaffold.ts b/apps/web/tests/scaffold.ts index 8316e75e3d..6b8067235f 100644 --- a/apps/web/tests/scaffold.ts +++ b/apps/web/tests/scaffold.ts @@ -85,6 +85,14 @@ const REPLAY_PROVIDERS = [{ models: [{ id: 'deepseek-v4-flash', name: 'DeepSeek-V4-Flash', contextWindow: 128_000 }], }] +function replayProviders(contextWindow: number | undefined): typeof REPLAY_PROVIDERS { + if (contextWindow === undefined) return REPLAY_PROVIDERS + return REPLAY_PROVIDERS.map(provider => ({ + ...provider, + models: provider.models.map(model => ({ ...model, contextWindow })), + })) +} + /** A booted web scaffold: real composition, mode-selected model backend, temp world. */ export interface WebScaffold { /** The active snapshot mode this scaffold booted under. */ @@ -134,6 +142,8 @@ export interface LaunchOptions { replayOverride?: string /** Per-chunk replay pacing (ms) so the browser observes genuinely incremental SSE; replay/refresh only. */ paceMs?: number + /** Synthetic model capacity for UI scenarios whose seeded history must remain uncompacted. */ + replayContextWindow?: number /** * Tool presentation mode patched onto the shipped `tools` row (`code` * collapses the wire to run_code + the SDK prompt section). Omit for the @@ -344,7 +354,7 @@ export async function launchWebScaffold(options: LaunchOptions = {}): Promise