| 1 | // Run: tsx src/__tests__/ui-perf-scenarios.test.ts |
| 2 | // |
| 3 | // Simulated-driver UI performance scenarios: the streaming pipeline's count |
| 4 | // budgets under realistic workloads, not just a normal chat answer. Wall-clock |
| 5 | // and input-latency budgets (UI-PERF-04) belong to the real-browser driver; |
| 6 | // here every assertion is deterministic — reducer passes per frame, markdown |
| 7 | // parses per stream, and the background-tab bump-skip invariant. |
| 8 | |
| 9 | import { streamingCommitTarget, streamingMarkdownCommitInterval } from "../components/Markdown"; |
| 10 | import { initialState, reducer } from "../lib/useController"; |
| 11 | import { coalesceStreamDeltas } from "../lib/streamDeltaBatch"; |
| 12 | import type { StreamDeltaEntry } from "../lib/streamDeltaBatch"; |
| 13 | import { generateScenarioChunks, UI_PERF_SCENARIOS } from "../lib/uiPerfScenarios"; |
| 14 | import type { UIPerfScenario } from "../lib/uiPerfScenarios"; |
| 15 | import type { WireEvent } from "../lib/types"; |
| 16 | |
| 17 | let passed = 0; |
| 18 | let failed = 0; |
| 19 | |
| 20 | function ok(cond: boolean, label: string) { |
| 21 | process.stdout.write(` ${cond ? "PASS" : "FAIL"} ${label}\n`); |
| 22 | if (cond) passed += 1; |
| 23 | else failed += 1; |
| 24 | } |
| 25 | |
| 26 | function eq(a: unknown, b: unknown, label: string) { |
| 27 | ok(a === b, a === b ? label : `${label}: expected ${JSON.stringify(b)}, got ${JSON.stringify(a)}`); |
| 28 | } |
| 29 | |
| 30 | const FRAME_MS = 1000 / 60; |
| 31 | |
| 32 | interface SimResult { |
| 33 | frames: number; |
| 34 | commits: number; |
| 35 | answerMarkdownParses: number; |
| 36 | reasoningMarkdownParses: number; |
| 37 | itemsIdentityBreaks: number; |
| 38 | bumpSkipViolations: number; |
| 39 | state: typeof initialState; |
| 40 | } |
| 41 | |
| 42 | // simulate streams a scenario at its chunk rate through the real pipeline: |
| 43 | // per 16.7ms frame the accumulated deltas coalesce into one stream_batch and |
| 44 | // fold through the reducer, while the markdown commit model advances at the |
| 45 | // tiered interval over the growing text. |
| 46 | function simulate(spec: UIPerfScenario, base?: typeof initialState): SimResult { |
| 47 | const chunks = generateScenarioChunks(spec); |
| 48 | let state = base ?? reducer({ ...initialState }, { type: "event", e: { kind: "turn_started" } as WireEvent }); |
| 49 | const chunksPerFrame = spec.chunksPerSec / 60; |
| 50 | |
| 51 | let frames = 0; |
| 52 | let commits = 0; |
| 53 | let answerMarkdownParses = 0; |
| 54 | let reasoningMarkdownParses = 0; |
| 55 | let itemsIdentityBreaks = 0; |
| 56 | let bumpSkipViolations = 0; |
| 57 | let text = ""; |
| 58 | let reasoning = ""; |
| 59 | let answerRenderedLen = 0; |
| 60 | let reasoningRenderedLen = 0; |
| 61 | let lastAnswerParseAt = -Infinity; |
| 62 | let lastReasoningParseAt = -Infinity; |
| 63 | let pendingChunks = 0; |
| 64 | let index = 0; |
| 65 | let firstBatch = true; |
| 66 | let reasoningFinalized = false; |
| 67 | |
| 68 | while (index < chunks.length) { |
| 69 | frames += 1; |
| 70 | pendingChunks += chunksPerFrame; |
| 71 | const batch: StreamDeltaEntry[] = []; |
| 72 | while (pendingChunks >= 1 && index < chunks.length) { |
| 73 | const chunk = chunks[index]; |
| 74 | index += 1; |
| 75 | pendingChunks -= 1; |
| 76 | if (chunk.kind === "text") text += chunk.delta; |
| 77 | else reasoning += chunk.delta; |
| 78 | batch.push({ tabId: "a", e: { kind: chunk.kind, text: chunk.delta } as WireEvent }); |
| 79 | } |
| 80 | if (batch.length === 0) continue; |
| 81 | for (const b of coalesceStreamDeltas(batch)) { |
| 82 | const prev = state; |
| 83 | state = reducer(prev, { type: "stream_batch", segments: b.segments } as never); |
| 84 | if (state !== prev) commits += 1; |
| 85 | if (!firstBatch && prev.items !== state.items) itemsIdentityBreaks += 1; |
| 86 | const bumpSkipped = |
| 87 | prev.items === state.items && |
| 88 | prev.currentAssistant === state.currentAssistant && |
| 89 | prev.pendingUser === state.pendingUser && |
| 90 | prev.retry === state.retry; |
| 91 | if (!firstBatch && !bumpSkipped) bumpSkipViolations += 1; |
| 92 | firstBatch = false; |
| 93 | } |
| 94 | const nowMs = frames * FRAME_MS; |
| 95 | if (spec.reasoningVisible && reasoning.length > 0 && text.length > 0 && !reasoningFinalized) { |
| 96 | reasoningMarkdownParses += 1; |
| 97 | reasoningRenderedLen = reasoning.length; |
| 98 | reasoningFinalized = true; |
| 99 | } |
| 100 | if (spec.reasoningVisible && !reasoningFinalized && nowMs - lastReasoningParseAt >= streamingMarkdownCommitInterval(reasoning.length)) { |
| 101 | const target = streamingCommitTarget(reasoning); |
| 102 | if (target.length > reasoningRenderedLen) { |
| 103 | reasoningMarkdownParses += 1; |
| 104 | reasoningRenderedLen = target.length; |
| 105 | lastReasoningParseAt = nowMs; |
| 106 | } |
| 107 | } |
| 108 | if (nowMs - lastAnswerParseAt >= streamingMarkdownCommitInterval(text.length)) { |
| 109 | const target = streamingCommitTarget(text); |
| 110 | if (target.length > answerRenderedLen) { |
| 111 | answerMarkdownParses += 1; |
| 112 | answerRenderedLen = target.length; |
| 113 | lastAnswerParseAt = nowMs; |
| 114 | } |
| 115 | } |
| 116 | } |
| 117 | if (text.length > answerRenderedLen) answerMarkdownParses += 1; |
| 118 | if (spec.reasoningVisible && reasoning.length > reasoningRenderedLen) reasoningMarkdownParses += 1; |
| 119 | return { frames, commits, answerMarkdownParses, reasoningMarkdownParses, itemsIdentityBreaks, bumpSkipViolations, state }; |
| 120 | } |
| 121 | |
| 122 | const byId = new Map(UI_PERF_SCENARIOS.map((s) => [s.id, s])); |
| 123 | const scenario = (id: string): UIPerfScenario => { |
| 124 | const s = byId.get(id); |
| 125 | if (!s) throw new Error(`missing scenario ${id}`); |
| 126 | return s; |
| 127 | }; |
| 128 | |
| 129 | // --- UI-PERF-01: normal 2KB markdown answer --- |
| 130 | { |
| 131 | const spec = scenario("UI-PERF-01"); |
| 132 | const r = simulate(spec); |
| 133 | ok(r.commits <= r.frames + 1, `01: one reducer pass per frame at most (${r.commits} commits / ${r.frames} frames)`); |
| 134 | ok( |
| 135 | r.answerMarkdownParses <= spec.paragraphs + 3, |
| 136 | `01: markdown parses bounded by blocks, not ticks (${r.answerMarkdownParses} for ${spec.paragraphs} paragraphs)`, |
| 137 | ); |
| 138 | ok(r.state.live !== undefined && r.state.live.text.length >= spec.textChars, "01: full answer reached the live stream"); |
| 139 | } |
| 140 | |
| 141 | // --- UI-PERF-02: 16KB reasoning + 8KB answer at 150 chunks/sec --- |
| 142 | { |
| 143 | const spec = scenario("UI-PERF-02"); |
| 144 | const r = simulate(spec); |
| 145 | const seconds = r.frames / 60; |
| 146 | ok(r.commits / seconds <= 61, `02: state commits stay at or under display FPS (${(r.commits / seconds).toFixed(1)}/s)`); |
| 147 | ok(r.reasoningMarkdownParses <= 2, `02: visible reasoning Markdown stays within its parse budget (${r.reasoningMarkdownParses})`); |
| 148 | ok(r.answerMarkdownParses <= spec.paragraphs + 3, `02: answer Markdown keeps its existing parse budget (${r.answerMarkdownParses})`); |
| 149 | ok(r.state.live !== undefined && r.state.live.reasoning.length >= spec.reasoningChars, "02: full reasoning reached the live stream"); |
| 150 | eq(r.state.live?.reasoningComplete, true, "02: answer text after reasoning completed it"); |
| 151 | } |
| 152 | |
| 153 | // --- UI-PERF-03: 10 code fences in 20KB --- |
| 154 | { |
| 155 | const spec = scenario("UI-PERF-03"); |
| 156 | const r = simulate(spec); |
| 157 | ok(r.commits <= r.frames + 1, `03: commits bounded by frames (${r.commits}/${r.frames})`); |
| 158 | ok( |
| 159 | r.answerMarkdownParses >= spec.codeFences, |
| 160 | `03: open fences keep committing so streamed code stays highlighted (${r.answerMarkdownParses} parses)`, |
| 161 | ); |
| 162 | ok( |
| 163 | r.answerMarkdownParses <= Math.ceil(r.frames / 3) + spec.paragraphs + spec.codeFences, |
| 164 | `03: parse cadence capped by the 50ms tier even inside fences (${r.answerMarkdownParses} parses / ${r.frames} frames)`, |
| 165 | ); |
| 166 | } |
| 167 | |
| 168 | // --- UI-PERF-05: streaming on top of a 100-turn, 30-tool-call transcript --- |
| 169 | { |
| 170 | let long = reducer({ ...initialState }, { type: "event", e: { kind: "turn_started" } as WireEvent }); |
| 171 | for (let turn = 0; turn < 100; turn += 1) { |
| 172 | long = reducer(long, { type: "user", text: `question ${turn}`, seq: long.seq, submissionId: `perf-${turn}` }); |
| 173 | if (turn < 30) { |
| 174 | const id = `tool-${turn}`; |
| 175 | long = reducer(long, { type: "event", e: { kind: "tool_dispatch", tool: { id, name: "edit_file", readOnly: false, args: "{}" } } as WireEvent }); |
| 176 | long = reducer(long, { type: "event", e: { kind: "tool_result", tool: { id, name: "edit_file", readOnly: false, output: "ok", diff: "@@ -1 +1 @@\n-a\n+b\n", added: 1, removed: 1 } } as WireEvent }); |
| 177 | } |
| 178 | long = reducer(long, { type: "event", e: { kind: "message", text: `answer ${turn} with \`code\`` } as WireEvent }); |
| 179 | } |
| 180 | long = reducer(long, { type: "event", e: { kind: "turn_started" } as WireEvent }); |
| 181 | const itemsBefore = long.items.length; |
| 182 | ok(itemsBefore >= 130, `05: transcript carries the long session (${itemsBefore} items)`); |
| 183 | |
| 184 | const r = simulate(scenario("UI-PERF-05"), long); |
| 185 | eq(r.itemsIdentityBreaks, 0, "05: per-delta cost is O(1) — no transcript cloning while streaming over a long session"); |
| 186 | ok(r.commits <= r.frames + 1, `05: commits bounded by frames on a long transcript (${r.commits}/${r.frames})`); |
| 187 | } |
| 188 | |
| 189 | // --- UI-PERF-06: tab A streams while tab B is active --- |
| 190 | { |
| 191 | const r = simulate(scenario("UI-PERF-06")); |
| 192 | eq(r.bumpSkipViolations, 0, "06: background streaming never trips the whole-App bump — liveStore only"); |
| 193 | eq(r.reasoningMarkdownParses, 0, "06: background reasoning performs no Markdown parsing"); |
| 194 | } |
| 195 | |
| 196 | process.stdout.write(`\n${passed} passed, ${failed} failed\n`); |
| 197 | if (failed > 0) process.exit(1); |
| 198 |