| 1 | import assert from "node:assert/strict"; |
| 2 | |
| 3 | export const TRANSCRIPT_PHASES = ["initialLoad", "historyPaging", "streaming", "input"]; |
| 4 | export const LONG_TASK_LIMIT_MS = 500; |
| 5 | export const INPUT_P95_LIMIT_MS = 200; |
| 6 | |
| 7 | export function percentile(values, ratio = 0.95) { |
| 8 | return [...values].sort((a, b) => a - b)[Math.max(0, Math.ceil(values.length * ratio) - 1)] ?? 0; |
| 9 | } |
| 10 | |
| 11 | function validateAttempt(attempt) { |
| 12 | assert.ok(attempt && Number.isInteger(attempt.attempt), "performance attempt is missing its ordinal"); |
| 13 | assert.deepEqual(attempt.errors, [], `performance attempt ${attempt.attempt} reported browser errors`); |
| 14 | assert.ok(attempt.inputCount >= 30, `performance attempt ${attempt.attempt} has incomplete input samples`); |
| 15 | assert.ok(Number.isFinite(attempt.inputP95), `performance attempt ${attempt.attempt} is missing input P95`); |
| 16 | for (const phase of TRANSCRIPT_PHASES) { |
| 17 | const sample = attempt.phases?.[phase]; |
| 18 | assert.ok(sample && Number.isFinite(sample.elapsedMs), `performance attempt ${attempt.attempt} is missing ${phase}`); |
| 19 | assert.ok(Array.isArray(sample.longTasks), `performance attempt ${attempt.attempt} is missing ${phase} long tasks`); |
| 20 | assert.ok(Number.isFinite(sample.longTaskMax), `performance attempt ${attempt.attempt} is missing ${phase} maximum`); |
| 21 | } |
| 22 | } |
| 23 | |
| 24 | export function needsBoundedRetry(attempt) { |
| 25 | validateAttempt(attempt); |
| 26 | return attempt.inputP95 > INPUT_P95_LIMIT_MS |
| 27 | || (attempt.longTaskSupported |
| 28 | && TRANSCRIPT_PHASES.some(phase => attempt.phases[phase].longTaskMax > LONG_TASK_LIMIT_MS)); |
| 29 | } |
| 30 | |
| 31 | export function decideTranscriptPerformance(attempts) { |
| 32 | assert.ok(attempts.length === 1 || attempts.length === 3, "performance decision requires one or three attempts"); |
| 33 | attempts.forEach(validateAttempt); |
| 34 | const firstExceeded = needsBoundedRetry(attempts[0]); |
| 35 | if (!firstExceeded) { |
| 36 | assert.equal(attempts.length, 1, "passing first attempt must not be retried"); |
| 37 | return { |
| 38 | status: "passed-first-attempt", |
| 39 | passed: true, |
| 40 | medians: phaseMedians(attempts), |
| 41 | inputP95Median: inputP95Median(attempts), |
| 42 | }; |
| 43 | } |
| 44 | assert.equal(attempts.length, 3, "an over-limit first attempt requires exactly two retries"); |
| 45 | const medians = phaseMedians(attempts); |
| 46 | const inputMedian = inputP95Median(attempts); |
| 47 | const passed = inputMedian <= INPUT_P95_LIMIT_MS |
| 48 | && TRANSCRIPT_PHASES.every(phase => medians[phase] <= LONG_TASK_LIMIT_MS); |
| 49 | return { |
| 50 | status: passed ? "passed-after-bounded-retry" : "failed-sustained-regression", |
| 51 | passed, |
| 52 | medians, |
| 53 | inputP95Median: inputMedian, |
| 54 | }; |
| 55 | } |
| 56 | |
| 57 | export async function collectTranscriptPerformance(sampleAttempt) { |
| 58 | const first = await sampleAttempt(1); |
| 59 | validateAttempt(first); |
| 60 | const attempts = [first]; |
| 61 | if (needsBoundedRetry(attempts[0])) { |
| 62 | const second = await sampleAttempt(2); |
| 63 | validateAttempt(second); |
| 64 | attempts.push(second); |
| 65 | const third = await sampleAttempt(3); |
| 66 | validateAttempt(third); |
| 67 | attempts.push(third); |
| 68 | } |
| 69 | return { attempts, decision: decideTranscriptPerformance(attempts) }; |
| 70 | } |
| 71 | |
| 72 | function phaseMedians(attempts) { |
| 73 | return Object.fromEntries(TRANSCRIPT_PHASES.map(phase => [phase, |
| 74 | percentile(attempts.map(attempt => attempt.phases[phase].longTaskMax), 0.5)])); |
| 75 | } |
| 76 | |
| 77 | function inputP95Median(attempts) { |
| 78 | return percentile(attempts.map(attempt => attempt.inputP95), 0.5); |
| 79 | } |
| 80 | |
| 81 | export function formatPerformanceSummary(browser, turns, decision, attempts) { |
| 82 | const label = decision.status === "passed-first-attempt" ? "passed on the first attempt" |
| 83 | : decision.status === "passed-after-bounded-retry" ? "passed after bounded retry" |
| 84 | : "failed with a sustained regression"; |
| 85 | const medians = `${TRANSCRIPT_PHASES.map(phase => `${phase}=${decision.medians[phase]}ms`).join(", ")}, inputP95=${decision.inputP95Median}ms`; |
| 86 | const samples = attempts.map(attempt => `#${attempt.attempt} [${TRANSCRIPT_PHASES |
| 87 | .map(phase => `${phase}=${attempt.phases[phase].longTaskMax}ms`).join(", ")}, inputP95=${attempt.inputP95}ms]`).join("; "); |
| 88 | return `- ${browser}, ${turns} turns: **${label}**; medians: ${medians}; samples: ${samples}`; |
| 89 | } |
| 90 | |
| 91 | export async function installTranscriptPerformanceObserver(page) { |
| 92 | await page.addInitScript(() => { |
| 93 | window.chatMetrics = { inputs: [], tasks: [], longTaskSupported: PerformanceObserver.supportedEntryTypes.includes("longtask") }; |
| 94 | if (window.chatMetrics.longTaskSupported) { |
| 95 | new PerformanceObserver(list => window.chatMetrics.tasks.push(...list.getEntries().map(entry => ({ |
| 96 | startTime: entry.startTime, |
| 97 | duration: entry.duration, |
| 98 | })))).observe({ type: "longtask" }); |
| 99 | } |
| 100 | document.addEventListener("keydown", event => { |
| 101 | if (!event.target.matches("textarea.composer__input")) return; |
| 102 | // The first animation frame is the browser's next paint opportunity for |
| 103 | // this input. A second frame measures an unrelated scheduling interval |
| 104 | // and made the strict input gate depend on runner descheduling. |
| 105 | const start = event.timeStamp; |
| 106 | requestAnimationFrame(() => window.chatMetrics.inputs.push(performance.now() - start)); |
| 107 | }, true); |
| 108 | }); |
| 109 | } |
| 110 | |
| 111 | async function measurePhase(page, operation) { |
| 112 | const start = await page.evaluate(() => performance.now()); |
| 113 | const detail = await operation(); |
| 114 | const end = await page.evaluate(() => performance.now()); |
| 115 | await page.waitForTimeout(0); |
| 116 | const tasks = await page.evaluate(({ start, end }) => window.chatMetrics.tasks |
| 117 | .filter(task => task.startTime >= start && task.startTime < end), { start, end }); |
| 118 | return { |
| 119 | elapsedMs: end - start, |
| 120 | longTasks: tasks, |
| 121 | longTaskMax: Math.max(0, ...tasks.map(task => task.duration)), |
| 122 | ...detail, |
| 123 | }; |
| 124 | } |
| 125 | |
| 126 | export async function measureTranscriptPerformance({ page, turns, attempt, frame, errors }) { |
| 127 | const errorStart = errors.length; |
| 128 | await page.evaluate(() => { window.chatMetrics.inputs = []; window.chatMetrics.tasks = []; }); |
| 129 | const phases = {}; |
| 130 | phases.initialLoad = await measurePhase(page, async () => { |
| 131 | await page.evaluate(count => window.chatFixture.reset(count), turns); |
| 132 | await page.locator(`[data-chat-anchor-key="u${Math.max(0, turns - 60)}"][data-chat-kind="user"]`).waitFor(); |
| 133 | await frame(page); |
| 134 | return {}; |
| 135 | }); |
| 136 | phases.historyPaging = await measurePhase(page, async () => { |
| 137 | const pages = []; |
| 138 | for (let loaded = 60; loaded < turns; loaded += 60) { |
| 139 | const pageStart = Date.now(); |
| 140 | await page.evaluate(() => window.chatFixture.older()); |
| 141 | const nextLoaded = Math.min(turns, loaded + 60); |
| 142 | await page.locator(`[data-chat-anchor-key="u${turns - nextLoaded}"][data-chat-kind="user"]`).waitFor(); |
| 143 | await frame(page); |
| 144 | pages.push(Date.now() - pageStart); |
| 145 | } |
| 146 | assert.equal(await page.locator(".transcript__window-item").count(), 0); |
| 147 | assert.equal(await page.locator('[data-chat-kind="user"]').count(), turns, `${turns} turns fully mounted`); |
| 148 | return { pages }; |
| 149 | }); |
| 150 | phases.streaming = await measurePhase(page, async () => { |
| 151 | for (let index = 0; index < 30; index++) await page.evaluate(value => window.chatFixture.tick(value), index); |
| 152 | await frame(page); |
| 153 | return {}; |
| 154 | }); |
| 155 | const input = page.locator("textarea.composer__input:not(.composer__input--measure)"); |
| 156 | phases.input = await measurePhase(page, async () => { |
| 157 | await input.fill(""); |
| 158 | await page.evaluate(() => { window.chatMetrics.inputs = []; window.chatFixture.tick(0); }); |
| 159 | for (let index = 0; index < 30; index++) { |
| 160 | await page.evaluate(value => window.chatFixture.tick(value), index); |
| 161 | await input.press("a"); |
| 162 | } |
| 163 | await frame(page); |
| 164 | assert.equal(await input.inputValue(), "a".repeat(30)); |
| 165 | return {}; |
| 166 | }); |
| 167 | const metrics = await page.evaluate(() => window.chatMetrics); |
| 168 | const attemptErrors = errors.slice(errorStart); |
| 169 | const result = { |
| 170 | attempt, |
| 171 | turns, |
| 172 | phases, |
| 173 | inputs: metrics.inputs, |
| 174 | inputCount: metrics.inputs.length, |
| 175 | inputP95: percentile(metrics.inputs), |
| 176 | longTaskSupported: metrics.longTaskSupported, |
| 177 | errors: attemptErrors, |
| 178 | dom: await page.locator("*").count(), |
| 179 | }; |
| 180 | return result; |
| 181 | } |
| 182 |