| 1 | import assert from "node:assert/strict"; |
| 2 | import { |
| 3 | formatElapsedMs, outputQuarters, tokensFromQuarters, turnMetrics, unbilledOutputTokens, |
| 4 | } from "../lib/turnMetrics"; |
| 5 | |
| 6 | const eq = (actual: unknown, expected: unknown, label: string) => |
| 7 | assert.equal(actual, expected, label); |
| 8 | const ok = (value: unknown, label: string) => assert.ok(value, label); |
| 9 | |
| 10 | // The flat `chars / 4` this module replaces. Every ASCII fixture in the suite |
| 11 | // was written against this expression, so any divergence is a regression. |
| 12 | const legacyEstimate = (chars: number) => Math.round(chars / 4); |
| 13 | |
| 14 | // --- ASCII identity: the load-bearing guarantee ----------------------------- |
| 15 | for (const n of [0, 1, 3, 4, 5, 7, 8, 15, 16, 17, 40, 41, 99, 100, 101, 1_000, 4_003]) { |
| 16 | const s = "x".repeat(n); |
| 17 | eq(outputQuarters(s), n, `ASCII weights one quarter per char (n=${n})`); |
| 18 | eq(tokensFromQuarters(outputQuarters(s)), legacyEstimate(n), `ASCII estimate matches chars/4 (n=${n})`); |
| 19 | } |
| 20 | eq(tokensFromQuarters(0), 0, "empty output estimates zero"); |
| 21 | |
| 22 | // --- CJK density ------------------------------------------------------------ |
| 23 | // Han is three UTF-8 bytes, so it is worth three times the old flat density. |
| 24 | eq(outputQuarters("中"), 3, "a Han character carries three quarters"); |
| 25 | eq(tokensFromQuarters(outputQuarters("中".repeat(16))), 12, "16 Han chars estimate 12 tokens"); |
| 26 | eq(legacyEstimate(16), 4, "the legacy flat estimate would have said 4"); |
| 27 | ok(tokensFromQuarters(outputQuarters("中".repeat(16))) > legacyEstimate(16) * 2, |
| 28 | "CJK output is no longer underestimated threefold"); |
| 29 | eq(outputQuarters("é"), 2, "a two-byte rune carries two quarters"); |
| 30 | eq(outputQuarters("😀"), 4, "an astral pair carries four quarters and consumes both units"); |
| 31 | eq(outputQuarters("中".repeat(4), 2), 6, "the start index skips leading characters"); |
| 32 | |
| 33 | // Mixed scripts: density must sit between the ASCII floor and the Han ceiling. |
| 34 | const mixed = "hi 中文 there"; |
| 35 | const mixedQuarters = outputQuarters(mixed); |
| 36 | ok(mixedQuarters > mixed.length, "mixed text outweighs its character count"); |
| 37 | ok(mixedQuarters < mixed.length * 3, "mixed text stays below the pure-Han ceiling"); |
| 38 | eq(outputQuarters("中文abc"), outputQuarters("abc中文"), "weight is order-independent"); |
| 39 | |
| 40 | // --- Unbilled window -------------------------------------------------------- |
| 41 | const buf = (text: string, reasoning = "") => ({ text, reasoning }); |
| 42 | eq(unbilledOutputTokens(undefined, 0, 0), 0, "no buffers estimates nothing"); |
| 43 | eq(unbilledOutputTokens(buf("x".repeat(40)), 0, 0), 10, "whole ASCII buffer matches chars/4"); |
| 44 | eq(unbilledOutputTokens(buf("x".repeat(40)), 0, 0), legacyEstimate(40), "ASCII parity at the buffer level"); |
| 45 | eq(unbilledOutputTokens(buf("中".repeat(40)), 0, 0), 30, "whole CJK buffer is weighted"); |
| 46 | eq(unbilledOutputTokens(buf("x".repeat(40)), 16, 0), 6, "billed prefix is excluded from the estimate"); |
| 47 | eq(unbilledOutputTokens(buf("x".repeat(40)), 16, 0), legacyEstimate(24), "partial-bill ASCII parity"); |
| 48 | eq(unbilledOutputTokens(buf("", ""), 0, 40), 10, "argument characters are charged as ASCII"); |
| 49 | eq(unbilledOutputTokens(buf("x".repeat(8)), 0, 8), 4, "buffer and arguments sum before rounding"); |
| 50 | eq(unbilledOutputTokens(buf("x".repeat(4)), 99, 0), 0, "an over-billed buffer floors at zero"); |
| 51 | // Tail slice: all of `text` is billed before any of `reasoning`. |
| 52 | eq(unbilledOutputTokens(buf("x".repeat(20), "中".repeat(4)), 8, 0), 4, |
| 53 | "a billed prefix is consumed from the text buffer first"); |
| 54 | eq(unbilledOutputTokens(buf("x".repeat(20), "中".repeat(4)), 4, 0), 5, |
| 55 | "a window ending exactly at the text boundary stays pure ASCII weight"); |
| 56 | eq(unbilledOutputTokens(buf("x".repeat(20), "中".repeat(8)), 2, 0), 10, |
| 57 | "a window reaching into the reasoning tail picks up Han weight"); |
| 58 | ok(unbilledOutputTokens(buf("x".repeat(20), "中".repeat(8)), 2, 0) > legacyEstimate(26), |
| 59 | "the reasoning tail is no longer priced as ASCII"); |
| 60 | |
| 61 | // --- The consolidated derivation matches the inline expression it replaces --- |
| 62 | const legacyRunMetrics = (i: Parameters<typeof turnMetrics>[0]) => { |
| 63 | if (!i.turnStartAt || (!i.running && !i.turnDoneAt)) return null; |
| 64 | const metricsNow = i.turnDoneAt || i.now; |
| 65 | const elapsedMs = Math.max(0, metricsNow - i.turnStartAt |
| 66 | - (i.turnDoneAt ? i.lastTurnWaitAccumMs ?? i.waitAccumMs : i.waitAccumMs)); |
| 67 | const liveChars = (i.live?.text.length ?? 0) + (i.live?.reasoning.length ?? 0); |
| 68 | const inFlightChars = Math.max(0, liveChars - (i.turnOutputCharsAtUsage ?? 0)) + (i.turnArgChars ?? 0); |
| 69 | const estimatedChars = !i.turnDoneAt ? legacyEstimate(inFlightChars) |
| 70 | : Math.max(0, (i.lastTurnOutputTokens ?? i.turnOutputTokens ?? 0) - (i.turnOutputTokens ?? 0)); |
| 71 | const outTok = (i.turnOutputTokens ?? 0) + estimatedChars; |
| 72 | const modelActiveAt = i.liveModelActiveAt ?? i.turnModelActiveAt; |
| 73 | const modelElapsedMs = Math.max(0, i.turnModelActiveMs |
| 74 | + (modelActiveAt && modelActiveAt > 0 ? Math.max(0, metricsNow - modelActiveAt) : 0)); |
| 75 | const tps = outTok > 0 && modelElapsedMs >= 500 ? Math.round(outTok / (modelElapsedMs / 1000)) : null; |
| 76 | return { tokens: (i.turnTokens ?? 0) + estimatedChars, outTok, elapsedMs, tps }; |
| 77 | }; |
| 78 | const base = { |
| 79 | now: 60_000, turnStartAt: 1_000, turnDoneAt: undefined, running: true, waitAccumMs: 0, |
| 80 | turnTokens: 8, turnOutputTokens: 10, turnModelActiveMs: 2_000, |
| 81 | turnOutputCharsAtUsage: 0, turnArgChars: 0, live: buf("x".repeat(40)), |
| 82 | }; |
| 83 | for (const fixture of [ |
| 84 | base, |
| 85 | { ...base, turnDoneAt: 20_000, lastTurnOutputTokens: 24, lastTurnWaitAccumMs: 5_000 }, |
| 86 | { ...base, waitAccumMs: 3_000, turnModelActiveAt: 55_000, liveModelActiveAt: 58_000 }, |
| 87 | { ...base, turnModelActiveMs: 400, live: undefined }, |
| 88 | { ...base, turnOutputTokens: 0, turnTokens: 0, live: buf("", "") }, |
| 89 | { ...base, live: buf("x".repeat(40), "y".repeat(8)), turnArgChars: 12 }, |
| 90 | ]) { |
| 91 | const next = turnMetrics(fixture); |
| 92 | const legacy = legacyRunMetrics(fixture); |
| 93 | ok(next && legacy, "both derivations produce a reading for the ASCII fixture"); |
| 94 | eq(next!.tokens, legacy!.tokens, "token reading matches the inline expression"); |
| 95 | eq(next!.outputTokens, legacy!.outTok, "output reading matches the inline expression"); |
| 96 | eq(next!.elapsedMs, legacy!.elapsedMs, "elapsed reading matches the inline expression"); |
| 97 | eq(next!.tps, legacy!.tps, "throughput reading matches the inline expression"); |
| 98 | } |
| 99 | eq(turnMetrics({ ...base, turnStartAt: 0 }), null, "a turn without a start anchor has no reading"); |
| 100 | eq(turnMetrics({ ...base, running: false, turnDoneAt: undefined }), null, |
| 101 | "neither running nor completed yields no reading"); |
| 102 | eq(turnMetrics({ ...base, running: false, turnDoneAt: 20_000 })!.tps, 5, |
| 103 | "a settled turn divides its billed output by the model-active window"); |
| 104 | eq(turnMetrics(base)!.tps, 10, "a streaming turn adds the in-flight estimate to the numerator"); |
| 105 | eq(turnMetrics({ ...base, turnModelActiveMs: 400 })!.tps, null, "sub-500ms windows report no throughput"); |
| 106 | eq(turnMetrics({ ...base, running: false, turnDoneAt: 20_000, lastTurnOutputEstimated: true })!.estimated, |
| 107 | true, "a settled estimate is flagged"); |
| 108 | eq(turnMetrics(base)!.estimated, true, "a streaming estimate is flagged"); |
| 109 | eq(turnMetrics({ ...base, turnOutputEstimated: false, live: buf("") })!.estimated, false, |
| 110 | "a stream with nothing estimated is not flagged"); |
| 111 | |
| 112 | // --- Elapsed label ---------------------------------------------------------- |
| 113 | eq(formatElapsedMs(0), "0s", "zero seconds"); |
| 114 | eq(formatElapsedMs(4_999), "4s", "sub-minute truncates"); |
| 115 | eq(formatElapsedMs(20_000), "20s", "seconds"); |
| 116 | eq(formatElapsedMs(60_000), "1m 0s", "minute boundary"); |
| 117 | eq(formatElapsedMs(83_421), "1m 23s", "minutes and seconds"); |
| 118 | |
| 119 | console.log("turn metrics: all assertions passed"); |
| 120 |