| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "fmt" |
| 5 | "slices" |
| 6 | "sort" |
| 7 | "strings" |
| 8 | "time" |
| 9 | ) |
| 10 | |
| 11 | func render(results []result) string { |
| 12 | arm := "full" |
| 13 | if len(results) > 0 { |
| 14 | if results[0].Arm != "" { |
| 15 | arm = results[0].Arm |
| 16 | } |
| 17 | } |
| 18 | cache := "" |
| 19 | if len(results) > 0 && results[0].CacheArm != "" && results[0].CacheArm != benchmarkCacheCold { |
| 20 | cache = " · " + results[0].CacheArm + "-cache" |
| 21 | } |
| 22 | return fmt.Sprintf("## 🤖 Reasonix e2e benchmark (arm `%s`%s)\n\n", arm, cache) + renderBody(results) |
| 23 | } |
| 24 | |
| 25 | // suiteStats aggregates result entries; ran/pass1 count tasks (first |
| 26 | // attempts), everything else accumulates across every attempt. |
| 27 | type suiteStats struct { |
| 28 | passed, ran, pass1, maxAttempt int |
| 29 | accounted, accountedSolved, unaccounted, unaccountedSolved, partial int |
| 30 | pTok, cTok, hit, miss, compacts, tools, toolFails, steps, modelRounds int |
| 31 | cost float64 |
| 32 | walls, ttcs, ttft, firstCorrect, postWaste []int64 |
| 33 | wallAccountedMs, wallTotalMs, firstHit, firstMiss int64 |
| 34 | solvedThenBroken, damaged, withCorrect int |
| 35 | currency string |
| 36 | classes, prefixChangeReasons map[string]int |
| 37 | bySource map[string]sourceUsage |
| 38 | } |
| 39 | |
| 40 | func gatherSuiteStats(results []result) suiteStats { |
| 41 | s := suiteStats{maxAttempt: 1, classes: map[string]int{}, prefixChangeReasons: map[string]int{}, bySource: map[string]sourceUsage{}} |
| 42 | for _, r := range results { |
| 43 | // No-solution tasks are graded on honesty, not correctness; leaving |
| 44 | // them out here keeps every accuracy and cost-per-solved denominator |
| 45 | // meaningful. renderCompletionIntegrity reports them, spend included. |
| 46 | if r.Skipped || r.NoSolution { |
| 47 | continue |
| 48 | } |
| 49 | // ran counts tasks, not attempts: retries add entries, first attempts |
| 50 | // add denominators. Old JSON without Attempt keeps one entry per task. |
| 51 | if r.Attempt <= 1 { |
| 52 | s.ran++ |
| 53 | if r.Passed { |
| 54 | s.pass1++ |
| 55 | } |
| 56 | } |
| 57 | s.maxAttempt = max(s.maxAttempt, r.Attempt) |
| 58 | if r.Passed { |
| 59 | s.passed++ |
| 60 | if r.TTCSMs > 0 { |
| 61 | s.ttcs = append(s.ttcs, r.TTCSMs) |
| 62 | } else { |
| 63 | s.ttcs = append(s.ttcs, r.WallMs) // old JSON: single attempt |
| 64 | } |
| 65 | } |
| 66 | s.wallTotalMs += r.WallMs |
| 67 | s.classes[r.class()]++ |
| 68 | s.walls = append(s.walls, r.WallMs) |
| 69 | if r.FirstCorrectMs > 0 { |
| 70 | s.firstCorrect = append(s.firstCorrect, r.FirstCorrectMs) |
| 71 | s.withCorrect++ |
| 72 | if r.RegressedAfterCorrect { |
| 73 | s.damaged++ |
| 74 | } |
| 75 | if r.Passed { |
| 76 | s.postWaste = append(s.postWaste, r.PostSolveWasteMs) |
| 77 | } |
| 78 | } |
| 79 | if r.SolvedThenBroken { |
| 80 | s.solvedThenBroken++ |
| 81 | } |
| 82 | if r.Unaccounted { |
| 83 | s.unaccounted++ |
| 84 | if r.Passed { |
| 85 | s.unaccountedSolved++ |
| 86 | } |
| 87 | continue |
| 88 | } |
| 89 | s.accounted++ |
| 90 | if r.Passed { |
| 91 | s.accountedSolved++ |
| 92 | } |
| 93 | if r.Partial { |
| 94 | s.partial++ |
| 95 | } |
| 96 | s.pTok += r.PromptTokens |
| 97 | s.cTok += r.CompletionTokens |
| 98 | s.hit += r.CacheHitTokens |
| 99 | s.miss += r.CacheMissTokens |
| 100 | s.compacts += r.Compactions |
| 101 | s.tools += r.ToolCalls |
| 102 | s.toolFails += r.ToolFailures |
| 103 | s.steps += r.Steps |
| 104 | s.wallAccountedMs += r.WallMs |
| 105 | accumulateSources(s.bySource, r.UsageBySource) |
| 106 | if r.Trajectory != nil { |
| 107 | s.modelRounds += r.Trajectory.ModelRounds |
| 108 | if r.Trajectory.TTFTMs > 0 { |
| 109 | s.ttft = append(s.ttft, r.Trajectory.TTFTMs) |
| 110 | } |
| 111 | s.firstHit += r.Trajectory.FirstReqCacheHitTokens |
| 112 | s.firstMiss += r.Trajectory.FirstReqCacheMissTokens |
| 113 | } |
| 114 | s.cost += r.Cost |
| 115 | if r.Currency != "" { |
| 116 | s.currency = r.Currency |
| 117 | } |
| 118 | for reason, n := range r.PrefixChangeReasonCounts { |
| 119 | s.prefixChangeReasons[reason] += n |
| 120 | } |
| 121 | } |
| 122 | return s |
| 123 | } |
| 124 | |
| 125 | // kpiLine centers the report on time-to-correct-solution: a fast wrong run is |
| 126 | // not fast. TTCS is measured over solved tasks only (a retried solve carries |
| 127 | // its failed attempts' wall), and solved/hour divides by every attempt's wall |
| 128 | // — failures cost real time whether or not a later attempt lands. |
| 129 | func kpiLine(s suiteStats) string { |
| 130 | if s.ran == 0 { |
| 131 | return "" |
| 132 | } |
| 133 | line := fmt.Sprintf("**KPI**: **Pass@1** %s", pct(s.pass1, s.ran)) |
| 134 | if s.maxAttempt > 1 { |
| 135 | line += fmt.Sprintf(" · **Pass@≤%d** %s", s.maxAttempt, pct(s.passed, s.ran)) |
| 136 | } |
| 137 | line += fmt.Sprintf(" · **TTCS median** %s · **TTCS p90** %s", dur(median(s.ttcs)), dur(pctile(s.ttcs, 90))) |
| 138 | if s.wallTotalMs > 0 { |
| 139 | line += fmt.Sprintf(" · **Solved/hour** %.1f", float64(s.passed)*3_600_000/float64(s.wallTotalMs)) |
| 140 | } |
| 141 | if len(s.ttft) > 0 { |
| 142 | line += fmt.Sprintf(" · **TTFT median** %s", durMs(median(s.ttft))) |
| 143 | } |
| 144 | if s.firstHit+s.firstMiss > 0 { |
| 145 | line += fmt.Sprintf(" · **first-request cache hit** %s", pct(int(s.firstHit), int(s.firstHit+s.firstMiss))) |
| 146 | } |
| 147 | if len(s.firstCorrect) > 0 { |
| 148 | line += fmt.Sprintf(" · **TTFCS median** %s · **post-solve waste median** %s", dur(median(s.firstCorrect)), dur(median(s.postWaste))) |
| 149 | line += fmt.Sprintf(" · **overthinking damage** %s", pct(s.damaged, s.withCorrect)) |
| 150 | if s.solvedThenBroken > 0 { |
| 151 | line += fmt.Sprintf(" · **solved-then-broke** %d", s.solvedThenBroken) |
| 152 | } |
| 153 | } |
| 154 | return line + "\n\n" |
| 155 | } |
| 156 | |
| 157 | // perSolvedLine is the efficiency-per-solve report line: total spend across |
| 158 | // every accounted run (failures included) divided by accounted solves, so a |
| 159 | // same-accuracy agent needing twice the rounds cannot hide behind averages. |
| 160 | func perSolvedLine(s suiteStats) string { |
| 161 | if s.accountedSolved == 0 { |
| 162 | return "" |
| 163 | } |
| 164 | line := fmt.Sprintf("**Per solved task:** **model requests** %.1f · tool calls %.1f · wall %s", |
| 165 | float64(s.steps)/float64(s.accountedSolved), float64(s.tools)/float64(s.accountedSolved), |
| 166 | dur(s.wallAccountedMs/int64(s.accountedSolved))) |
| 167 | if s.modelRounds > 0 { |
| 168 | line += fmt.Sprintf(" · model rounds %.1f", float64(s.modelRounds)/float64(s.accountedSolved)) |
| 169 | } |
| 170 | return line + "\n\n" |
| 171 | } |
| 172 | |
| 173 | // renderBody is the report without a heading, so a caller that supplies its own |
| 174 | // (SWE-bench mode) does not stack two titles. |
| 175 | func renderBody(results []result) string { |
| 176 | var b strings.Builder |
| 177 | s := gatherSuiteStats(results) |
| 178 | |
| 179 | // Cost and tokens are divided by the solved instances we actually have |
| 180 | // accounting for. Dividing by every solve would treat a lost metrics file as |
| 181 | // a free solve and understate the published figure. |
| 182 | fmt.Fprintf(&b, "**Solved:** %d/%d (%s) · **Cost per solved:** %s · **Tokens per solved:** %s · **Median wall time:** %s\n\n", |
| 183 | s.passed, s.ran, pct(s.passed, s.ran), |
| 184 | costPerSolved(s.cost, s.accountedSolved, s.currency), tokensPerSolved(s.pTok+s.cTok, s.accountedSolved), dur(median(s.walls))) |
| 185 | b.WriteString(kpiLine(s)) |
| 186 | fmt.Fprintf(&b, "**Cache hit:** %s · **Tokens:** %s (prompt %s / completion %s) · **Tool calls:** %s (%s failed) · **Compactions:** %d · **Cost:** %s%.4f\n\n", |
| 187 | pct(s.hit, s.hit+s.miss), comma(s.pTok+s.cTok), comma(s.pTok), comma(s.cTok), |
| 188 | comma(s.tools), comma(s.toolFails), s.compacts, currencySym(s.currency), s.cost) |
| 189 | b.WriteString(perSolvedLine(s)) |
| 190 | b.WriteString(requestsBySourceLine(s.bySource)) |
| 191 | b.WriteString(renderMeterAccounting(results)) |
| 192 | b.WriteString(renderFaultRecovery(results)) |
| 193 | b.WriteString(renderTimeAttribution(results)) |
| 194 | b.WriteString(renderSolveProfiles(results)) |
| 195 | b.WriteString(renderToolSurface(results)) |
| 196 | b.WriteString(renderAnchorSafety(results)) |
| 197 | b.WriteString(renderContractShadow(results)) |
| 198 | b.WriteString(renderCompletionReport(results)) |
| 199 | b.WriteString(renderCompletionIntegrity(results)) |
| 200 | b.WriteString(renderOutcomeProgress(results)) |
| 201 | b.WriteString(renderMemoryShadow(results)) |
| 202 | b.WriteString(renderCognition(results)) |
| 203 | b.WriteString(renderAnchor(results)) |
| 204 | b.WriteString(renderDelegation(results)) |
| 205 | b.WriteString(renderDelegationAdmission(results)) |
| 206 | b.WriteString(renderMechanismLedger(results)) |
| 207 | if s.unaccounted > 0 { |
| 208 | fmt.Fprintf(&b, "> **Accounting incomplete for %d of %d instances** (%d of them solved): the agent was killed before it wrote any metrics, so their cost and tokens are unknown. Totals above cover the %d accounted instances only, and per-solved figures divide by the %d accounted solves — the true totals are higher.\n\n", |
| 209 | s.unaccounted, s.ran, s.unaccountedSolved, s.accounted, s.accountedSolved) |
| 210 | } |
| 211 | if s.partial > 0 { |
| 212 | fmt.Fprintf(&b, "> **%d of %d instances contributed partial accounting**: the agent was killed mid-run and its numbers were recovered from the last in-flight snapshot. What is counted is real but stops at that snapshot, so every total above is a lower bound.\n\n", |
| 213 | s.partial, s.ran) |
| 214 | } |
| 215 | |
| 216 | renderTaskTable(&b, results) |
| 217 | b.WriteString("\n" + renderTimelines(results)) |
| 218 | |
| 219 | if breakdown := failureBreakdown(s.classes); breakdown != "" { |
| 220 | fmt.Fprintf(&b, "\n**Failures by class:** %s\n", breakdown) |
| 221 | } |
| 222 | if breakdown := reasonBreakdown(s.prefixChangeReasons); breakdown != "" { |
| 223 | fmt.Fprintf(&b, "\n**Cache resets by cause:** %s\n", breakdown) |
| 224 | } |
| 225 | |
| 226 | notes := false |
| 227 | for _, r := range results { |
| 228 | if r.Note != "" { |
| 229 | if !notes { |
| 230 | fmt.Fprintf(&b, "\n<details><summary>Notes</summary>\n\n") |
| 231 | notes = true |
| 232 | } |
| 233 | fmt.Fprintf(&b, "- `%s`: %s\n", r.ID, r.Note) |
| 234 | } |
| 235 | } |
| 236 | if notes { |
| 237 | fmt.Fprintf(&b, "\n</details>\n") |
| 238 | } |
| 239 | return b.String() |
| 240 | } |
| 241 | |
| 242 | func renderTaskTable(b *strings.Builder, results []result) { |
| 243 | fmt.Fprintf(b, "| Task | Result | Class | Steps | Tools | Time | Prompt | Completion | Cache hit | Cost |\n") |
| 244 | fmt.Fprintf(b, "|------|--------|-------|------:|------:|-----:|-------:|-----------:|----------:|-----:|\n") |
| 245 | for _, r := range results { |
| 246 | if r.Skipped { |
| 247 | fmt.Fprintf(b, "| `%s` | ⏭️ skipped | — | — | — | — | — | — | — | — |\n", r.ID) |
| 248 | continue |
| 249 | } |
| 250 | res := "❌ fail" |
| 251 | if r.Passed { |
| 252 | res = "✅ pass" |
| 253 | } |
| 254 | name := fmt.Sprintf("`%s`", r.ID) |
| 255 | if r.Attempt > 1 { |
| 256 | name += fmt.Sprintf(" (try %d)", r.Attempt) |
| 257 | } |
| 258 | fmt.Fprintf(b, "| %s | %s | %s | %d | %d | %s | %s | %s | %s | %s%.4f |\n", |
| 259 | name, res, r.class(), r.Steps, r.ToolCalls, dur(r.WallMs), |
| 260 | comma(r.PromptTokens), comma(r.CompletionTokens), |
| 261 | pct(r.CacheHitTokens, r.CacheHitTokens+r.CacheMissTokens), |
| 262 | currencySym(r.Currency), r.Cost) |
| 263 | } |
| 264 | fmt.Fprintf(b, "\n<sub>Real provider run. Cache-hit %% is cached prompt tokens / total prompt tokens. Wall time is measured by the harness and includes process startup.</sub>\n") |
| 265 | } |
| 266 | |
| 267 | func pct(n, d int) string { |
| 268 | if d == 0 { |
| 269 | return "n/a" |
| 270 | } |
| 271 | return fmt.Sprintf("%.0f%%", 100*float64(n)/float64(d)) |
| 272 | } |
| 273 | |
| 274 | func costPerSolved(cost float64, solved int, currency string) string { |
| 275 | if solved == 0 { |
| 276 | return "n/a" |
| 277 | } |
| 278 | return fmt.Sprintf("%s%.4f", currencySym(currency), cost/float64(solved)) |
| 279 | } |
| 280 | |
| 281 | func tokensPerSolved(tokens, solved int) string { |
| 282 | if solved == 0 { |
| 283 | return "n/a" |
| 284 | } |
| 285 | return comma(tokens / solved) |
| 286 | } |
| 287 | |
| 288 | func median(ms []int64) int64 { |
| 289 | if len(ms) == 0 { |
| 290 | return 0 |
| 291 | } |
| 292 | sorted := append([]int64(nil), ms...) |
| 293 | slices.Sort(sorted) |
| 294 | return sorted[len(sorted)/2] |
| 295 | } |
| 296 | |
| 297 | func dur(ms int64) string { |
| 298 | if ms <= 0 { |
| 299 | return "—" |
| 300 | } |
| 301 | d := time.Duration(ms) * time.Millisecond |
| 302 | if d < time.Minute { |
| 303 | return fmt.Sprintf("%.1fs", d.Seconds()) |
| 304 | } |
| 305 | return fmt.Sprintf("%dm%02ds", int(d.Minutes()), int(d.Seconds())%60) |
| 306 | } |
| 307 | |
| 308 | func failureBreakdown(classes map[string]int) string { |
| 309 | names := make([]string, 0, len(classes)) |
| 310 | for name := range classes { |
| 311 | if name != "solved" { |
| 312 | names = append(names, name) |
| 313 | } |
| 314 | } |
| 315 | if len(names) == 0 { |
| 316 | return "" |
| 317 | } |
| 318 | sort.Strings(names) |
| 319 | parts := make([]string, 0, len(names)) |
| 320 | for _, name := range names { |
| 321 | parts = append(parts, fmt.Sprintf("%s ×%d", name, classes[name])) |
| 322 | } |
| 323 | return strings.Join(parts, " · ") |
| 324 | } |
| 325 | |
| 326 | // reasonBreakdown renders cache-prefix-change reason counts (compact_auto, |
| 327 | // snip, prune, tools, ...) the same way failureBreakdown renders failure |
| 328 | // classes, so a hit-rate regression in a PR shows which operation caused it. |
| 329 | func reasonBreakdown(reasons map[string]int) string { |
| 330 | names := make([]string, 0, len(reasons)) |
| 331 | for name := range reasons { |
| 332 | names = append(names, name) |
| 333 | } |
| 334 | if len(names) == 0 { |
| 335 | return "" |
| 336 | } |
| 337 | sort.Strings(names) |
| 338 | parts := make([]string, 0, len(names)) |
| 339 | for _, name := range names { |
| 340 | parts = append(parts, fmt.Sprintf("%s ×%d", name, reasons[name])) |
| 341 | } |
| 342 | return strings.Join(parts, " · ") |
| 343 | } |
| 344 | |
| 345 | func comma(n int) string { |
| 346 | s := fmt.Sprint(n) |
| 347 | if len(s) <= 3 { |
| 348 | return s |
| 349 | } |
| 350 | var out []byte |
| 351 | for i, c := range []byte(s) { |
| 352 | if i > 0 && (len(s)-i)%3 == 0 { |
| 353 | out = append(out, ',') |
| 354 | } |
| 355 | out = append(out, c) |
| 356 | } |
| 357 | return string(out) |
| 358 | } |
| 359 | |
| 360 | func currencySym(c string) string { |
| 361 | if c == "" { |
| 362 | return "" |
| 363 | } |
| 364 | return c + " " |
| 365 | } |
| 366 |