| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "encoding/json" |
| 5 | "flag" |
| 6 | "fmt" |
| 7 | "os" |
| 8 | "path/filepath" |
| 9 | "sort" |
| 10 | "strings" |
| 11 | ) |
| 12 | |
| 13 | // requestsBySourceLine breaks total model requests down by origin so an |
| 14 | // ablation arm shows exactly where its requests went (planner, subagents, |
| 15 | // compaction) instead of one opaque total. |
| 16 | func requestsBySourceLine(bySource map[string]sourceUsage) string { |
| 17 | if len(bySource) == 0 { |
| 18 | return "" |
| 19 | } |
| 20 | sources := make([]string, 0, len(bySource)) |
| 21 | for source, usage := range bySource { |
| 22 | if usage.Calls > 0 { |
| 23 | sources = append(sources, source) |
| 24 | } |
| 25 | } |
| 26 | if len(sources) == 0 { |
| 27 | return "" |
| 28 | } |
| 29 | sort.Slice(sources, func(i, j int) bool { |
| 30 | if bySource[sources[i]].Calls != bySource[sources[j]].Calls { |
| 31 | return bySource[sources[i]].Calls > bySource[sources[j]].Calls |
| 32 | } |
| 33 | return sources[i] < sources[j] |
| 34 | }) |
| 35 | parts := make([]string, 0, len(sources)) |
| 36 | for _, source := range sources { |
| 37 | usage := bySource[source] |
| 38 | parts = append(parts, fmt.Sprintf("%s %s (%s tok)", source, comma(usage.Calls), comma(usage.PromptTokens+usage.CompletionTokens))) |
| 39 | } |
| 40 | return "**Requests by source:** " + strings.Join(parts, " · ") + "\n\n" |
| 41 | } |
| 42 | |
| 43 | // armStats is one arm's aggregate over a -json report, using the same |
| 44 | // accounting conventions as renderBody: spend totals cover accounted runs |
| 45 | // (failures included) and per-solved figures divide by accounted solves. |
| 46 | type armStats struct { |
| 47 | Ran, Pass1, Solved, AccountedSolved int |
| 48 | Steps, Tools, Rounds, PlannerCalls int |
| 49 | Tokens, Hit, Miss int |
| 50 | Cost float64 |
| 51 | WallMs int64 |
| 52 | FirstHit, FirstMiss int64 |
| 53 | Damaged, WithCorrect int |
| 54 | TTCS, TTFT []int64 |
| 55 | ByClass map[string]classStats |
| 56 | } |
| 57 | |
| 58 | type classStats struct { |
| 59 | Ran, Solved int |
| 60 | WallMs int64 |
| 61 | TTCS []int64 |
| 62 | } |
| 63 | |
| 64 | func aggregateArm(results []result) armStats { |
| 65 | s := armStats{ByClass: map[string]classStats{}} |
| 66 | for _, r := range results { |
| 67 | // No-solution tasks never enter an accuracy comparison; see |
| 68 | // gatherSuiteStats. |
| 69 | if r.Skipped || r.NoSolution { |
| 70 | continue |
| 71 | } |
| 72 | // Retry entries share their task's denominator: only first attempts |
| 73 | // count into Ran, matching renderBody's task-not-attempt convention. |
| 74 | if r.Attempt <= 1 { |
| 75 | s.Ran++ |
| 76 | if r.Passed { |
| 77 | s.Pass1++ |
| 78 | } |
| 79 | } |
| 80 | if r.Passed { |
| 81 | s.Solved++ |
| 82 | if r.TTCSMs > 0 { |
| 83 | s.TTCS = append(s.TTCS, r.TTCSMs) |
| 84 | } else { |
| 85 | s.TTCS = append(s.TTCS, r.WallMs) |
| 86 | } |
| 87 | } |
| 88 | label := r.Class |
| 89 | if label == "" { |
| 90 | label = "unclassified" |
| 91 | } |
| 92 | c := s.ByClass[label] |
| 93 | if r.Attempt <= 1 { |
| 94 | c.Ran++ |
| 95 | } |
| 96 | if r.Passed { |
| 97 | c.Solved++ |
| 98 | if r.TTCSMs > 0 { |
| 99 | c.TTCS = append(c.TTCS, r.TTCSMs) |
| 100 | } else { |
| 101 | c.TTCS = append(c.TTCS, r.WallMs) |
| 102 | } |
| 103 | } |
| 104 | c.WallMs += r.WallMs |
| 105 | s.ByClass[label] = c |
| 106 | if r.Unaccounted { |
| 107 | continue |
| 108 | } |
| 109 | if r.Passed { |
| 110 | s.AccountedSolved++ |
| 111 | } |
| 112 | s.Steps += r.Steps |
| 113 | s.Tools += r.ToolCalls |
| 114 | s.Tokens += r.PromptTokens + r.CompletionTokens |
| 115 | s.Hit += r.CacheHitTokens |
| 116 | s.Miss += r.CacheMissTokens |
| 117 | s.Cost += r.Cost |
| 118 | s.WallMs += r.WallMs |
| 119 | s.PlannerCalls += r.UsageBySource["planner"].Calls |
| 120 | if r.Trajectory != nil { |
| 121 | s.Rounds += r.Trajectory.ModelRounds |
| 122 | if r.Trajectory.TTFTMs > 0 { |
| 123 | s.TTFT = append(s.TTFT, r.Trajectory.TTFTMs) |
| 124 | } |
| 125 | s.FirstHit += r.Trajectory.FirstReqCacheHitTokens |
| 126 | s.FirstMiss += r.Trajectory.FirstReqCacheMissTokens |
| 127 | } |
| 128 | if r.FirstCorrectMs > 0 { |
| 129 | s.WithCorrect++ |
| 130 | if r.RegressedAfterCorrect { |
| 131 | s.Damaged++ |
| 132 | } |
| 133 | } |
| 134 | } |
| 135 | return s |
| 136 | } |
| 137 | |
| 138 | func perSolved(total float64, solved int) string { |
| 139 | if solved == 0 { |
| 140 | return "—" |
| 141 | } |
| 142 | return fmt.Sprintf("%.1f", total/float64(solved)) |
| 143 | } |
| 144 | |
| 145 | func runCompareMode(outMD string) { |
| 146 | if flag.NArg() < 2 { |
| 147 | fmt.Fprintln(os.Stderr, "compare mode wants two or more -json report files: e2ebench -mode compare a.json b.json [c.json ...]") |
| 148 | os.Exit(2) |
| 149 | } |
| 150 | var report string |
| 151 | var err error |
| 152 | if flag.NArg() == 2 { |
| 153 | report, err = compareReports(flag.Arg(0), flag.Arg(1)) |
| 154 | } else { |
| 155 | report, err = multiCompareReport(flag.Args()) |
| 156 | } |
| 157 | if err != nil { |
| 158 | fmt.Fprintln(os.Stderr, "compare:", err) |
| 159 | os.Exit(1) |
| 160 | } |
| 161 | emit(report, outMD, "") |
| 162 | } |
| 163 | |
| 164 | func loadArm(path string) (armStats, error) { |
| 165 | data, err := os.ReadFile(path) |
| 166 | if err != nil { |
| 167 | return armStats{}, err |
| 168 | } |
| 169 | var results []result |
| 170 | if err := json.Unmarshal(data, &results); err != nil { |
| 171 | return armStats{}, fmt.Errorf("%s: %w", path, err) |
| 172 | } |
| 173 | return aggregateArm(results), nil |
| 174 | } |
| 175 | |
| 176 | // multiCompareReport is the N-arm readout: one KPI row per arm, then the |
| 177 | // Pareto section — the question for a lineup is frontier position, not |
| 178 | // pairwise deltas. |
| 179 | func multiCompareReport(paths []string) (string, error) { |
| 180 | var b strings.Builder |
| 181 | fmt.Fprintf(&b, "## e2ebench comparison: %d arms\n\n", len(paths)) |
| 182 | b.WriteString("| Arm | Pass@1 | Solved | TTFT | TTCS median | TTCS p90 | Solved/hour | 1st-req cache | Requests/solved | Tokens/solved | Cost/solved |\n") |
| 183 | b.WriteString("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") |
| 184 | points := make([]paretoPoint, 0, len(paths)) |
| 185 | arms := make([]armStats, 0, len(paths)) |
| 186 | for _, path := range paths { |
| 187 | s, err := loadArm(path) |
| 188 | if err != nil { |
| 189 | return "", err |
| 190 | } |
| 191 | arms = append(arms, s) |
| 192 | p := newParetoPoint(path, s) |
| 193 | points = append(points, p) |
| 194 | solvedPerHour := "—" |
| 195 | if s.WallMs > 0 { |
| 196 | solvedPerHour = fmt.Sprintf("%.1f", float64(s.Solved)*3_600_000/float64(s.WallMs)) |
| 197 | } |
| 198 | cost := "—" |
| 199 | if s.AccountedSolved > 0 { |
| 200 | cost = fmt.Sprintf("%.4f", s.Cost/float64(s.AccountedSolved)) |
| 201 | } |
| 202 | fmt.Fprintf(&b, "| `%s` | %s | %d/%d | %s | %s | %s | %s | %s | %s | %s | %s |\n", |
| 203 | p.label, pct(s.Pass1, s.Ran), s.Solved, s.Ran, durMs(median(s.TTFT)), |
| 204 | dur(median(s.TTCS)), dur(pctile(s.TTCS, 90)), solvedPerHour, |
| 205 | pct(int(s.FirstHit), int(s.FirstHit+s.FirstMiss)), |
| 206 | perSolved(float64(s.Steps), s.AccountedSolved), |
| 207 | tokensPerSolved(s.Tokens, s.AccountedSolved), cost) |
| 208 | } |
| 209 | b.WriteString("\n" + paretoSection(points)) |
| 210 | b.WriteString(perClassWinners(paths, arms)) |
| 211 | b.WriteString("<sub>Per-solved figures divide each arm's accounted totals (failures included) by its accounted solves; TTCS charges a retried solve with its failed attempts' wall.</sub>\n") |
| 212 | return b.String(), nil |
| 213 | } |
| 214 | |
| 215 | // perClassWinners is the routing readout: per task class, each arm's solve |
| 216 | // rate and TTCS median, and the winner (best solve rate, ties to the faster |
| 217 | // arm). A global default hides exactly this — the class that a leaner arm |
| 218 | // wins outright is a host-side routing opportunity, no classifier call needed. |
| 219 | func perClassWinners(paths []string, arms []armStats) string { |
| 220 | classes := map[string]bool{} |
| 221 | for _, a := range arms { |
| 222 | for class := range a.ByClass { |
| 223 | if class != "unclassified" { |
| 224 | classes[class] = true |
| 225 | } |
| 226 | } |
| 227 | } |
| 228 | if len(classes) == 0 || len(arms) < 2 { |
| 229 | return "" |
| 230 | } |
| 231 | names := make([]string, 0, len(classes)) |
| 232 | for class := range classes { |
| 233 | names = append(names, class) |
| 234 | } |
| 235 | sort.Strings(names) |
| 236 | |
| 237 | var b strings.Builder |
| 238 | b.WriteString("### Per-class winners\n\n| Class |") |
| 239 | labels := make([]string, len(paths)) |
| 240 | for i, path := range paths { |
| 241 | labels[i] = strings.TrimSuffix(filepath.Base(path), ".json") |
| 242 | fmt.Fprintf(&b, " `%s` |", labels[i]) |
| 243 | } |
| 244 | b.WriteString(" Winner |\n|---|") |
| 245 | b.WriteString(strings.Repeat("---:|", len(paths)) + "---|\n") |
| 246 | for _, class := range names { |
| 247 | fmt.Fprintf(&b, "| %s |", class) |
| 248 | winner, bestSolve, bestTTCS := "—", -1.0, int64(0) |
| 249 | for i, a := range arms { |
| 250 | c := a.ByClass[class] |
| 251 | if c.Ran == 0 { |
| 252 | b.WriteString(" — |") |
| 253 | continue |
| 254 | } |
| 255 | ttcs := median(c.TTCS) |
| 256 | fmt.Fprintf(&b, " %s · %s |", pct(c.Solved, c.Ran), dur(ttcs)) |
| 257 | solve := float64(c.Solved) / float64(c.Ran) |
| 258 | if solve > bestSolve || (solve == bestSolve && c.Solved > 0 && ttcs < bestTTCS) { |
| 259 | winner, bestSolve, bestTTCS = labels[i], solve, ttcs |
| 260 | } |
| 261 | } |
| 262 | fmt.Fprintf(&b, " %s |\n", winner) |
| 263 | } |
| 264 | return b.String() + "\n" |
| 265 | } |
| 266 | |
| 267 | // accumulateSources folds one run's per-origin usage into the suite totals. |
| 268 | func accumulateSources(total map[string]sourceUsage, run map[string]sourceUsage) { |
| 269 | for source, usage := range run { |
| 270 | agg := total[source] |
| 271 | agg.Calls += usage.Calls |
| 272 | agg.PromptTokens += usage.PromptTokens |
| 273 | agg.CompletionTokens += usage.CompletionTokens |
| 274 | agg.Cost += usage.Cost |
| 275 | total[source] = agg |
| 276 | } |
| 277 | } |
| 278 | |
| 279 | // compareReports renders an A/B delta table from two -json report files — |
| 280 | // the readout for an ablation experiment (e.g. control vs -ablate planner). |
| 281 | func compareReports(pathA, pathB string) (string, error) { |
| 282 | arms := make([]armStats, 0, 2) |
| 283 | for _, path := range []string{pathA, pathB} { |
| 284 | s, err := loadArm(path) |
| 285 | if err != nil { |
| 286 | return "", err |
| 287 | } |
| 288 | arms = append(arms, s) |
| 289 | } |
| 290 | a, bStats := arms[0], arms[1] |
| 291 | var b strings.Builder |
| 292 | fmt.Fprintf(&b, "## e2ebench A/B: `%s` vs `%s`\n\n", pathA, pathB) |
| 293 | fmt.Fprintf(&b, "| Metric | A | B |\n|---|---:|---:|\n") |
| 294 | fmt.Fprintf(&b, "| Solved | %d/%d (%s) | %d/%d (%s) |\n", a.Solved, a.Ran, pct(a.Solved, a.Ran), bStats.Solved, bStats.Ran, pct(bStats.Solved, bStats.Ran)) |
| 295 | fmt.Fprintf(&b, "| Pass@1 | %s | %s |\n", pct(a.Pass1, a.Ran), pct(bStats.Pass1, bStats.Ran)) |
| 296 | fmt.Fprintf(&b, "| TTFT median | %s | %s |\n", durMs(median(a.TTFT)), durMs(median(bStats.TTFT))) |
| 297 | fmt.Fprintf(&b, "| TTCS median | %s | %s |\n", dur(median(a.TTCS)), dur(median(bStats.TTCS))) |
| 298 | fmt.Fprintf(&b, "| TTCS p90 | %s | %s |\n", dur(pctile(a.TTCS, 90)), dur(pctile(bStats.TTCS, 90))) |
| 299 | fmt.Fprintf(&b, "| Cache hit | %s | %s |\n", pct(a.Hit, a.Hit+a.Miss), pct(bStats.Hit, bStats.Hit+bStats.Miss)) |
| 300 | fmt.Fprintf(&b, "| First-request cache hit | %s | %s |\n", pct(int(a.FirstHit), int(a.FirstHit+a.FirstMiss)), pct(int(bStats.FirstHit), int(bStats.FirstHit+bStats.FirstMiss))) |
| 301 | fmt.Fprintf(&b, "| Overthinking damage | %s | %s |\n", pct(a.Damaged, a.WithCorrect), pct(bStats.Damaged, bStats.WithCorrect)) |
| 302 | fmt.Fprintf(&b, "| Model requests / solved | %s | %s |\n", perSolved(float64(a.Steps), a.AccountedSolved), perSolved(float64(bStats.Steps), bStats.AccountedSolved)) |
| 303 | fmt.Fprintf(&b, "| Planner requests / solved | %s | %s |\n", perSolved(float64(a.PlannerCalls), a.AccountedSolved), perSolved(float64(bStats.PlannerCalls), bStats.AccountedSolved)) |
| 304 | fmt.Fprintf(&b, "| Model rounds / solved | %s | %s |\n", perSolved(float64(a.Rounds), a.AccountedSolved), perSolved(float64(bStats.Rounds), bStats.AccountedSolved)) |
| 305 | fmt.Fprintf(&b, "| Tool calls / solved | %s | %s |\n", perSolved(float64(a.Tools), a.AccountedSolved), perSolved(float64(bStats.Tools), bStats.AccountedSolved)) |
| 306 | fmt.Fprintf(&b, "| Tokens / solved | %s | %s |\n", perSolved(float64(a.Tokens), a.AccountedSolved), perSolved(float64(bStats.Tokens), bStats.AccountedSolved)) |
| 307 | fmt.Fprintf(&b, "| Wall seconds / solved | %s | %s |\n", perSolved(float64(a.WallMs)/1000, a.AccountedSolved), perSolved(float64(bStats.WallMs)/1000, bStats.AccountedSolved)) |
| 308 | fmt.Fprintf(&b, "| Cost / solved | %s | %s |\n", perSolved(a.Cost, a.AccountedSolved), perSolved(bStats.Cost, bStats.AccountedSolved)) |
| 309 | b.WriteString(marginalUtilitySection(a, bStats)) |
| 310 | b.WriteString(memoryUtilitySection(pathA, pathB)) |
| 311 | b.WriteString("\n" + paretoSection([]paretoPoint{newParetoPoint(pathA, a), newParetoPoint(pathB, bStats)})) |
| 312 | b.WriteString("<sub>Per-solved figures divide each arm's accounted totals (failures included) by its accounted solves.</sub>\n") |
| 313 | return b.String(), nil |
| 314 | } |
| 315 | |
| 316 | func solveRate(solved, ran int) float64 { |
| 317 | if ran == 0 { |
| 318 | return 0 |
| 319 | } |
| 320 | return float64(solved) * 100 / float64(ran) |
| 321 | } |
| 322 | |
| 323 | func wallPerTask(wallMs int64, ran int) float64 { |
| 324 | if ran == 0 { |
| 325 | return 0 |
| 326 | } |
| 327 | return float64(wallMs) / 1000 / float64(ran) |
| 328 | } |
| 329 | |
| 330 | // marginalUtilitySection is the decision readout: not "does A help" but what |
| 331 | // each accuracy point costs in latency, overall and per task class, so a |
| 332 | // subsystem can be routed per class instead of globally defaulted. |
| 333 | func marginalUtilitySection(a, b armStats) string { |
| 334 | var out strings.Builder |
| 335 | fmt.Fprintf(&out, "\n**Marginal utility (A − B):** accuracy %+.1fpp · wall/task %+.1fs\n\n", |
| 336 | solveRate(a.Solved, a.Ran)-solveRate(b.Solved, b.Ran), |
| 337 | wallPerTask(a.WallMs, a.Ran)-wallPerTask(b.WallMs, b.Ran)) |
| 338 | classes := make([]string, 0, len(a.ByClass)+len(b.ByClass)) |
| 339 | seen := map[string]bool{} |
| 340 | for _, m := range []map[string]classStats{a.ByClass, b.ByClass} { |
| 341 | for class := range m { |
| 342 | if !seen[class] { |
| 343 | seen[class] = true |
| 344 | classes = append(classes, class) |
| 345 | } |
| 346 | } |
| 347 | } |
| 348 | if len(classes) == 0 || (len(classes) == 1 && classes[0] == "unclassified") { |
| 349 | return out.String() |
| 350 | } |
| 351 | sort.Strings(classes) |
| 352 | out.WriteString("| Class | A solved | B solved | Δ accuracy | A wall/task | B wall/task | Δ wall |\n|---|---:|---:|---:|---:|---:|---:|\n") |
| 353 | for _, class := range classes { |
| 354 | ca, cb := a.ByClass[class], b.ByClass[class] |
| 355 | fmt.Fprintf(&out, "| %s | %d/%d | %d/%d | %+.1fpp | %.1fs | %.1fs | %+.1fs |\n", |
| 356 | class, ca.Solved, ca.Ran, cb.Solved, cb.Ran, |
| 357 | solveRate(ca.Solved, ca.Ran)-solveRate(cb.Solved, cb.Ran), |
| 358 | wallPerTask(ca.WallMs, ca.Ran), wallPerTask(cb.WallMs, cb.Ran), |
| 359 | wallPerTask(ca.WallMs, ca.Ran)-wallPerTask(cb.WallMs, cb.Ran)) |
| 360 | } |
| 361 | out.WriteString("\n") |
| 362 | return out.String() |
| 363 | } |
| 364 |