| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "encoding/json" |
| 5 | "os" |
| 6 | "path/filepath" |
| 7 | "strings" |
| 8 | "testing" |
| 9 | ) |
| 10 | |
| 11 | func writeArm(t *testing.T, dir, name string, results []result) string { |
| 12 | t.Helper() |
| 13 | data, err := json.Marshal(results) |
| 14 | if err != nil { |
| 15 | t.Fatalf("marshal: %v", err) |
| 16 | } |
| 17 | path := filepath.Join(dir, name) |
| 18 | if err := os.WriteFile(path, data, 0o644); err != nil { |
| 19 | t.Fatalf("write: %v", err) |
| 20 | } |
| 21 | return path |
| 22 | } |
| 23 | |
| 24 | func TestCompareReportsRendersPerSolvedDeltas(t *testing.T) { |
| 25 | dir := t.TempDir() |
| 26 | control := result{task: task{ID: "a"}, Passed: true, WallMs: 60_000} |
| 27 | control.Steps = 7 |
| 28 | control.ToolCalls = 10 |
| 29 | control.PromptTokens = 900 |
| 30 | control.CompletionTokens = 100 |
| 31 | control.UsageBySource = map[string]sourceUsage{"executor": {Calls: 6}, "planner": {Calls: 1}} |
| 32 | control.Trajectory = &trajectorySummary{ModelRounds: 5} |
| 33 | |
| 34 | ablated := result{task: task{ID: "a"}, Passed: true, WallMs: 45_000} |
| 35 | ablated.Steps = 5 |
| 36 | ablated.ToolCalls = 10 |
| 37 | ablated.PromptTokens = 700 |
| 38 | ablated.CompletionTokens = 100 |
| 39 | ablated.UsageBySource = map[string]sourceUsage{"executor": {Calls: 5}} |
| 40 | ablated.Trajectory = &trajectorySummary{ModelRounds: 5} |
| 41 | |
| 42 | pathA := writeArm(t, dir, "control.json", []result{control}) |
| 43 | pathB := writeArm(t, dir, "ablated.json", []result{ablated}) |
| 44 | |
| 45 | got, err := compareReports(pathA, pathB) |
| 46 | if err != nil { |
| 47 | t.Fatalf("compareReports: %v", err) |
| 48 | } |
| 49 | for _, want := range []string{ |
| 50 | "| Solved | 1/1 (100%) | 1/1 (100%) |", |
| 51 | "| Model requests / solved | 7.0 | 5.0 |", |
| 52 | "| Planner requests / solved | 1.0 | 0.0 |", |
| 53 | "| Wall seconds / solved | 60.0 | 45.0 |", |
| 54 | } { |
| 55 | if !strings.Contains(got, want) { |
| 56 | t.Fatalf("compare table missing %q:\n%s", want, got) |
| 57 | } |
| 58 | } |
| 59 | } |
| 60 | |
| 61 | func TestCompareReportsMarginalUtilityByClass(t *testing.T) { |
| 62 | dir := t.TempDir() |
| 63 | mk := func(id, class string, passed bool, wallMs int64) result { |
| 64 | r := result{task: task{ID: id, Class: class}, Passed: passed, WallMs: wallMs} |
| 65 | r.Steps = 1 |
| 66 | return r |
| 67 | } |
| 68 | withPlanner := []result{ |
| 69 | mk("bug1", "bugfix", true, 30_000), mk("bug2", "bugfix", true, 30_000), |
| 70 | mk("ref1", "refactor", true, 112_000), |
| 71 | } |
| 72 | without := []result{ |
| 73 | mk("bug1", "bugfix", true, 18_000), mk("bug2", "bugfix", true, 18_000), |
| 74 | mk("ref1", "refactor", false, 74_000), |
| 75 | } |
| 76 | pathA := writeArm(t, dir, "with.json", withPlanner) |
| 77 | pathB := writeArm(t, dir, "without.json", without) |
| 78 | |
| 79 | got, err := compareReports(pathA, pathB) |
| 80 | if err != nil { |
| 81 | t.Fatalf("compareReports: %v", err) |
| 82 | } |
| 83 | for _, want := range []string{ |
| 84 | "**Marginal utility (A − B):** accuracy +33.3pp · wall/task +20.7s", |
| 85 | "| bugfix | 2/2 | 2/2 | +0.0pp | 30.0s | 18.0s | +12.0s |", |
| 86 | "| refactor | 1/1 | 0/1 | +100.0pp | 112.0s | 74.0s | +38.0s |", |
| 87 | } { |
| 88 | if !strings.Contains(got, want) { |
| 89 | t.Fatalf("marginal utility missing %q:\n%s", want, got) |
| 90 | } |
| 91 | } |
| 92 | } |
| 93 | |
| 94 | func TestRequestsBySourceLineOrdersByCalls(t *testing.T) { |
| 95 | line := requestsBySourceLine(map[string]sourceUsage{ |
| 96 | "planner": {Calls: 3, PromptTokens: 300}, |
| 97 | "executor": {Calls: 9, PromptTokens: 900, CompletionTokens: 100}, |
| 98 | "title": {Calls: 0}, |
| 99 | }) |
| 100 | if !strings.Contains(line, "executor 9 (1,000 tok) · planner 3 (300 tok)") { |
| 101 | t.Fatalf("source line = %q", line) |
| 102 | } |
| 103 | if strings.Contains(line, "title") { |
| 104 | t.Fatalf("zero-call sources must be omitted: %q", line) |
| 105 | } |
| 106 | if requestsBySourceLine(nil) != "" { |
| 107 | t.Fatal("no sources must render nothing") |
| 108 | } |
| 109 | } |
| 110 |