| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "os" |
| 5 | "strings" |
| 6 | "testing" |
| 7 | ) |
| 8 | |
| 9 | func TestRenderMechanismLedgerSplitsFiredVsQuiet(t *testing.T) { |
| 10 | fired := result{task: task{ID: "a"}} |
| 11 | fired.Trajectory = &trajectorySummary{ |
| 12 | HandoffNudges: 2, |
| 13 | RoundOutcomeMs: map[string]int64{"handoff_retry": 7_000}, |
| 14 | StreamRetries: 1, |
| 15 | RecoveryGapMsByKind: map[string]int64{ |
| 16 | "stream_retry": 9_000, |
| 17 | }, |
| 18 | } |
| 19 | quiet := result{task: task{ID: "b"}, Passed: true} |
| 20 | quiet.Trajectory = &trajectorySummary{} |
| 21 | |
| 22 | got := renderMechanismLedger([]result{fired, quiet}) |
| 23 | for _, want := range []string{ |
| 24 | "**Mechanism ledger**", |
| 25 | "causal rescue rates need an `-ablate` A/B", |
| 26 | "| handoff_nudge | 2 | 1/2 | 7.0s | 0% | 100% |", |
| 27 | "| stream_retry | 1 | 1/2 | 9.0s | 0% | 100% |", |
| 28 | } { |
| 29 | if !strings.Contains(got, want) { |
| 30 | t.Fatalf("ledger missing %q:\n%s", want, got) |
| 31 | } |
| 32 | } |
| 33 | if strings.Contains(got, "| planner |") { |
| 34 | t.Fatalf("mechanisms that never fired must not render rows:\n%s", got) |
| 35 | } |
| 36 | |
| 37 | allQuiet := renderMechanismLedger([]result{quiet}) |
| 38 | if !strings.Contains(allQuiet, "all quiet — no extra-round machinery fired across 1 recorded runs") { |
| 39 | t.Fatalf("all-quiet suite must state the absence:\n%s", allQuiet) |
| 40 | } |
| 41 | } |
| 42 | |
| 43 | func TestSummarizeTrajectoryCollectsToolSurface(t *testing.T) { |
| 44 | path := t.TempDir() + "/surface.trajectory.jsonl" |
| 45 | lines := []string{ |
| 46 | `{"seq":1,"ts":1000,"event":{"kind":"turn_started"}}`, |
| 47 | `{"seq":2,"ts":1500,"event":{"kind":"usage","usage":{"source":"executor","promptTokens":14000,"cacheDiagnostics":{"toolSchemaTokens":2000,"prefixChanged":false}}}}`, |
| 48 | `{"seq":3,"ts":2000,"event":{"kind":"tool_dispatch","tool":{"id":"a","name":"connect_tool_source","args":"{\"source\":\"web\"}"}}}`, |
| 49 | `{"seq":4,"ts":2100,"event":{"kind":"tool_result","tool":{"id":"a","name":"connect_tool_source","durationMs":100,"startedAt":2000,"endedAt":2100}}}`, |
| 50 | `{"seq":5,"ts":3000,"event":{"kind":"usage","usage":{"source":"executor","promptTokens":16000,"cacheDiagnostics":{"toolSchemaTokens":5000,"prefixChanged":true}}}}`, |
| 51 | `{"seq":6,"ts":3500,"event":{"kind":"usage","usage":{"source":"subagent","promptTokens":9000,"cacheDiagnostics":{"toolSchemaTokens":9999,"prefixChanged":true}}}}`, |
| 52 | `{"seq":7,"ts":4000,"event":{"kind":"turn_done"}}`, |
| 53 | } |
| 54 | if err := writeLines(path, lines); err != nil { |
| 55 | t.Fatalf("write fixture: %v", err) |
| 56 | } |
| 57 | s, err := summarizeTrajectory(path) |
| 58 | if err != nil { |
| 59 | t.Fatalf("summarizeTrajectory: %v", err) |
| 60 | } |
| 61 | if s.SchemaTokensMax != 5000 || s.SchemaTokensTotal != 7000 { |
| 62 | t.Errorf("schema max=%d total=%d, want 5000/7000 (subagent usage excluded)", s.SchemaTokensMax, s.SchemaTokensTotal) |
| 63 | } |
| 64 | if s.PromptTokensSeen != 30000 { |
| 65 | t.Errorf("prompt tokens = %d, want 30000", s.PromptTokensSeen) |
| 66 | } |
| 67 | if s.PrefixResets != 1 { |
| 68 | t.Errorf("prefix resets = %d, want 1 (subagent reset must not count)", s.PrefixResets) |
| 69 | } |
| 70 | if s.ConnectCalls != 1 { |
| 71 | t.Errorf("connect calls = %d, want 1", s.ConnectCalls) |
| 72 | } |
| 73 | } |
| 74 | |
| 75 | func TestKPILineIncludesTTFTAndFirstRequestCacheHit(t *testing.T) { |
| 76 | cold := result{task: task{ID: "a"}, Passed: true, WallMs: 10_000, Attempt: 1, TTCSMs: 10_000} |
| 77 | cold.Trajectory = &trajectorySummary{ |
| 78 | TTFTMs: 2800, |
| 79 | FirstReqCacheHitTokens: 400, |
| 80 | FirstReqCacheMissTokens: 13_600, |
| 81 | } |
| 82 | got := renderBody([]result{cold}) |
| 83 | for _, want := range []string{ |
| 84 | "**TTFT median** 2.8s", |
| 85 | "**first-request cache hit** 3%", |
| 86 | } { |
| 87 | if !strings.Contains(got, want) { |
| 88 | t.Fatalf("KPI line missing %q:\n%s", want, got) |
| 89 | } |
| 90 | } |
| 91 | } |
| 92 | |
| 93 | func TestRenderToolSurfaceLine(t *testing.T) { |
| 94 | r := result{task: task{ID: "a"}, Passed: true} |
| 95 | r.Trajectory = &trajectorySummary{ |
| 96 | SchemaTokensMax: 12784, SchemaTokensTotal: 89488, |
| 97 | PromptTokensSeen: 140000, PrefixResets: 2, ConnectCalls: 1, |
| 98 | } |
| 99 | got := renderToolSurface([]result{r}) |
| 100 | for _, want := range []string{ |
| 101 | "**schema footprint** 12,784 tok/request", |
| 102 | "**Σ schema tax** 89,488 tok (64% of prompt)", |
| 103 | "**connect_tool_source** ×1", |
| 104 | "**prefix resets** 2", |
| 105 | } { |
| 106 | if !strings.Contains(got, want) { |
| 107 | t.Fatalf("tool surface line missing %q:\n%s", want, got) |
| 108 | } |
| 109 | } |
| 110 | if renderToolSurface([]result{{task: task{ID: "b"}}}) != "" { |
| 111 | t.Fatal("runs without schema data must not render the line") |
| 112 | } |
| 113 | } |
| 114 | |
| 115 | func TestSummarizeTrajectorySplitsRecoveryGapByKind(t *testing.T) { |
| 116 | path := t.TempDir() + "/kinds.trajectory.jsonl" |
| 117 | lines := []string{ |
| 118 | `{"seq":1,"ts":1000,"event":{"kind":"turn_started"}}`, |
| 119 | `{"seq":2,"ts":2000,"event":{"kind":"retrying","retryScope":"stream"}}`, |
| 120 | `{"seq":3,"ts":6000,"event":{"kind":"tool_dispatch","tool":{"id":"a","name":"bash","args":"{}"}}}`, |
| 121 | `{"seq":4,"ts":6100,"event":{"kind":"tool_result","tool":{"id":"a","name":"bash","durationMs":100,"startedAt":6000,"endedAt":6100}}}`, |
| 122 | `{"seq":5,"ts":7000,"protocol_recovery":"missing_reasoning_retry_attempted"}`, |
| 123 | `{"seq":6,"ts":9000,"event":{"kind":"tool_dispatch","tool":{"id":"b","name":"bash","args":"{\"x\":1}"}}}`, |
| 124 | `{"seq":7,"ts":9100,"event":{"kind":"tool_result","tool":{"id":"b","name":"bash","durationMs":100,"startedAt":9000,"endedAt":9100}}}`, |
| 125 | `{"seq":8,"ts":10000,"event":{"kind":"turn_done"}}`, |
| 126 | } |
| 127 | if err := writeLines(path, lines); err != nil { |
| 128 | t.Fatalf("write fixture: %v", err) |
| 129 | } |
| 130 | s, err := summarizeTrajectory(path) |
| 131 | if err != nil { |
| 132 | t.Fatalf("summarizeTrajectory: %v", err) |
| 133 | } |
| 134 | if s.RecoveryGapMsByKind["stream_retry"] != 5000 { |
| 135 | t.Errorf("stream_retry gap = %d, want 5000", s.RecoveryGapMsByKind["stream_retry"]) |
| 136 | } |
| 137 | if s.RecoveryGapMsByKind["reasoning_replay"] != 2900 { |
| 138 | t.Errorf("reasoning_replay gap = %d, want 2900", s.RecoveryGapMsByKind["reasoning_replay"]) |
| 139 | } |
| 140 | if s.RecoveryRounds != 2 { |
| 141 | t.Errorf("recovery rounds = %d, want 2", s.RecoveryRounds) |
| 142 | } |
| 143 | } |
| 144 | |
| 145 | func writeLines(path string, lines []string) error { |
| 146 | return os.WriteFile(path, []byte(strings.Join(lines, "\n")), 0o644) |
| 147 | } |
| 148 | |
| 149 | func TestRenderContractShadowAgreement(t *testing.T) { |
| 150 | agree := result{task: task{ID: "a"}, Passed: true} |
| 151 | agree.Trajectory = &trajectorySummary{ShadowVerdict: "complete", ShadowComplete: true, ShadowIntent: "mutation"} |
| 152 | miss := result{task: task{ID: "b"}, Passed: true} |
| 153 | miss.Trajectory = &trajectorySummary{ShadowVerdict: "continue", ShadowComplete: false} |
| 154 | got := renderContractShadow([]result{agree, miss}) |
| 155 | for _, want := range []string{ |
| 156 | "verdicts complete ×1 · continue ×1", |
| 157 | "**agreement with grader** 50% (1/2)", |
| 158 | } { |
| 159 | if !strings.Contains(got, want) { |
| 160 | t.Fatalf("shadow line missing %q:\n%s", want, got) |
| 161 | } |
| 162 | } |
| 163 | if renderContractShadow([]result{{task: task{ID: "c"}}}) != "" { |
| 164 | t.Fatal("runs without shadow audits must render nothing") |
| 165 | } |
| 166 | } |
| 167 | |
| 168 | func TestRenderCompletionReportPricesOverclaim(t *testing.T) { |
| 169 | honest := result{task: task{ID: "a"}, Passed: true} |
| 170 | honest.Trajectory = &trajectorySummary{CompletionVerdict: "done"} |
| 171 | overclaimed := result{task: task{ID: "b"}, Passed: false} |
| 172 | overclaimed.Trajectory = &trajectorySummary{CompletionVerdict: "done"} |
| 173 | warned := result{task: task{ID: "c"}, Passed: false} |
| 174 | warned.Trajectory = &trajectorySummary{ |
| 175 | CompletionVerdict: "partial", CompletionGaps: 2, |
| 176 | CompletionGapKinds: []string{"unreviewed_change", "stale_verification"}, |
| 177 | ClaimsVerified: 4, ClaimsUnbacked: 1, |
| 178 | } |
| 179 | got := renderCompletionReport([]result{honest, overclaimed, warned}) |
| 180 | for _, want := range []string{ |
| 181 | "verdicts done ×2 · partial ×1", |
| 182 | "**overclaim** 50% (1/2 done runs the grader failed)", |
| 183 | "**caught** 50% (1/2 failed runs declared a gap)", |
| 184 | "gaps stale_verification ×1 · unreviewed_change ×1", |
| 185 | "**unbacked claims** 25% (1/4 asserted verifications the ledger denied)", |
| 186 | } { |
| 187 | if !strings.Contains(got, want) { |
| 188 | t.Fatalf("completion line missing %q:\n%s", want, got) |
| 189 | } |
| 190 | } |
| 191 | if renderCompletionReport([]result{{task: task{ID: "d"}}}) != "" { |
| 192 | t.Fatal("runs without completion audits must render nothing") |
| 193 | } |
| 194 | } |
| 195 | |
| 196 | func TestSummarizeTrajectoryReadsCompletionReport(t *testing.T) { |
| 197 | path := t.TempDir() + "/completion.trajectory.jsonl" |
| 198 | lines := []string{ |
| 199 | `{"seq":1,"ts":1000,"event":{"kind":"turn_started"}}`, |
| 200 | `{"seq":2,"ts":2000,"completion_report":{"verdict":"partial","gaps":1,"gap_kinds":["unreviewed_change"]}}`, |
| 201 | `{"seq":3,"ts":3000,"event":{"kind":"turn_done"}}`, |
| 202 | } |
| 203 | if err := writeLines(path, lines); err != nil { |
| 204 | t.Fatal(err) |
| 205 | } |
| 206 | s, err := summarizeTrajectory(path) |
| 207 | if err != nil { |
| 208 | t.Fatal(err) |
| 209 | } |
| 210 | if s.CompletionVerdict != "partial" || s.CompletionGaps != 1 || len(s.CompletionGapKinds) != 1 { |
| 211 | t.Fatalf("completion = %q/%d/%v", s.CompletionVerdict, s.CompletionGaps, s.CompletionGapKinds) |
| 212 | } |
| 213 | } |
| 214 | |
| 215 | func TestSummarizeTrajectoryReadsContractShadow(t *testing.T) { |
| 216 | path := t.TempDir() + "/shadow.trajectory.jsonl" |
| 217 | lines := []string{ |
| 218 | `{"seq":1,"ts":1000,"event":{"kind":"turn_started"}}`, |
| 219 | `{"seq":2,"ts":2000,"contract_shadow":{"intent":"mutation","verdict":"complete","complete":true}}`, |
| 220 | `{"seq":3,"ts":3000,"event":{"kind":"turn_done"}}`, |
| 221 | } |
| 222 | if err := writeLines(path, lines); err != nil { |
| 223 | t.Fatal(err) |
| 224 | } |
| 225 | s, err := summarizeTrajectory(path) |
| 226 | if err != nil { |
| 227 | t.Fatal(err) |
| 228 | } |
| 229 | if s.ShadowVerdict != "complete" || !s.ShadowComplete || s.ShadowIntent != "mutation" { |
| 230 | t.Fatalf("shadow = %q/%v/%q", s.ShadowVerdict, s.ShadowComplete, s.ShadowIntent) |
| 231 | } |
| 232 | } |
| 233 |