| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "os" |
| 5 | "path/filepath" |
| 6 | "strings" |
| 7 | "testing" |
| 8 | ) |
| 9 | |
| 10 | func writeTrajectory(t *testing.T, name string, lines []string) string { |
| 11 | t.Helper() |
| 12 | path := filepath.Join(t.TempDir(), name) |
| 13 | if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")), 0o644); err != nil { |
| 14 | t.Fatalf("write fixture: %v", err) |
| 15 | } |
| 16 | return path |
| 17 | } |
| 18 | |
| 19 | func TestSummarizeOutcomeFromRecordedShadowSamples(t *testing.T) { |
| 20 | path := writeTrajectory(t, "shadow.trajectory.jsonl", []string{ |
| 21 | `{"seq":1,"ts":500,"event":{"kind":"turn_started"}}`, |
| 22 | `{"seq":2,"ts":1000,"outcome_progress":{"round":1,"exploration":1,"legacy_gain":1}}`, |
| 23 | `{"seq":3,"ts":2000,"outcome_progress":{"round":2,"verification":1,"legacy_gain":2}}`, |
| 24 | `{"seq":4,"ts":3000,"outcome_progress":{"round":3,"churn":1,"legacy_gain":3}}`, |
| 25 | `{"seq":5,"ts":4000,"outcome_progress":{"round":4,"verification":1,"objective":1,"legacy_gain":2}}`, |
| 26 | `{"seq":6,"ts":5000,"outcome_progress":{"round":5,"churn":1,"legacy_gain":3}}`, |
| 27 | `{"seq":7,"ts":6000,"outcome_progress":{"round":6,"verification":1,"regression":1}}`, |
| 28 | `{"seq":8,"ts":7000,"event":{"kind":"turn_done"}}`, |
| 29 | }) |
| 30 | s, err := summarizeTrajectory(path) |
| 31 | if err != nil { |
| 32 | t.Fatalf("summarizeTrajectory: %v", err) |
| 33 | } |
| 34 | o := s.Outcome |
| 35 | if o == nil || o.Backfilled { |
| 36 | t.Fatalf("outcome = %+v, want recorded (not backfilled)", o) |
| 37 | } |
| 38 | if o.Rounds != 6 || o.ProgressRounds != 5 { |
| 39 | t.Errorf("rounds=%d progress=%d, want 6/5", o.Rounds, o.ProgressRounds) |
| 40 | } |
| 41 | // Round 5 claimed legacy progress (a mutation) with no objective transition |
| 42 | // inside the redemption window — the false-progress case. |
| 43 | if o.FalseProgressRounds != 1 { |
| 44 | t.Errorf("false progress = %d, want 1", o.FalseProgressRounds) |
| 45 | } |
| 46 | if o.SolutionStallMax != 2 { |
| 47 | t.Errorf("solution stall max = %d, want 2", o.SolutionStallMax) |
| 48 | } |
| 49 | if o.Objective != 1 || o.Regression != 1 || o.BestScore != 1 || o.FinalScore != 0 { |
| 50 | t.Errorf("objective=%d regression=%d best=%d final=%d, want 1/1/1/0", |
| 51 | o.Objective, o.Regression, o.BestScore, o.FinalScore) |
| 52 | } |
| 53 | if !o.RegressedFromBest || o.SearchRegretMs != 3000 { |
| 54 | t.Errorf("regressed=%v regret=%d, want true/3000 (best at ts 4000, end at 7000)", |
| 55 | o.RegressedFromBest, o.SearchRegretMs) |
| 56 | } |
| 57 | } |
| 58 | |
| 59 | func TestSummarizeOutcomeBackfillsFromVerificationReceipts(t *testing.T) { |
| 60 | args := `{\"command\":\"go test ./x\"}` |
| 61 | path := writeTrajectory(t, "old.trajectory.jsonl", []string{ |
| 62 | `{"seq":1,"ts":1000,"event":{"kind":"turn_started"}}`, |
| 63 | `{"seq":2,"ts":2000,"event":{"kind":"tool_result","tool":{"name":"bash","args":"` + args + `","err":"exit 1","execution":{"verification":"failed"}}}}`, |
| 64 | `{"seq":3,"ts":3000,"event":{"kind":"tool_result","tool":{"name":"bash","args":"` + args + `","execution":{"verification":"passed"}}}}`, |
| 65 | `{"seq":4,"ts":4000,"event":{"kind":"tool_result","tool":{"name":"bash","args":"` + args + `","err":"exit 1","execution":{"verification":"failed"}}}}`, |
| 66 | `{"seq":5,"ts":5000,"event":{"kind":"turn_done"}}`, |
| 67 | }) |
| 68 | s, err := summarizeTrajectory(path) |
| 69 | if err != nil { |
| 70 | t.Fatalf("summarizeTrajectory: %v", err) |
| 71 | } |
| 72 | o := s.Outcome |
| 73 | if o == nil || !o.Backfilled { |
| 74 | t.Fatalf("outcome = %+v, want a backfilled summary", o) |
| 75 | } |
| 76 | if o.Objective != 1 || o.Regression != 1 || o.BestScore != 1 || o.FinalScore != 0 { |
| 77 | t.Errorf("objective=%d regression=%d best=%d final=%d, want 1/1/1/0", |
| 78 | o.Objective, o.Regression, o.BestScore, o.FinalScore) |
| 79 | } |
| 80 | if !o.RegressedFromBest || o.SearchRegretMs != 2000 { |
| 81 | t.Errorf("regressed=%v regret=%d, want true/2000", o.RegressedFromBest, o.SearchRegretMs) |
| 82 | } |
| 83 | if o.FalseProgressRounds != 0 || o.ProgressRounds != 0 { |
| 84 | t.Errorf("backfill cannot price legacy claims, got progress=%d false=%d", |
| 85 | o.ProgressRounds, o.FalseProgressRounds) |
| 86 | } |
| 87 | // A subagent's verification must not pollute the parent series. |
| 88 | sub := writeTrajectory(t, "sub.trajectory.jsonl", []string{ |
| 89 | `{"seq":1,"ts":1000,"event":{"kind":"turn_started"}}`, |
| 90 | `{"seq":2,"ts":2000,"event":{"kind":"tool_result","tool":{"name":"bash","args":"` + args + `","parentId":"task-1","execution":{"verification":"failed"}}}}`, |
| 91 | `{"seq":3,"ts":3000,"event":{"kind":"turn_done"}}`, |
| 92 | }) |
| 93 | if s, err = summarizeTrajectory(sub); err != nil { |
| 94 | t.Fatalf("summarizeTrajectory: %v", err) |
| 95 | } |
| 96 | if s.Outcome != nil { |
| 97 | t.Errorf("subagent-only verification must yield no outcome summary, got %+v", s.Outcome) |
| 98 | } |
| 99 | } |
| 100 | |
| 101 | func TestSummarizeOutcomeTracksDebtAndTTFDC(t *testing.T) { |
| 102 | path := writeTrajectory(t, "debt.trajectory.jsonl", []string{ |
| 103 | `{"seq":1,"ts":1000,"event":{"kind":"turn_started"}}`, |
| 104 | `{"seq":2,"ts":2000,"outcome_progress":{"round":1,"churn":1,"legacy_gain":3,"debt_age":1}}`, |
| 105 | `{"seq":3,"ts":3000,"outcome_progress":{"round":2,"exploration":1,"legacy_gain":1,"debt_age":2}}`, |
| 106 | `{"seq":4,"ts":4000,"outcome_progress":{"round":3,"churn":1,"legacy_gain":3,"debt_age":3}}`, |
| 107 | `{"seq":5,"ts":9000,"outcome_progress":{"round":4,"discriminating":1,"verification":1}}`, |
| 108 | `{"seq":6,"ts":10000,"event":{"kind":"turn_done"}}`, |
| 109 | }) |
| 110 | s, err := summarizeTrajectory(path) |
| 111 | if err != nil { |
| 112 | t.Fatalf("summarizeTrajectory: %v", err) |
| 113 | } |
| 114 | o := s.Outcome |
| 115 | if o == nil || o.DebtAgeMax != 3 { |
| 116 | t.Fatalf("outcome = %+v, want debt age max 3", o) |
| 117 | } |
| 118 | if o.TTFDCMs != 8000 { |
| 119 | t.Fatalf("TTFDC = %d, want 8000 (run start 1000 → first discriminating 9000)", o.TTFDCMs) |
| 120 | } |
| 121 | got := renderOutcomeProgress([]result{{Trajectory: s}}) |
| 122 | for _, want := range []string{"**discriminating checks** in 1/1 runs (TTFDC p50 8.0s)", "**verification debt max** 3 rounds"} { |
| 123 | if !strings.Contains(got, want) { |
| 124 | t.Errorf("render missing %q in:\n%s", want, got) |
| 125 | } |
| 126 | } |
| 127 | } |
| 128 | |
| 129 | func TestSummarizeRunwayShadowDistinguishesOldDataFromZeroBalance(t *testing.T) { |
| 130 | path := writeTrajectory(t, "runway.trajectory.jsonl", []string{ |
| 131 | `{"seq":1,"ts":1000,"event":{"kind":"turn_started"}}`, |
| 132 | `{"seq":2,"ts":2000,"outcome_progress":{"round":1,"runway":8,"runway_dry":4,"runway_idle":4}}`, |
| 133 | `{"seq":3,"ts":3000,"outcome_progress":{"round":2,"runway":4,"runway_dry":5,"runway_idle":5}}`, |
| 134 | `{"seq":4,"ts":4000,"outcome_progress":{"round":3,"runway":0,"runway_dry":6,"runway_idle":6,"runway_spent":true}}`, |
| 135 | }) |
| 136 | s, err := summarizeTrajectory(path) |
| 137 | if err != nil { |
| 138 | t.Fatalf("summarizeTrajectory: %v", err) |
| 139 | } |
| 140 | o := s.Outcome |
| 141 | if o == nil || o.RunwaySamples != 3 || o.RunwayMin != 0 || o.RunwayFinal != 0 || o.RunwayFirstSpentRound != 3 { |
| 142 | t.Fatalf("runway summary = %+v, want 3 samples and spent at round 3", o) |
| 143 | } |
| 144 | if o.RunwayDryMax != 6 || o.RunwayIdleMax != 6 { |
| 145 | t.Fatalf("runway maxima = dry %d idle %d, want 6/6", o.RunwayDryMax, o.RunwayIdleMax) |
| 146 | } |
| 147 | got := renderOutcomeProgress([]result{{Trajectory: s}}) |
| 148 | for _, want := range []string{"**runway shadow** would spend in 1/1 runs (100%)", "median round 3", "max dry/idle 6/6"} { |
| 149 | if !strings.Contains(got, want) { |
| 150 | t.Errorf("render missing %q in:\n%s", want, got) |
| 151 | } |
| 152 | } |
| 153 | |
| 154 | old := writeTrajectory(t, "pre-runway.trajectory.jsonl", []string{ |
| 155 | `{"seq":1,"ts":1000,"outcome_progress":{"round":1,"exploration":1}}`, |
| 156 | }) |
| 157 | legacy, err := summarizeTrajectory(old) |
| 158 | if err != nil { |
| 159 | t.Fatalf("summarize old trajectory: %v", err) |
| 160 | } |
| 161 | if legacy.Outcome == nil || legacy.Outcome.RunwaySamples != 0 { |
| 162 | t.Fatalf("old outcome = %+v, want no runway observations", legacy.Outcome) |
| 163 | } |
| 164 | if strings.Contains(renderOutcomeProgress([]result{{Trajectory: legacy}}), "runway shadow") { |
| 165 | t.Fatal("old trajectory was misclassified as a spent runway") |
| 166 | } |
| 167 | } |
| 168 | |
| 169 | func TestSummarizeOutcomeDerivesEBMChain(t *testing.T) { |
| 170 | path := writeTrajectory(t, "ebm.trajectory.jsonl", []string{ |
| 171 | `{"seq":1,"ts":1000,"event":{"kind":"turn_started"}}`, |
| 172 | `{"seq":2,"ts":2000,"outcome_progress":{"round":1,"churn":2,"debt_age":1,"blind_mutations":2}}`, |
| 173 | `{"seq":3,"ts":3000,"outcome_progress":{"round":2,"churn":1,"debt_age":2,"blind_mutations":3,"ebm_eligible":true,"ebm_fired":true}}`, |
| 174 | `{"seq":4,"ts":4000,"outcome_progress":{"round":3,"exploration":1,"debt_age":3,"blind_mutations":3}}`, |
| 175 | `{"seq":5,"ts":9000,"outcome_progress":{"round":4,"discriminating":1,"verification":1}}`, |
| 176 | `{"seq":6,"ts":10000,"event":{"kind":"turn_done"}}`, |
| 177 | }) |
| 178 | s, err := summarizeTrajectory(path) |
| 179 | if err != nil { |
| 180 | t.Fatalf("summarizeTrajectory: %v", err) |
| 181 | } |
| 182 | o := s.Outcome |
| 183 | if o == nil || o.EBMFiredRound != 2 || o.EBMEligibleRound != 2 { |
| 184 | t.Fatalf("outcome = %+v, want EBM fired/eligible at round 2", o) |
| 185 | } |
| 186 | if o.EBMBlindAtFire != 3 || o.EBMDebtAgeAtFire != 2 { |
| 187 | t.Errorf("at-fire = blind %d debt %d, want 3/2", o.EBMBlindAtFire, o.EBMDebtAgeAtFire) |
| 188 | } |
| 189 | if o.EBMRoundsToCheck != 2 || o.EBMMsToCheck != 6000 || o.EBMCheckWithin1 || !o.EBMCheckWithin2 { |
| 190 | t.Errorf("chain = rounds %d ms %d w1 %v w2 %v, want 2/6000/false/true", |
| 191 | o.EBMRoundsToCheck, o.EBMMsToCheck, o.EBMCheckWithin1, o.EBMCheckWithin2) |
| 192 | } |
| 193 | if o.DebtArea != 6 || o.BlindPeak != 3 { |
| 194 | t.Errorf("debt area %d blind peak %d, want 6/3", o.DebtArea, o.BlindPeak) |
| 195 | } |
| 196 | if o.EBMPostSequence != "RV" || o.EBMExtraBlind != 0 { |
| 197 | t.Errorf("post sequence %q extra %d, want RV/0", o.EBMPostSequence, o.EBMExtraBlind) |
| 198 | } |
| 199 | got := renderOutcomeProgress([]result{{Trajectory: s}}) |
| 200 | if !strings.Contains(got, "**EBM** eligible 1 · fired 1 (compliance ≤1 extra mutation 100%, median rounds-to-check 2)") { |
| 201 | t.Errorf("render missing EBM segment:\n%s", got) |
| 202 | } |
| 203 | } |
| 204 | |
| 205 | func TestRenderOutcomeProgressAggregatesRuns(t *testing.T) { |
| 206 | results := []result{ |
| 207 | {Trajectory: &trajectorySummary{Outcome: &outcomeSummary{ |
| 208 | Rounds: 8, ProgressRounds: 5, FalseProgressRounds: 2, |
| 209 | Objective: 2, BestScore: 2, FinalScore: 2, |
| 210 | }}}, |
| 211 | {Trajectory: &trajectorySummary{Outcome: &outcomeSummary{ |
| 212 | Objective: 1, Regression: 1, BestScore: 1, FinalScore: 0, |
| 213 | RegressedFromBest: true, SearchRegretMs: 4000, Backfilled: true, |
| 214 | }}}, |
| 215 | {Trajectory: &trajectorySummary{}}, // no outcome data: excluded |
| 216 | } |
| 217 | got := renderOutcomeProgress(results) |
| 218 | for _, want := range []string{ |
| 219 | "**Outcome shadow** (2 runs)", |
| 220 | "**objective transitions** 3", |
| 221 | "**regressed from best** 1 (50%)", |
| 222 | "**false progress** 2/5 (40%)", |
| 223 | "**avg search regret** 4.0s", |
| 224 | "backfilled 1", |
| 225 | } { |
| 226 | if !strings.Contains(got, want) { |
| 227 | t.Errorf("render missing %q in:\n%s", want, got) |
| 228 | } |
| 229 | } |
| 230 | if renderOutcomeProgress(nil) != "" { |
| 231 | t.Error("no runs must render nothing") |
| 232 | } |
| 233 | } |
| 234 |