| 1 | package evidence |
| 2 | |
| 3 | import ( |
| 4 | "encoding/json" |
| 5 | "fmt" |
| 6 | "testing" |
| 7 | ) |
| 8 | |
| 9 | func TestOutcomeTrackerSeparatesExplorationFromObjective(t *testing.T) { |
| 10 | tr := NewOutcomeTracker() |
| 11 | |
| 12 | read := readReceipt("a.go") |
| 13 | read.OutputBytes = 10 |
| 14 | s := tr.ScoreRound([]Receipt{read}) |
| 15 | if s.Exploration != 1 || s.Objective != 0 { |
| 16 | t.Fatalf("new read = %+v, want exploration 1 objective 0", s) |
| 17 | } |
| 18 | |
| 19 | // First verification failure: an attempt plus localization, no objective. |
| 20 | s = tr.ScoreRound([]Receipt{bashReceipt("go test ./x", false)}) |
| 21 | if s.Verification != 1 || s.Exploration != 1 || s.Objective != 0 { |
| 22 | t.Fatalf("first failing verify = %+v, want verification 1 exploration 1", s) |
| 23 | } |
| 24 | |
| 25 | // A mutation is churn, not objective progress — the legacy scorer disagrees. |
| 26 | write := ReceiptFromToolCall("write_file", json.RawMessage(`{"path":"b.go","content":"x"}`), true, false) |
| 27 | s = tr.ScoreRound([]Receipt{write}) |
| 28 | if s.Churn != 1 || s.Objective != 0 || s.Exploration != 0 { |
| 29 | t.Fatalf("mutation = %+v, want churn 1 only", s) |
| 30 | } |
| 31 | if s.LegacyGain != gainMutation { |
| 32 | t.Fatalf("mutation legacy gain = %d, want %d", s.LegacyGain, gainMutation) |
| 33 | } |
| 34 | |
| 35 | // The failing verification turning green is the objective transition. |
| 36 | s = tr.ScoreRound([]Receipt{bashReceipt("go test ./x", true)}) |
| 37 | if s.Objective != 1 || s.Verification != 1 || s.Regression != 0 { |
| 38 | t.Fatalf("fail→pass verify = %+v, want objective 1", s) |
| 39 | } |
| 40 | |
| 41 | // The same verification breaking again is a regression. |
| 42 | s = tr.ScoreRound([]Receipt{bashReceipt("go test ./x", false)}) |
| 43 | if s.Regression != 1 || s.Objective != 0 { |
| 44 | t.Fatalf("pass→fail verify = %+v, want regression 1", s) |
| 45 | } |
| 46 | } |
| 47 | |
| 48 | func TestOutcomeTrackerDelegationAndRepeatsAreExplorationAtBest(t *testing.T) { |
| 49 | tr := NewOutcomeTracker() |
| 50 | |
| 51 | task := ReceiptFromToolCall("task", json.RawMessage(`{"prompt":"dig"}`), true, false) |
| 52 | s := tr.ScoreRound([]Receipt{task}) |
| 53 | if s.Exploration != 1 || s.Objective != 0 { |
| 54 | t.Fatalf("delegation = %+v, want exploration 1 objective 0", s) |
| 55 | } |
| 56 | |
| 57 | // A first passing verification run establishes a baseline, not progress. |
| 58 | s = tr.ScoreRound([]Receipt{bashReceipt("go vet ./...", true)}) |
| 59 | if s.Verification != 1 || s.Objective != 0 || s.Exploration != 0 { |
| 60 | t.Fatalf("baseline verify = %+v, want verification 1 only", s) |
| 61 | } |
| 62 | s = tr.ScoreRound([]Receipt{bashReceipt("go vet ./...", true)}) |
| 63 | if s.Verification != 1 || s.Objective != 0 { |
| 64 | t.Fatalf("repeated passing verify = %+v, want no objective", s) |
| 65 | } |
| 66 | |
| 67 | // A repeat delegation still returned content the host cannot judge — it |
| 68 | // stays exploration and can never move the objective dimension. |
| 69 | repeat := ReceiptFromToolCall("task", json.RawMessage(`{"prompt":"dig"}`), true, false) |
| 70 | s = tr.ScoreRound([]Receipt{repeat}) |
| 71 | if s.Exploration != 1 || s.Objective != 0 { |
| 72 | t.Fatalf("repeated delegation = %+v, want exploration 1 objective 0", s) |
| 73 | } |
| 74 | |
| 75 | var nilTracker *OutcomeTracker |
| 76 | if got := nilTracker.ScoreRound([]Receipt{task}); got != (OutcomeSample{}) { |
| 77 | t.Fatalf("nil tracker sample = %+v, want zero", got) |
| 78 | } |
| 79 | } |
| 80 | |
| 81 | func TestOutcomeTrackerVerificationDebtLifecycle(t *testing.T) { |
| 82 | tr := NewOutcomeTracker() |
| 83 | |
| 84 | // A mutation opens debt; silent rounds age it. |
| 85 | write := ReceiptFromToolCall("write_file", json.RawMessage(`{"path":"pkg/repro.py","content":"x"}`), true, false) |
| 86 | if s := tr.ScoreRound([]Receipt{write}); s.DebtAge != 1 || s.Discriminating != 0 { |
| 87 | t.Fatalf("mutation round = %+v, want debt age 1", s) |
| 88 | } |
| 89 | read := readReceipt("other.go") |
| 90 | read.OutputBytes = 5 |
| 91 | if s := tr.ScoreRound([]Receipt{read}); s.DebtAge != 2 { |
| 92 | t.Fatalf("silent round = %+v, want debt age 2", s) |
| 93 | } |
| 94 | // An unrelated command does not discriminate. |
| 95 | if s := tr.ScoreRound([]Receipt{bashReceipt("ls -la", true)}); s.DebtAge != 3 || s.Discriminating != 0 { |
| 96 | t.Fatalf("unrelated command = %+v, want debt age 3", s) |
| 97 | } |
| 98 | // Reading the mutated file is inspection, not discrimination: debt ages on. |
| 99 | if s := tr.ScoreRound([]Receipt{bashReceipt("cat pkg/repro.py", true)}); s.Discriminating != 0 || s.DebtAge != 4 { |
| 100 | t.Fatalf("read-only inspection = %+v, want no discrimination, debt age 4", s) |
| 101 | } |
| 102 | // A second mutation raises the blind count; the counter tracks mutations, |
| 103 | // not rounds. |
| 104 | if s := tr.ScoreRound([]Receipt{ReceiptFromToolCall("write_file", json.RawMessage(`{"path":"pkg/b.py","content":"y"}`), true, false)}); s.BlindMutations != 2 { |
| 105 | t.Fatalf("second mutation = %+v, want blind 2", s) |
| 106 | } |
| 107 | // Running the mutated file is a discriminating observation even though it |
| 108 | // is not delivery verification: debt and the blind count settle. |
| 109 | if s := tr.ScoreRound([]Receipt{bashReceipt("python3 pkg/repro.py", false)}); s.Discriminating != 1 || s.DebtAge != 0 || s.BlindMutations != 0 { |
| 110 | t.Fatalf("repro run = %+v, want discriminating 1, debt and blind settled", s) |
| 111 | } |
| 112 | // Debt stays settled until the next mutation; delivery verification also |
| 113 | // counts as discriminating without any mutated-path match. |
| 114 | if s := tr.ScoreRound([]Receipt{bashReceipt("go test ./pkg", true)}); s.Discriminating != 1 || s.DebtAge != 0 { |
| 115 | t.Fatalf("verification round = %+v, want discriminating 1, no debt", s) |
| 116 | } |
| 117 | } |
| 118 | |
| 119 | func TestOutcomeTrackerCarriesRunwayShadowAcrossForks(t *testing.T) { |
| 120 | tracker := NewOutcomeTracker() |
| 121 | var before OutcomeSample |
| 122 | for range 5 { |
| 123 | before = tracker.ScoreRound(nil) |
| 124 | } |
| 125 | if before.Runway != runwayRoundCost || before.RunwaySpent { |
| 126 | t.Fatalf("pre-fork runway = %+v, want one empty round remaining", before) |
| 127 | } |
| 128 | |
| 129 | restored := RestoreOutcomeTracker(tracker.ForkSeed()) |
| 130 | after := restored.ScoreRound(nil) |
| 131 | if after.Runway != 0 || !after.RunwaySpent || after.RunwayDry != 6 || after.RunwayIdle != 6 { |
| 132 | t.Fatalf("post-fork runway = %+v, want continuous spent transition", after) |
| 133 | } |
| 134 | } |
| 135 | |
| 136 | func TestRunwayShadowDoesNotReplaceTheLiveNoveltyScorer(t *testing.T) { |
| 137 | tracker := NewOutcomeTracker() |
| 138 | var sample OutcomeSample |
| 139 | for i := range explorationRunLimit + 1 { |
| 140 | read := readReceipt(fmt.Sprintf("file-%d.go", i)) |
| 141 | read.OutputBytes = 1 |
| 142 | sample = tracker.ScoreRound([]Receipt{read}) |
| 143 | } |
| 144 | if sample.Exploration != 1 || sample.LegacyGain != 0 { |
| 145 | t.Fatalf("comparison sample = %+v, want outcome exploration while the unchanged live scorer is zero", sample) |
| 146 | } |
| 147 | if sample.Runway != runwayStartBalance-(explorationRunLimit+1) { |
| 148 | t.Fatalf("runway balance = %d, want independent shadow accounting", sample.Runway) |
| 149 | } |
| 150 | } |
| 151 |