| 1 | package evidence |
| 2 | |
| 3 | import ( |
| 4 | "path" |
| 5 | "sort" |
| 6 | "strings" |
| 7 | |
| 8 | "reasonix/internal/shellsafe" |
| 9 | ) |
| 10 | |
| 11 | // OutcomeSample decomposes one tool round's receipts by outcome: information |
| 12 | // gathered (Exploration), verification attempts run, and verification-command |
| 13 | // state transitions (Objective fail→pass, Regression pass→fail). Counts are |
| 14 | // unit-weighted; policy weighting is an offline concern. |
| 15 | type OutcomeSample struct { |
| 16 | Round int |
| 17 | Exploration int |
| 18 | Verification int |
| 19 | Objective int |
| 20 | Regression int |
| 21 | Churn int |
| 22 | // LegacyGain is the live novelty scorer's verdict on the same receipts, so |
| 23 | // offline analysis can compare the two policies without replaying. |
| 24 | LegacyGain int |
| 25 | // Discriminating counts observations able to falsify the working |
| 26 | // hypothesis: verification commands, or commands exercising a mutated |
| 27 | // file — deliberately broader than delivery verification (repro scripts). |
| 28 | Discriminating int |
| 29 | // DebtAge counts consecutive rounds carrying an unverified mutation with |
| 30 | // no discriminating observation; 0 while no verification debt is open. |
| 31 | DebtAge int |
| 32 | // BlindMutations counts mutations since the last discriminating |
| 33 | // observation — the EBM policy's trigger input. |
| 34 | BlindMutations int |
| 35 | // EBMEligible/EBMFired mark the Evidence-Before-More-Mutation trigger |
| 36 | // holding and its nudge firing; the agent stamps both so every arm — |
| 37 | // baseline included — carries the eligibility shadow. |
| 38 | EBMEligible bool |
| 39 | EBMFired bool |
| 40 | // LocalExecSeen reports whether this turn has executed any local |
| 41 | // interpreter/test command yet — the self-check-propensity observable |
| 42 | // (studied set: python/node/go run/pytest; ecosystem bias documented). |
| 43 | LocalExecSeen bool |
| 44 | // GovernorEligible/GovernorEngaged mark the reasoning governor's |
| 45 | // exploration trigger holding and its depth override riding requests; |
| 46 | // eligibility is stamped on every arm so baselines carry the shadow. |
| 47 | GovernorEligible bool |
| 48 | GovernorEngaged bool |
| 49 | // Runway fields are a telemetry-only counterfactual stamped by the outcome |
| 50 | // shadow. No runtime guard or provider-visible message reads them. |
| 51 | Runway int |
| 52 | RunwayDry int |
| 53 | RunwayIdle int |
| 54 | RunwaySpent bool |
| 55 | } |
| 56 | |
| 57 | // OutcomeTracker is the shadow counterpart of ProgressTracker: same per-round |
| 58 | // receipts, scored by outcome instead of novelty. It never influences guard |
| 59 | // behavior — samples exist only for trajectory recording and offline analysis. |
| 60 | type OutcomeTracker struct { |
| 61 | legacy *ProgressTracker |
| 62 | round int |
| 63 | readPaths map[string]bool |
| 64 | commands map[string]bool |
| 65 | failures map[string]bool |
| 66 | actions map[string]bool |
| 67 | verifySeen map[string]bool |
| 68 | verifyPass map[string]bool |
| 69 | mutatedBases map[string]bool |
| 70 | debt bool |
| 71 | debtAge int |
| 72 | blind int |
| 73 | localExec bool |
| 74 | runway runwayShadow |
| 75 | } |
| 76 | |
| 77 | // OutcomeSeed is the fork-portable slice of tracker state: what a |
| 78 | // counterfactual continuation must inherit for its shadow to stay continuous. |
| 79 | type OutcomeSeed struct { |
| 80 | MutatedBases []string `json:"mutated_bases,omitempty"` |
| 81 | DebtAge int `json:"debt_age"` |
| 82 | BlindMutations int `json:"blind_mutations"` |
| 83 | LocalExecSeen bool `json:"local_exec_seen"` |
| 84 | RunwayBalance int `json:"runway_balance,omitempty"` |
| 85 | RunwayDry int `json:"runway_dry,omitempty"` |
| 86 | RunwayIdle int `json:"runway_idle,omitempty"` |
| 87 | RunwayObserved bool `json:"runway_observed,omitempty"` |
| 88 | } |
| 89 | |
| 90 | // ForkSeed exports the state a counterfactual fork must carry so post-fork |
| 91 | // discriminating detection stays continuous with the original run. |
| 92 | func (t *OutcomeTracker) ForkSeed() OutcomeSeed { |
| 93 | seed := OutcomeSeed{ |
| 94 | DebtAge: t.debtAge, BlindMutations: t.blind, LocalExecSeen: t.localExec, |
| 95 | RunwayBalance: t.runway.balance, RunwayDry: t.runway.dry, |
| 96 | RunwayIdle: t.runway.idle, RunwayObserved: t.runway.observed, |
| 97 | } |
| 98 | for base := range t.mutatedBases { |
| 99 | seed.MutatedBases = append(seed.MutatedBases, base) |
| 100 | } |
| 101 | sort.Strings(seed.MutatedBases) |
| 102 | return seed |
| 103 | } |
| 104 | |
| 105 | // RestoreOutcomeTracker rebuilds a tracker from a fork seed. Novelty maps |
| 106 | // start empty — post-fork exploration novelty is intentionally relative to the |
| 107 | // fork point, while debt state continues from the original trajectory. |
| 108 | func RestoreOutcomeTracker(seed OutcomeSeed) *OutcomeTracker { |
| 109 | t := NewOutcomeTracker() |
| 110 | for _, base := range seed.MutatedBases { |
| 111 | t.mutatedBases[base] = true |
| 112 | } |
| 113 | t.debtAge = seed.DebtAge |
| 114 | t.blind = seed.BlindMutations |
| 115 | t.debt = seed.DebtAge > 0 || seed.BlindMutations > 0 |
| 116 | t.localExec = seed.LocalExecSeen |
| 117 | t.runway = runwayShadow{ |
| 118 | balance: seed.RunwayBalance, dry: seed.RunwayDry, |
| 119 | idle: seed.RunwayIdle, observed: seed.RunwayObserved, |
| 120 | } |
| 121 | return t |
| 122 | } |
| 123 | |
| 124 | func NewOutcomeTracker() *OutcomeTracker { |
| 125 | return &OutcomeTracker{ |
| 126 | legacy: NewProgressTracker(), |
| 127 | readPaths: map[string]bool{}, |
| 128 | commands: map[string]bool{}, |
| 129 | failures: map[string]bool{}, |
| 130 | actions: map[string]bool{}, |
| 131 | verifySeen: map[string]bool{}, |
| 132 | verifyPass: map[string]bool{}, |
| 133 | mutatedBases: map[string]bool{}, |
| 134 | } |
| 135 | } |
| 136 | |
| 137 | // ScoreRound folds one round's receipts into the tracker and returns the |
| 138 | // round's outcome decomposition. |
| 139 | func (t *OutcomeTracker) ScoreRound(receipts []Receipt) OutcomeSample { |
| 140 | if t == nil { |
| 141 | return OutcomeSample{} |
| 142 | } |
| 143 | t.round++ |
| 144 | s := OutcomeSample{Round: t.round} |
| 145 | for _, r := range receipts { |
| 146 | t.scoreReceipt(r, &s) |
| 147 | } |
| 148 | s.LegacyGain = t.legacy.ScoreRound(receipts) |
| 149 | // Verification debt: a discriminating observation settles it; otherwise a |
| 150 | // mutation opens it and every silent round ages it, mutation round included. |
| 151 | if s.Discriminating > 0 { |
| 152 | t.debt, t.debtAge, t.blind = false, 0, 0 |
| 153 | } else { |
| 154 | if s.Churn > 0 { |
| 155 | t.debt = true |
| 156 | t.blind += s.Churn |
| 157 | } |
| 158 | if t.debt { |
| 159 | t.debtAge++ |
| 160 | } |
| 161 | } |
| 162 | s.DebtAge = t.debtAge |
| 163 | s.BlindMutations = t.blind |
| 164 | s.LocalExecSeen = t.localExec |
| 165 | runway := t.runway.observe(s) |
| 166 | s.Runway, s.RunwayDry, s.RunwayIdle, s.RunwaySpent = |
| 167 | runway.balance, runway.dry, runway.idle, runway.spent |
| 168 | return s |
| 169 | } |
| 170 | |
| 171 | // localExecCommand matches the exact command families the affordance study |
| 172 | // validated. Deliberately narrow and Python-ecosystem biased for now; |
| 173 | // generalizing to Local Discriminating Execution needs cross-language |
| 174 | // replication first. |
| 175 | func localExecCommand(command string) bool { |
| 176 | for _, marker := range []string{"python", "node ", "go run", "pytest", "py.test"} { |
| 177 | if strings.Contains(command, marker) { |
| 178 | return true |
| 179 | } |
| 180 | } |
| 181 | return false |
| 182 | } |
| 183 | |
| 184 | // noteMutatedPaths remembers mutated file basenames so a later command that |
| 185 | // mentions one (running a repro script, a targeted test file) reads as a |
| 186 | // discriminating observation even when it is not delivery verification. |
| 187 | func (t *OutcomeTracker) noteMutatedPaths(paths []string) { |
| 188 | for _, p := range paths { |
| 189 | if base := path.Base(strings.ReplaceAll(p, "\\", "/")); len(base) >= 3 { |
| 190 | t.mutatedBases[base] = true |
| 191 | } |
| 192 | } |
| 193 | } |
| 194 | |
| 195 | func (t *OutcomeTracker) commandExercisesMutation(command string) bool { |
| 196 | // Inspecting a mutated file (cat/grep/head) cannot falsify anything; only |
| 197 | // a command that can execute it discriminates. |
| 198 | if !shellsafe.ClassifyBash(command).AnyMutation() { |
| 199 | return false |
| 200 | } |
| 201 | for base := range t.mutatedBases { |
| 202 | if strings.Contains(command, base) { |
| 203 | return true |
| 204 | } |
| 205 | } |
| 206 | return false |
| 207 | } |
| 208 | |
| 209 | func (t *OutcomeTracker) scoreReceipt(r Receipt, s *OutcomeSample) { |
| 210 | if command := strings.TrimSpace(r.Command); command != "" { |
| 211 | t.scoreCommand(command, r, s) |
| 212 | return |
| 213 | } |
| 214 | switch { |
| 215 | case r.Success && (r.Mutation || r.Write): |
| 216 | // A mutation is a state transition, not proof of progress: it counts |
| 217 | // as churn until a verification transition vouches for it. |
| 218 | s.Churn++ |
| 219 | t.noteMutatedPaths(r.Paths) |
| 220 | case r.Success && (r.ToolName == "task" || r.ToolName == "parallel_tasks" || r.ToolName == "fleet"): |
| 221 | // A delegation return is new information at best — never objective |
| 222 | // progress on its own. |
| 223 | s.Exploration++ |
| 224 | case r.Success && (r.StepProof || r.TodoStep != nil || len(r.Todos) > 0): |
| 225 | // Bookkeeping moves no outcome dimension. |
| 226 | case r.Success && r.Read && r.OutputBytes > 0 && len(r.Paths) > 0: |
| 227 | fresh := 0 |
| 228 | for _, path := range r.Paths { |
| 229 | if path == "" || t.readPaths[path] { |
| 230 | continue |
| 231 | } |
| 232 | t.readPaths[path] = true |
| 233 | fresh++ |
| 234 | } |
| 235 | // A path already read can still answer a question never asked: a new |
| 236 | // grep pattern over the same package, the next window of a long file. |
| 237 | if newQuestion := t.noteQuestion(r); fresh == 0 && newQuestion { |
| 238 | fresh = 1 |
| 239 | } |
| 240 | s.Exploration += fresh |
| 241 | case r.Success: |
| 242 | if t.noteQuestion(r) { |
| 243 | s.Exploration++ |
| 244 | } |
| 245 | } |
| 246 | } |
| 247 | |
| 248 | func (t *OutcomeTracker) noteQuestion(r Receipt) bool { |
| 249 | sig := r.ToolName + "\x00" + string(r.Args) |
| 250 | if t.actions[sig] { |
| 251 | return false |
| 252 | } |
| 253 | t.actions[sig] = true |
| 254 | return true |
| 255 | } |
| 256 | |
| 257 | func (t *OutcomeTracker) scoreCommand(command string, r Receipt, s *OutcomeSample) { |
| 258 | if r.Success && (r.Mutation || r.Write) { |
| 259 | s.Churn++ |
| 260 | t.noteMutatedPaths(r.Paths) |
| 261 | } |
| 262 | verify := IsVerificationCommand(command) |
| 263 | if verify || t.commandExercisesMutation(command) { |
| 264 | s.Discriminating++ |
| 265 | } |
| 266 | if localExecCommand(command) { |
| 267 | t.localExec = true |
| 268 | } |
| 269 | if verify { |
| 270 | s.Verification++ |
| 271 | seen, wasPass := t.verifySeen[command], t.verifyPass[command] |
| 272 | t.verifySeen[command] = true |
| 273 | t.verifyPass[command] = r.Success |
| 274 | if seen && r.Success && !wasPass { |
| 275 | s.Objective++ |
| 276 | } |
| 277 | if seen && !r.Success && wasPass { |
| 278 | s.Regression++ |
| 279 | } |
| 280 | } |
| 281 | if r.Success { |
| 282 | if !verify && !t.commands[command] { |
| 283 | s.Exploration++ |
| 284 | } |
| 285 | t.commands[command] = true |
| 286 | return |
| 287 | } |
| 288 | if !t.failures[command] { |
| 289 | t.failures[command] = true |
| 290 | s.Exploration++ |
| 291 | } |
| 292 | } |
| 293 |