| 1 | package main |
| 2 | |
| 3 | import "fmt" |
| 4 | |
| 5 | // outcomeSummary condenses one run's outcome-progress series: did claimed |
| 6 | // progress become objective transitions, and did the run end below its best |
| 7 | // verified state. Backfilled summaries derive from shell verification receipts |
| 8 | // when the recording predates the runtime shadow scorer. |
| 9 | type outcomeSummary struct { |
| 10 | Rounds int `json:"rounds,omitempty"` |
| 11 | ProgressRounds int `json:"progress_rounds,omitempty"` |
| 12 | FalseProgressRounds int `json:"false_progress_rounds,omitempty"` |
| 13 | SolutionStallMax int `json:"solution_stall_max,omitempty"` |
| 14 | Objective int `json:"objective,omitempty"` |
| 15 | Regression int `json:"regression,omitempty"` |
| 16 | BestScore int `json:"best_score,omitempty"` |
| 17 | FinalScore int `json:"final_score"` |
| 18 | RegressedFromBest bool `json:"regressed_from_best,omitempty"` |
| 19 | SearchRegretMs int64 `json:"search_regret_ms,omitempty"` |
| 20 | // TTFDCMs is run start to the first discriminating observation (zero = |
| 21 | // never); DebtAgeMax the worst stretch of rounds with an unverified mutation. |
| 22 | TTFDCMs int64 `json:"ttfdc_ms,omitempty"` |
| 23 | DebtAgeMax int `json:"debt_age_max,omitempty"` |
| 24 | Backfilled bool `json:"backfilled,omitempty"` |
| 25 | |
| 26 | // EBM chain: when the Evidence-Before-More-Mutation trigger held, when its |
| 27 | // nudge fired, and what followed — compliance is the mechanism-health |
| 28 | // readout that separates "nudge ignored" from "early evidence useless". |
| 29 | DebtArea int `json:"debt_area,omitempty"` |
| 30 | BlindPeak int `json:"blind_peak,omitempty"` |
| 31 | EBMEligibleRound int `json:"ebm_eligible_round,omitempty"` |
| 32 | EBMFiredRound int `json:"ebm_fired_round,omitempty"` |
| 33 | EBMBlindAtFire int `json:"ebm_blind_at_fire,omitempty"` |
| 34 | EBMDebtAgeAtFire int `json:"ebm_debt_age_at_fire,omitempty"` |
| 35 | EBMRoundsToCheck int `json:"ebm_rounds_to_check,omitempty"` |
| 36 | EBMMsToCheck int64 `json:"ebm_ms_to_check,omitempty"` |
| 37 | EBMReasoningToCheck int64 `json:"ebm_reasoning_to_check,omitempty"` |
| 38 | EBMCheckWithin1 bool `json:"ebm_check_within_1,omitempty"` |
| 39 | EBMCheckWithin2 bool `json:"ebm_check_within_2,omitempty"` |
| 40 | // EBMExtraBlind counts mutations between the nudge and the first |
| 41 | // discriminating check: 0 = strong compliance, 1 = finishing the minimum |
| 42 | // coherent patch the copy permits, >=2 = real non-compliance. |
| 43 | EBMExtraBlind int `json:"ebm_extra_blind_mutations"` |
| 44 | EBMPostSequence string `json:"ebm_post_sequence,omitempty"` |
| 45 | |
| 46 | // Governor chain: rounds the exploration trigger held, rounds the depth |
| 47 | // override actually rode requests, and where the first engagement sat. |
| 48 | GovernorEligibleRounds int `json:"governor_eligible_rounds,omitempty"` |
| 49 | GovernorEngagedRounds int `json:"governor_engaged_rounds,omitempty"` |
| 50 | GovernorFirstRound int `json:"governor_first_round,omitempty"` |
| 51 | |
| 52 | // Runway shadow fields are absent for trajectories recorded before the |
| 53 | // experiment. FirstSpentRound is the counterfactual intervention point. |
| 54 | RunwaySamples int `json:"runway_samples,omitempty"` |
| 55 | RunwayMin int `json:"runway_min,omitempty"` |
| 56 | RunwayFinal int `json:"runway_final,omitempty"` |
| 57 | RunwayDryMax int `json:"runway_dry_max,omitempty"` |
| 58 | RunwayIdleMax int `json:"runway_idle_max,omitempty"` |
| 59 | RunwayFirstSpentRound int `json:"runway_first_spent_round,omitempty"` |
| 60 | } |
| 61 | |
| 62 | // outcomePoint is one recorded shadow sample plus its observation time. |
| 63 | type outcomePoint struct { |
| 64 | ts int64 |
| 65 | round int |
| 66 | exploration, verification, objective, regression, churn int |
| 67 | legacyGain, discriminating, debtAge, blindMutations int |
| 68 | ebmEligible, ebmFired bool |
| 69 | governorEligible, governorEngaged bool |
| 70 | runway *int |
| 71 | runwayDry, runwayIdle int |
| 72 | runwaySpent bool |
| 73 | } |
| 74 | |
| 75 | // verifyPoint is one backfilled verification-transition observation. |
| 76 | type verifyPoint struct { |
| 77 | ts int64 |
| 78 | objective, regression int |
| 79 | } |
| 80 | |
| 81 | // falseProgressWindow bounds how many later rounds may redeem a legacy |
| 82 | // progress claim with an objective transition before the round counts false. |
| 83 | const falseProgressWindow = 3 |
| 84 | |
| 85 | // observeVerification folds one verification-classified shell result into the |
| 86 | // backfill series; key identity is the exact (name, args) announcement. |
| 87 | func (t *trajScan) observeVerification(key string, passed bool, ts int64) { |
| 88 | if t.verifyPass == nil { |
| 89 | t.verifySeen = map[string]bool{} |
| 90 | t.verifyPass = map[string]bool{} |
| 91 | } |
| 92 | seen, was := t.verifySeen[key], t.verifyPass[key] |
| 93 | t.verifySeen[key] = true |
| 94 | t.verifyPass[key] = passed |
| 95 | p := verifyPoint{ts: ts} |
| 96 | if seen && passed && !was { |
| 97 | p.objective = 1 |
| 98 | } |
| 99 | if seen && !passed && was { |
| 100 | p.regression = 1 |
| 101 | } |
| 102 | t.verifyPoints = append(t.verifyPoints, p) |
| 103 | } |
| 104 | |
| 105 | // summarizeOutcome prefers recorded shadow samples; older recordings fall back |
| 106 | // to the verification backfill, which cannot price legacy-scorer claims. |
| 107 | func (t *trajScan) summarizeOutcome() *outcomeSummary { |
| 108 | if len(t.outcomePoints) > 0 { |
| 109 | o := summarizeOutcomePoints(t.outcomePoints, t.firstTS, t.lastTS) |
| 110 | t.attachEBMChain(o) |
| 111 | return o |
| 112 | } |
| 113 | if len(t.verifyPoints) > 0 { |
| 114 | return summarizeVerifyBackfill(t.verifyPoints, t.lastTS) |
| 115 | } |
| 116 | return nil |
| 117 | } |
| 118 | |
| 119 | func summarizeOutcomePoints(points []outcomePoint, firstTS, lastTS int64) *outcomeSummary { |
| 120 | o := &outcomeSummary{Rounds: len(points)} |
| 121 | verifying, solution, stall := false, false, 0 |
| 122 | score, best := 0, 0 |
| 123 | var bestTS int64 |
| 124 | for i, p := range points { |
| 125 | if p.verification > 0 { |
| 126 | verifying = true |
| 127 | } |
| 128 | if p.discriminating > 0 && o.TTFDCMs == 0 && p.ts > firstTS { |
| 129 | o.TTFDCMs = p.ts - firstTS |
| 130 | } |
| 131 | o.DebtAgeMax = max(o.DebtAgeMax, p.debtAge) |
| 132 | o.Objective += p.objective |
| 133 | o.Regression += p.regression |
| 134 | if p.legacyGain > 0 { |
| 135 | o.ProgressRounds++ |
| 136 | } |
| 137 | if p.runway != nil { |
| 138 | if o.RunwaySamples == 0 { |
| 139 | o.RunwayMin = *p.runway |
| 140 | } |
| 141 | o.RunwaySamples++ |
| 142 | o.RunwayMin = min(o.RunwayMin, *p.runway) |
| 143 | o.RunwayFinal = *p.runway |
| 144 | o.RunwayDryMax = max(o.RunwayDryMax, p.runwayDry) |
| 145 | o.RunwayIdleMax = max(o.RunwayIdleMax, p.runwayIdle) |
| 146 | if o.RunwayFirstSpentRound == 0 && p.runwaySpent { |
| 147 | o.RunwayFirstSpentRound = p.round |
| 148 | if o.RunwayFirstSpentRound == 0 { |
| 149 | o.RunwayFirstSpentRound = i + 1 |
| 150 | } |
| 151 | } |
| 152 | } |
| 153 | // The solution stall clock only starts once the run enters its solution |
| 154 | // phase (a verification attempt or a mutation); pure research runs with |
| 155 | // no verification would otherwise read as one long stall. |
| 156 | if p.verification > 0 || p.churn > 0 { |
| 157 | solution = true |
| 158 | } |
| 159 | if solution { |
| 160 | if p.objective > 0 { |
| 161 | stall = 0 |
| 162 | } else { |
| 163 | stall++ |
| 164 | o.SolutionStallMax = max(o.SolutionStallMax, stall) |
| 165 | } |
| 166 | } |
| 167 | score += p.objective - p.regression |
| 168 | if score > best { |
| 169 | best, bestTS = score, p.ts |
| 170 | } |
| 171 | } |
| 172 | if verifying { |
| 173 | for i, p := range points { |
| 174 | if p.legacyGain <= 0 { |
| 175 | continue |
| 176 | } |
| 177 | redeemed := false |
| 178 | for j := i; j < min(i+1+falseProgressWindow, len(points)); j++ { |
| 179 | if points[j].objective > 0 { |
| 180 | redeemed = true |
| 181 | break |
| 182 | } |
| 183 | } |
| 184 | if !redeemed { |
| 185 | o.FalseProgressRounds++ |
| 186 | } |
| 187 | } |
| 188 | } |
| 189 | finishScore(o, score, best, bestTS, lastTS) |
| 190 | return o |
| 191 | } |
| 192 | |
| 193 | // attachEBMChain condenses the per-round EBM shadow into the run's chain |
| 194 | // facts; rounds-to-check joins the cognition digests for the tokens spent |
| 195 | // between fire and first discriminating observation. |
| 196 | func (t *trajScan) attachEBMChain(o *outcomeSummary) { |
| 197 | pts := t.outcomePoints |
| 198 | fire := -1 |
| 199 | for i, p := range pts { |
| 200 | o.DebtArea += p.debtAge |
| 201 | o.BlindPeak = max(o.BlindPeak, p.blindMutations) |
| 202 | if o.EBMEligibleRound == 0 && p.ebmEligible { |
| 203 | o.EBMEligibleRound = i + 1 |
| 204 | } |
| 205 | if p.governorEligible { |
| 206 | o.GovernorEligibleRounds++ |
| 207 | } |
| 208 | if p.governorEngaged { |
| 209 | o.GovernorEngagedRounds++ |
| 210 | if o.GovernorFirstRound == 0 { |
| 211 | o.GovernorFirstRound = i + 1 |
| 212 | } |
| 213 | } |
| 214 | if fire < 0 && p.ebmFired { |
| 215 | fire = i |
| 216 | } |
| 217 | } |
| 218 | if fire < 0 { |
| 219 | return |
| 220 | } |
| 221 | o.EBMFiredRound = fire + 1 |
| 222 | o.EBMBlindAtFire = pts[fire].blindMutations |
| 223 | o.EBMDebtAgeAtFire = pts[fire].debtAge |
| 224 | for j := fire + 1; j < len(pts); j++ { |
| 225 | o.EBMPostSequence += postEBMCategory(pts[j]) |
| 226 | if pts[j].discriminating == 0 { |
| 227 | o.EBMExtraBlind += pts[j].churn |
| 228 | continue |
| 229 | } |
| 230 | o.EBMRoundsToCheck = j - fire |
| 231 | o.EBMMsToCheck = pts[j].ts - pts[fire].ts |
| 232 | o.EBMCheckWithin1 = j-fire <= 1 |
| 233 | o.EBMCheckWithin2 = j-fire <= 2 |
| 234 | for k := fire + 1; k <= j && k < len(t.s.Rounds); k++ { |
| 235 | o.EBMReasoningToCheck += t.s.Rounds[k].ReasoningTokens |
| 236 | } |
| 237 | return |
| 238 | } |
| 239 | } |
| 240 | |
| 241 | // postEBMCategory letters the rounds after a nudge: V discriminating check, |
| 242 | // M mutation, R new information, "." quiet. |
| 243 | func postEBMCategory(p outcomePoint) string { |
| 244 | switch { |
| 245 | case p.discriminating > 0: |
| 246 | return "V" |
| 247 | case p.churn > 0: |
| 248 | return "M" |
| 249 | case p.exploration > 0: |
| 250 | return "R" |
| 251 | default: |
| 252 | return "." |
| 253 | } |
| 254 | } |
| 255 | |
| 256 | // governorShadowLine aggregates the reasoning-governor shadow across runs; |
| 257 | // empty when no round was eligible. |
| 258 | func governorShadowLine(results []result) string { |
| 259 | runs, elig, engaged := 0, 0, 0 |
| 260 | for _, r := range results { |
| 261 | if r.Trajectory == nil || r.Trajectory.Outcome == nil { |
| 262 | continue |
| 263 | } |
| 264 | o := r.Trajectory.Outcome |
| 265 | if o.GovernorEligibleRounds > 0 { |
| 266 | runs++ |
| 267 | } |
| 268 | elig += o.GovernorEligibleRounds |
| 269 | engaged += o.GovernorEngagedRounds |
| 270 | } |
| 271 | if elig == 0 { |
| 272 | return "" |
| 273 | } |
| 274 | return fmt.Sprintf(" · **governor** eligible %d rounds in %d runs · engaged %d rounds", elig, runs, engaged) |
| 275 | } |
| 276 | |
| 277 | func summarizeVerifyBackfill(points []verifyPoint, lastTS int64) *outcomeSummary { |
| 278 | o := &outcomeSummary{Backfilled: true} |
| 279 | score, best := 0, 0 |
| 280 | var bestTS int64 |
| 281 | for _, p := range points { |
| 282 | o.Objective += p.objective |
| 283 | o.Regression += p.regression |
| 284 | score += p.objective - p.regression |
| 285 | if score > best { |
| 286 | best, bestTS = score, p.ts |
| 287 | } |
| 288 | } |
| 289 | finishScore(o, score, best, bestTS, lastTS) |
| 290 | return o |
| 291 | } |
| 292 | |
| 293 | func finishScore(o *outcomeSummary, score, best int, bestTS, lastTS int64) { |
| 294 | o.BestScore, o.FinalScore = best, score |
| 295 | if score < best { |
| 296 | o.RegressedFromBest = true |
| 297 | if lastTS > bestTS { |
| 298 | o.SearchRegretMs = lastTS - bestTS |
| 299 | } |
| 300 | } |
| 301 | } |
| 302 | |
| 303 | // renderOutcomeProgress aggregates the shadow scorer's verdicts: how often the |
| 304 | // live novelty scorer claimed progress that never became an objective |
| 305 | // transition, and how many runs peaked above their final verified state. |
| 306 | func renderOutcomeProgress(results []result) string { |
| 307 | runs, backfilled := 0, 0 |
| 308 | progress, falseProgress := 0, 0 |
| 309 | objective, regression, regressed := 0, 0, 0 |
| 310 | var regretMs int64 |
| 311 | stallMax := 0 |
| 312 | for _, r := range results { |
| 313 | if r.Trajectory == nil || r.Trajectory.Outcome == nil { |
| 314 | continue |
| 315 | } |
| 316 | o := r.Trajectory.Outcome |
| 317 | runs++ |
| 318 | if o.Backfilled { |
| 319 | backfilled++ |
| 320 | } |
| 321 | progress += o.ProgressRounds |
| 322 | falseProgress += o.FalseProgressRounds |
| 323 | objective += o.Objective |
| 324 | regression += o.Regression |
| 325 | stallMax = max(stallMax, o.SolutionStallMax) |
| 326 | if o.RegressedFromBest { |
| 327 | regressed++ |
| 328 | regretMs += o.SearchRegretMs |
| 329 | } |
| 330 | } |
| 331 | if runs == 0 { |
| 332 | return "" |
| 333 | } |
| 334 | discRuns, debtMax := 0, 0 |
| 335 | var ttfdcs []int64 |
| 336 | for _, r := range results { |
| 337 | if r.Trajectory == nil || r.Trajectory.Outcome == nil { |
| 338 | continue |
| 339 | } |
| 340 | o := r.Trajectory.Outcome |
| 341 | debtMax = max(debtMax, o.DebtAgeMax) |
| 342 | if o.TTFDCMs > 0 { |
| 343 | discRuns++ |
| 344 | ttfdcs = append(ttfdcs, o.TTFDCMs) |
| 345 | } |
| 346 | } |
| 347 | line := fmt.Sprintf("**Outcome shadow** (%d runs): **objective transitions** %d · **regressions** %d · **regressed from best** %d (%s)", |
| 348 | runs, objective, regression, regressed, pct(regressed, runs)) |
| 349 | if discRuns > 0 { |
| 350 | line += fmt.Sprintf(" · **discriminating checks** in %d/%d runs (TTFDC p50 %s)", |
| 351 | discRuns, runs, dur(median(ttfdcs))) |
| 352 | } |
| 353 | if debtMax > 0 { |
| 354 | line += fmt.Sprintf(" · **verification debt max** %d rounds", debtMax) |
| 355 | } |
| 356 | elig, fired, comply := 0, 0, 0 |
| 357 | var toCheck []int64 |
| 358 | for _, r := range results { |
| 359 | if r.Trajectory == nil || r.Trajectory.Outcome == nil { |
| 360 | continue |
| 361 | } |
| 362 | o := r.Trajectory.Outcome |
| 363 | if o.EBMEligibleRound > 0 { |
| 364 | elig++ |
| 365 | } |
| 366 | if o.EBMFiredRound > 0 { |
| 367 | fired++ |
| 368 | // Compliance judges behavior, not speed: at most one mutation |
| 369 | // between nudge and check — the coherent-patch allowance. |
| 370 | if o.EBMRoundsToCheck > 0 && o.EBMExtraBlind <= 1 { |
| 371 | comply++ |
| 372 | } |
| 373 | if o.EBMRoundsToCheck > 0 { |
| 374 | toCheck = append(toCheck, int64(o.EBMRoundsToCheck)) |
| 375 | } |
| 376 | } |
| 377 | } |
| 378 | if elig > 0 { |
| 379 | line += fmt.Sprintf(" · **EBM** eligible %d · fired %d", elig, fired) |
| 380 | if fired > 0 { |
| 381 | line += fmt.Sprintf(" (compliance ≤1 extra mutation %s, median rounds-to-check %d)", |
| 382 | pct(comply, fired), median(toCheck)) |
| 383 | } |
| 384 | } |
| 385 | line += governorShadowLine(results) |
| 386 | line += runwayShadowLine(results) |
| 387 | if progress > 0 { |
| 388 | line += fmt.Sprintf(" · **false progress** %d/%d (%s)", falseProgress, progress, pct(falseProgress, progress)) |
| 389 | } |
| 390 | if stallMax > 0 { |
| 391 | line += fmt.Sprintf(" · **solution stall max** %d rounds", stallMax) |
| 392 | } |
| 393 | if regressed > 0 { |
| 394 | line += fmt.Sprintf(" · **avg search regret** %s", dur(regretMs/int64(regressed))) |
| 395 | } |
| 396 | if backfilled > 0 { |
| 397 | line += fmt.Sprintf(" · backfilled %d", backfilled) |
| 398 | } |
| 399 | return line + "\n\n" |
| 400 | } |
| 401 | |
| 402 | func runwayShadowLine(results []result) string { |
| 403 | measured, spent, dryMax, idleMax := 0, 0, 0, 0 |
| 404 | var spentRounds []int64 |
| 405 | for _, r := range results { |
| 406 | if r.Trajectory == nil || r.Trajectory.Outcome == nil || r.Trajectory.Outcome.RunwaySamples == 0 { |
| 407 | continue |
| 408 | } |
| 409 | o := r.Trajectory.Outcome |
| 410 | measured++ |
| 411 | dryMax = max(dryMax, o.RunwayDryMax) |
| 412 | idleMax = max(idleMax, o.RunwayIdleMax) |
| 413 | if o.RunwayFirstSpentRound > 0 { |
| 414 | spent++ |
| 415 | spentRounds = append(spentRounds, int64(o.RunwayFirstSpentRound)) |
| 416 | } |
| 417 | } |
| 418 | if measured == 0 { |
| 419 | return "" |
| 420 | } |
| 421 | line := fmt.Sprintf(" · **runway shadow** would spend in %d/%d runs (%s)", spent, measured, pct(spent, measured)) |
| 422 | if spent > 0 { |
| 423 | line += fmt.Sprintf(" at median round %d", median(spentRounds)) |
| 424 | } |
| 425 | return fmt.Sprintf("%s · max dry/idle %d/%d", line, dryMax, idleMax) |
| 426 | } |
| 427 |