| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "fmt" |
| 5 | "strings" |
| 6 | ) |
| 7 | |
| 8 | // mechanismRow aggregates one extra-round mechanism across a suite: how often |
| 9 | // it fired, what its rounds cost, and how runs where it fired graded versus |
| 10 | // runs where it stayed quiet. Correlation, not causation — the causal rescue |
| 11 | // rate needs an ablation arm A/B (-ablate + -mode compare). |
| 12 | type mechanismRow struct { |
| 13 | fires int |
| 14 | ms int64 |
| 15 | msKnown bool |
| 16 | firedRuns, firedSolved int |
| 17 | quietRuns, quietSolved int |
| 18 | } |
| 19 | |
| 20 | // mechanismOrder fixes the ledger's row order: correctness nudges first, then |
| 21 | // provider recovery, then structural overhead. |
| 22 | var mechanismOrder = []string{ |
| 23 | "handoff_nudge", "empty_final_retry", "no_progress_signal", |
| 24 | "stream_retry", "header_retry", "reasoning_replay", |
| 25 | "planner", "compaction", "bookkeeping", "duplicate_work", |
| 26 | "subagent", "capability_router", "goal_evaluator", "tool_source_connect", "prefix_reset", |
| 27 | } |
| 28 | |
| 29 | // mechanismFacts extracts one run's (fires, attributed ms, ms known) per |
| 30 | // mechanism from its digest and metrics. |
| 31 | func mechanismFacts(r result) map[string]mechanismRow { |
| 32 | t := r.Trajectory |
| 33 | if t == nil { |
| 34 | return nil |
| 35 | } |
| 36 | byKind := func(kind string) int64 { return t.RecoveryGapMsByKind[kind] } |
| 37 | facts := map[string]mechanismRow{ |
| 38 | "handoff_nudge": {fires: t.HandoffNudges, ms: t.RoundOutcomeMs["handoff_retry"], msKnown: true}, |
| 39 | "empty_final_retry": {fires: t.EmptyFinalRetries, ms: byKind("empty_final_retry"), msKnown: true}, |
| 40 | "no_progress_signal": {fires: t.NoProgressSignals, msKnown: false}, |
| 41 | "stream_retry": {fires: t.StreamRetries, ms: byKind("stream_retry"), msKnown: true}, |
| 42 | "header_retry": {fires: t.HeaderRetries, ms: byKind("header_retry"), msKnown: true}, |
| 43 | "reasoning_replay": {fires: t.ReasoningReplays, ms: byKind("reasoning_replay"), msKnown: true}, |
| 44 | "planner": {fires: t.PlannerRequests, ms: t.RoundOutcomeMs["planning"], msKnown: true}, |
| 45 | "compaction": {fires: t.Compactions, ms: t.RoundOutcomeMs["compaction"], msKnown: true}, |
| 46 | "bookkeeping": {fires: t.RoundOutcomes["bookkeeping"], ms: t.RoundOutcomeMs["bookkeeping"], msKnown: true}, |
| 47 | "duplicate_work": {fires: t.RoundOutcomes["duplicate_work"], ms: t.RoundOutcomeMs["duplicate_work"], msKnown: true}, |
| 48 | "subagent": {fires: t.SubagentRequests, msKnown: false}, |
| 49 | "capability_router": {fires: r.CapabilityRoutes, ms: r.CapabilityRouterLatencyMs, msKnown: true}, |
| 50 | "tool_source_connect": {fires: t.ConnectCalls, msKnown: false}, |
| 51 | "prefix_reset": {fires: t.PrefixResets, msKnown: false}, |
| 52 | "goal_evaluator": {fires: t.RequestsBySource["goal-evaluator"], msKnown: false}, |
| 53 | } |
| 54 | return facts |
| 55 | } |
| 56 | |
| 57 | // renderToolSurface is the schema-tax line: what every request re-pays for |
| 58 | // the visible tool surface, and the churn (connects, prefix resets) the |
| 59 | // adaptive runtime trades that tax against. Fresh-session benchmarks re-pay the |
| 60 | // miss on every task, so the surface size prices differently than in a |
| 61 | // long-lived session. |
| 62 | func renderToolSurface(results []result) string { |
| 63 | var schemaMax, schemaTotal, promptTotal int64 |
| 64 | connects, resets, runs := 0, 0, 0 |
| 65 | for _, r := range results { |
| 66 | t := r.Trajectory |
| 67 | if t == nil || t.SchemaTokensTotal == 0 { |
| 68 | continue |
| 69 | } |
| 70 | runs++ |
| 71 | schemaMax = max(schemaMax, t.SchemaTokensMax) |
| 72 | schemaTotal += t.SchemaTokensTotal |
| 73 | promptTotal += t.PromptTokensSeen |
| 74 | connects += t.ConnectCalls |
| 75 | resets += t.PrefixResets |
| 76 | } |
| 77 | if runs == 0 { |
| 78 | return "" |
| 79 | } |
| 80 | line := fmt.Sprintf("**Tool surface**: **schema footprint** %s tok/request (max) · **Σ schema tax** %s tok", comma(int(schemaMax)), comma(int(schemaTotal))) |
| 81 | if promptTotal > 0 { |
| 82 | line += fmt.Sprintf(" (%s of prompt)", pct(int(schemaTotal), int(promptTotal))) |
| 83 | } |
| 84 | line += fmt.Sprintf(" · **connect_tool_source** ×%d · **prefix resets** %d\n\n", connects, resets) |
| 85 | return line |
| 86 | } |
| 87 | |
| 88 | // renderMechanismLedger is the measure-before-cutting table: per mechanism, |
| 89 | // incidence, attributed model time, and solved rates fired-vs-quiet. All-quiet |
| 90 | // suites render a single line so absence is a stated result, not a blank. |
| 91 | func renderMechanismLedger(results []result) string { |
| 92 | rows := map[string]mechanismRow{} |
| 93 | recorded := 0 |
| 94 | for _, r := range results { |
| 95 | facts := mechanismFacts(r) |
| 96 | if facts == nil { |
| 97 | continue |
| 98 | } |
| 99 | recorded++ |
| 100 | for name, f := range facts { |
| 101 | row := rows[name] |
| 102 | row.fires += f.fires |
| 103 | row.ms += f.ms |
| 104 | row.msKnown = row.msKnown || f.msKnown |
| 105 | if f.fires > 0 { |
| 106 | row.firedRuns++ |
| 107 | if r.Passed { |
| 108 | row.firedSolved++ |
| 109 | } |
| 110 | } else { |
| 111 | row.quietRuns++ |
| 112 | if r.Passed { |
| 113 | row.quietSolved++ |
| 114 | } |
| 115 | } |
| 116 | rows[name] = row |
| 117 | } |
| 118 | } |
| 119 | if recorded == 0 { |
| 120 | return "" |
| 121 | } |
| 122 | var b strings.Builder |
| 123 | b.WriteString("**Mechanism ledger** (incidence → cost → outcome; correlation only — causal rescue rates need an `-ablate` A/B):\n\n") |
| 124 | fired := 0 |
| 125 | b.WriteString("| Mechanism | Fires | Runs fired | Time | Solved (fired) | Solved (quiet) |\n") |
| 126 | b.WriteString("|---|---:|---:|---:|---:|---:|\n") |
| 127 | for _, name := range mechanismOrder { |
| 128 | row := rows[name] |
| 129 | if row.fires == 0 { |
| 130 | continue |
| 131 | } |
| 132 | fired++ |
| 133 | ms := "—" |
| 134 | if row.msKnown { |
| 135 | ms = dur(row.ms) |
| 136 | } |
| 137 | fmt.Fprintf(&b, "| %s | %d | %d/%d | %s | %s | %s |\n", |
| 138 | name, row.fires, row.firedRuns, recorded, ms, |
| 139 | pct(row.firedSolved, row.firedRuns), pct(row.quietSolved, row.quietRuns)) |
| 140 | } |
| 141 | if fired == 0 { |
| 142 | return fmt.Sprintf("**Mechanism ledger**: all quiet — no extra-round machinery fired across %d recorded runs.\n\n", recorded) |
| 143 | } |
| 144 | return b.String() + "\n" |
| 145 | } |
| 146 | |
| 147 | // renderContractShadow prices the shadow contract against the hidden grader: |
| 148 | // agreement is the number the adoption decision is made on. Absent audits |
| 149 | // (agent without shadow wiring) render nothing. |
| 150 | func renderContractShadow(results []result) string { |
| 151 | agree, disagree := 0, 0 |
| 152 | verdicts := map[string]int{} |
| 153 | for _, r := range results { |
| 154 | t := r.Trajectory |
| 155 | if t == nil || t.ShadowVerdict == "" { |
| 156 | continue |
| 157 | } |
| 158 | verdicts[t.ShadowVerdict]++ |
| 159 | if t.ShadowComplete == r.Passed { |
| 160 | agree++ |
| 161 | } else { |
| 162 | disagree++ |
| 163 | } |
| 164 | } |
| 165 | if agree+disagree == 0 { |
| 166 | return "" |
| 167 | } |
| 168 | parts := make([]string, 0, len(verdicts)) |
| 169 | for _, v := range []string{"complete", "continue", "blocked", "uncertain"} { |
| 170 | if verdicts[v] > 0 { |
| 171 | parts = append(parts, fmt.Sprintf("%s ×%d", v, verdicts[v])) |
| 172 | } |
| 173 | } |
| 174 | return fmt.Sprintf("**Contract shadow**: verdicts %s · **agreement with grader** %s (%d/%d)\n\n", |
| 175 | strings.Join(parts, " · "), pct(agree, agree+disagree), agree, agree+disagree) |
| 176 | } |
| 177 | |
| 178 | // renderCompletionReport prices the host-authored receipt against the hidden |
| 179 | // grader. Overclaim — "done" on a task the grader failed — is the number this |
| 180 | // whole mechanism exists to drive down; caught is its counterpart, the share |
| 181 | // of failed runs whose receipt already named a gap. |
| 182 | func renderCompletionReport(results []result) string { |
| 183 | verdicts := map[string]int{} |
| 184 | kinds := map[string]int{} |
| 185 | recorded, done, overclaim, failed, caught := 0, 0, 0, 0, 0 |
| 186 | claimed, unbacked := 0, 0 |
| 187 | for _, r := range results { |
| 188 | t := r.Trajectory |
| 189 | if t == nil || t.CompletionVerdict == "" { |
| 190 | continue |
| 191 | } |
| 192 | recorded++ |
| 193 | claimed += t.ClaimsVerified |
| 194 | unbacked += t.ClaimsUnbacked |
| 195 | verdicts[t.CompletionVerdict]++ |
| 196 | for _, kind := range t.CompletionGapKinds { |
| 197 | kinds[kind]++ |
| 198 | } |
| 199 | if t.CompletionVerdict == "done" { |
| 200 | done++ |
| 201 | if !r.Passed { |
| 202 | overclaim++ |
| 203 | } |
| 204 | } |
| 205 | if !r.Passed { |
| 206 | failed++ |
| 207 | if t.CompletionGaps > 0 { |
| 208 | caught++ |
| 209 | } |
| 210 | } |
| 211 | } |
| 212 | if recorded == 0 { |
| 213 | return "" |
| 214 | } |
| 215 | parts := make([]string, 0, len(verdicts)) |
| 216 | for _, v := range []string{"done", "partial", "incomplete", "unknown"} { |
| 217 | if verdicts[v] > 0 { |
| 218 | parts = append(parts, fmt.Sprintf("%s ×%d", v, verdicts[v])) |
| 219 | } |
| 220 | } |
| 221 | line := fmt.Sprintf("**Completion report**: verdicts %s · **overclaim** %s (%d/%d done runs the grader failed)", |
| 222 | strings.Join(parts, " · "), pct(overclaim, done), overclaim, done) |
| 223 | if failed > 0 { |
| 224 | line += fmt.Sprintf(" · **caught** %s (%d/%d failed runs declared a gap)", pct(caught, failed), caught, failed) |
| 225 | } |
| 226 | if claimed > 0 { |
| 227 | line += fmt.Sprintf(" · **unbacked claims** %s (%d/%d asserted verifications the ledger denied)", pct(unbacked, claimed), unbacked, claimed) |
| 228 | } |
| 229 | if census := gapCensus(kinds); census != "" { |
| 230 | line += " · gaps " + census |
| 231 | } |
| 232 | return line + "\n\n" |
| 233 | } |
| 234 | |
| 235 | func gapCensus(kinds map[string]int) string { |
| 236 | var parts []string |
| 237 | for _, kind := range []string{"unproven_criterion", "missing_check", "failed_verification", "stale_verification", "unverified_change", "unreviewed_change"} { |
| 238 | if kinds[kind] > 0 { |
| 239 | parts = append(parts, fmt.Sprintf("%s ×%d", kind, kinds[kind])) |
| 240 | } |
| 241 | } |
| 242 | return strings.Join(parts, " · ") |
| 243 | } |
| 244 |