| 1 | package main |
| 2 | |
| 3 | import "fmt" |
| 4 | |
| 5 | // roundDigest is one model round's cost/outcome line: the gap that preceded |
| 6 | // the batch, the executor thinking that gap bought, and what the round did. |
| 7 | type roundDigest struct { |
| 8 | Index int `json:"i"` |
| 9 | Outcome string `json:"outcome"` |
| 10 | GapMs int64 `json:"gap_ms"` |
| 11 | ToolMs int64 `json:"tool_ms,omitempty"` |
| 12 | ReasoningTokens int64 `json:"reasoning_tokens,omitempty"` |
| 13 | CompletionTokens int64 `json:"completion_tokens,omitempty"` |
| 14 | PromptTokens int64 `json:"prompt_tokens,omitempty"` |
| 15 | Actions []string `json:"actions,omitempty"` |
| 16 | } |
| 17 | |
| 18 | // slowRoundGapMs is the census threshold: a model gap this long is a large |
| 19 | // cognition purchase worth itemizing (p90-tail rounds sit well above it). |
| 20 | const slowRoundGapMs = 8000 |
| 21 | |
| 22 | // recordRound books one classified round into both the outcome tallies and |
| 23 | // the per-round cognition digest. A nil batch is a finalization round. |
| 24 | func (t *trajScan) recordRound(outcome string, gap gapInfo, b *toolBatch) { |
| 25 | t.recordOutcome(outcome, gap.ms) |
| 26 | d := roundDigest{ |
| 27 | Index: len(t.s.Rounds) + 1, Outcome: outcome, GapMs: gap.ms, |
| 28 | ReasoningTokens: gap.reasonTok, CompletionTokens: gap.complTok, |
| 29 | PromptTokens: gap.promptTok, |
| 30 | } |
| 31 | if b != nil { |
| 32 | d.ToolMs = b.serialMs |
| 33 | d.Actions = append([]string(nil), b.names...) |
| 34 | } |
| 35 | t.s.Rounds = append(t.s.Rounds, d) |
| 36 | // Recovery- and compaction-tainted gaps are provider/host time, not a |
| 37 | // cognition purchase; letting them into the census would misattribute a |
| 38 | // 30s retry as a slow thinking round. |
| 39 | if gap.ms >= slowRoundGapMs && !gap.tainted && !gap.compaction { |
| 40 | t.s.SlowRounds++ |
| 41 | t.s.SlowRoundGapMs += gap.ms |
| 42 | t.s.SlowRoundReasoningTokens += gap.reasonTok |
| 43 | } |
| 44 | } |
| 45 | |
| 46 | // renderDelegationAdmission aggregates the shadow admission verdicts: how many |
| 47 | // expensive delegation calls a local-fix boundary would have refused, and the |
| 48 | // subagent time those refusals would have reclaimed. |
| 49 | func renderDelegationAdmission(results []result) string { |
| 50 | calls, denies := 0, 0 |
| 51 | var deniedMs int64 |
| 52 | for _, r := range results { |
| 53 | if r.Trajectory == nil { |
| 54 | continue |
| 55 | } |
| 56 | calls += r.Trajectory.DelegationCalls |
| 57 | denies += r.Trajectory.DelegationDenies |
| 58 | deniedMs += r.Trajectory.DeniedDelegationMs |
| 59 | } |
| 60 | if calls == 0 { |
| 61 | return "" |
| 62 | } |
| 63 | line := fmt.Sprintf("**Delegation admission** (shadow): **%d** gated calls · **would deny** %d (%s)", |
| 64 | calls, denies, pct(denies, calls)) |
| 65 | if deniedMs > 0 { |
| 66 | line += fmt.Sprintf(" · **subagent time behind denied calls** %s", dur(deniedMs)) |
| 67 | } |
| 68 | return line + "\n\n" |
| 69 | } |
| 70 | |
| 71 | // renderCognition prices what the model's thinking bought: totals, the output |
| 72 | // rate (uniform rates indict token volume, not serving), the slow-round |
| 73 | // census, and delegation cost. Empty when no run carried usage-joined rounds. |
| 74 | func renderCognition(results []result) string { |
| 75 | var reason, compl, slowGapMs, gapMs, delegToolMs int64 |
| 76 | slow, delegRounds, runs, solved := 0, 0, 0, 0 |
| 77 | var slowReason int64 |
| 78 | var rates []int64 |
| 79 | for _, r := range results { |
| 80 | if r.Passed { |
| 81 | solved++ |
| 82 | } |
| 83 | t := r.Trajectory |
| 84 | if t == nil || len(t.Rounds) == 0 { |
| 85 | continue |
| 86 | } |
| 87 | runs++ |
| 88 | reason += t.ReasoningTokensTotal |
| 89 | compl += t.CompletionTokensTotal |
| 90 | slow += t.SlowRounds |
| 91 | slowGapMs += t.SlowRoundGapMs |
| 92 | slowReason += t.SlowRoundReasoningTokens |
| 93 | gapMs += t.ModelGapTotalMs |
| 94 | for _, d := range t.Rounds { |
| 95 | if d.Outcome == "delegation" { |
| 96 | delegRounds++ |
| 97 | delegToolMs += d.ToolMs |
| 98 | } |
| 99 | if d.GapMs >= 1000 && d.ReasoningTokens+d.CompletionTokens > 0 { |
| 100 | rates = append(rates, (d.ReasoningTokens+d.CompletionTokens)*1000/d.GapMs) |
| 101 | } |
| 102 | } |
| 103 | } |
| 104 | if runs == 0 { |
| 105 | return "" |
| 106 | } |
| 107 | line := fmt.Sprintf("**Cognition** (%d recorded runs): **reasoning** %s tok · **completion** %s tok", |
| 108 | runs, comma(int(reason)), comma(int(compl))) |
| 109 | if solved > 0 { |
| 110 | line += fmt.Sprintf(" (**%s reasoning/solved**)", comma(int(reason/int64(solved)))) |
| 111 | } |
| 112 | if len(rates) > 0 { |
| 113 | line += fmt.Sprintf(" · **output rate** p50 %d · p90 %d tok/s", pctile(rates, 50), pctile(rates, 90)) |
| 114 | } |
| 115 | if slow > 0 { |
| 116 | line += fmt.Sprintf(" · **slow rounds** (≥%ds) %d = %s of model time, %s reasoning tok", |
| 117 | slowRoundGapMs/1000, slow, pct(int(slowGapMs), int(gapMs)), comma(int(slowReason))) |
| 118 | } |
| 119 | if delegRounds > 0 { |
| 120 | line += fmt.Sprintf(" · **delegation** %d rounds (%s in subagents)", delegRounds, dur(delegToolMs)) |
| 121 | } |
| 122 | return line + "\n\n" |
| 123 | } |
| 124 |