| 1 | package capability |
| 2 | |
| 3 | import "sync" |
| 4 | |
| 5 | // Audit is a non-persisted capability/routing counters sink, mirroring |
| 6 | // readiness audit collection for run --metrics and e2ebench. |
| 7 | type Audit struct { |
| 8 | mu sync.Mutex |
| 9 | |
| 10 | Routes int |
| 11 | RoutedCandidates int |
| 12 | RoutedRequire int |
| 13 | RoutedPrefer int |
| 14 | RoutedSuggest int |
| 15 | Declines int |
| 16 | SemanticRoutes int |
| 17 | SemanticFallbacks int |
| 18 | RequireMissing int |
| 19 | RequireRecovered int |
| 20 | PreferMissing int |
| 21 | PreferRecovered int |
| 22 | SkillInvocations int |
| 23 | SkillFailures int |
| 24 | SkillUnavailable int |
| 25 | MCPInspect int |
| 26 | MCPCall int |
| 27 | MCPCallFailures int |
| 28 | ReviewBlocks int |
| 29 | SecurityReviewBlocks int |
| 30 | RouterPromptTokens int |
| 31 | RouterCompletionTokens int |
| 32 | RouterCost float64 |
| 33 | RouterLatencyMs int64 |
| 34 | } |
| 35 | |
| 36 | // RecordDecision captures the route-to-invocation funnel before the model acts. |
| 37 | func (a *Audit) RecordDecision(decision RouteDecision) { |
| 38 | if a == nil { |
| 39 | return |
| 40 | } |
| 41 | a.mu.Lock() |
| 42 | defer a.mu.Unlock() |
| 43 | for _, candidate := range decision.Candidates { |
| 44 | a.RoutedCandidates++ |
| 45 | switch candidate.Policy { |
| 46 | case AutoUseRequire: |
| 47 | a.RoutedRequire++ |
| 48 | case AutoUsePrefer: |
| 49 | a.RoutedPrefer++ |
| 50 | case AutoUseSuggest: |
| 51 | a.RoutedSuggest++ |
| 52 | } |
| 53 | } |
| 54 | } |
| 55 | |
| 56 | // RecordDecline counts an explicit model decision not to use a preferred route. |
| 57 | func (a *Audit) RecordDecline() { |
| 58 | if a == nil { |
| 59 | return |
| 60 | } |
| 61 | a.mu.Lock() |
| 62 | a.Declines++ |
| 63 | a.mu.Unlock() |
| 64 | } |
| 65 | |
| 66 | // RecordRoute increments deterministic/hybrid route counts. |
| 67 | func (a *Audit) RecordRoute(semantic, fallback bool) { |
| 68 | if a == nil { |
| 69 | return |
| 70 | } |
| 71 | a.mu.Lock() |
| 72 | defer a.mu.Unlock() |
| 73 | a.Routes++ |
| 74 | if semantic { |
| 75 | a.SemanticRoutes++ |
| 76 | } |
| 77 | if fallback { |
| 78 | a.SemanticFallbacks++ |
| 79 | } |
| 80 | } |
| 81 | |
| 82 | // RecordGate records require/prefer missing and recovery. |
| 83 | func (a *Audit) RecordGate(requireMissing, preferMissing, recovered bool) { |
| 84 | if a == nil { |
| 85 | return |
| 86 | } |
| 87 | a.mu.Lock() |
| 88 | defer a.mu.Unlock() |
| 89 | if requireMissing { |
| 90 | a.RequireMissing++ |
| 91 | } |
| 92 | if preferMissing { |
| 93 | a.PreferMissing++ |
| 94 | } |
| 95 | if recovered { |
| 96 | if requireMissing { |
| 97 | a.RequireRecovered++ |
| 98 | } |
| 99 | if preferMissing { |
| 100 | a.PreferRecovered++ |
| 101 | } |
| 102 | } |
| 103 | } |
| 104 | |
| 105 | // RecordSkill records skill invocation outcomes. |
| 106 | func (a *Audit) RecordSkill(failed, unavailable bool) { |
| 107 | if a == nil { |
| 108 | return |
| 109 | } |
| 110 | a.mu.Lock() |
| 111 | defer a.mu.Unlock() |
| 112 | a.SkillInvocations++ |
| 113 | if failed { |
| 114 | a.SkillFailures++ |
| 115 | } |
| 116 | if unavailable { |
| 117 | a.SkillUnavailable++ |
| 118 | } |
| 119 | } |
| 120 | |
| 121 | // RecordMCPProxy records use_capability proxy activity. |
| 122 | func (a *Audit) RecordMCPProxy(inspect, call, failed bool) { |
| 123 | if a == nil { |
| 124 | return |
| 125 | } |
| 126 | a.mu.Lock() |
| 127 | defer a.mu.Unlock() |
| 128 | if inspect { |
| 129 | a.MCPInspect++ |
| 130 | } |
| 131 | if call { |
| 132 | a.MCPCall++ |
| 133 | } |
| 134 | if failed { |
| 135 | a.MCPCallFailures++ |
| 136 | } |
| 137 | } |
| 138 | |
| 139 | // RecordGateRecovery records that gate kinds which missed earlier in the turn |
| 140 | // later passed cleanly — the capability was actually invoked after the nudge. |
| 141 | // Kept separate from RecordGate so a recovery never double-counts as a miss. |
| 142 | func (a *Audit) RecordGateRecovery(require, prefer bool) { |
| 143 | if a == nil { |
| 144 | return |
| 145 | } |
| 146 | a.mu.Lock() |
| 147 | defer a.mu.Unlock() |
| 148 | if require { |
| 149 | a.RequireRecovered++ |
| 150 | } |
| 151 | if prefer { |
| 152 | a.PreferRecovered++ |
| 153 | } |
| 154 | } |
| 155 | |
| 156 | // RecordRouterUsage accumulates the semantic router's own model spend: |
| 157 | // prompt/completion tokens, priced cost, and wall-clock latency per call. |
| 158 | func (a *Audit) RecordRouterUsage(promptTokens, completionTokens int, cost float64, latencyMs int64) { |
| 159 | if a == nil { |
| 160 | return |
| 161 | } |
| 162 | a.mu.Lock() |
| 163 | defer a.mu.Unlock() |
| 164 | a.RouterPromptTokens += promptTokens |
| 165 | a.RouterCompletionTokens += completionTokens |
| 166 | a.RouterCost += cost |
| 167 | a.RouterLatencyMs += latencyMs |
| 168 | } |
| 169 | |
| 170 | // RecordReviewBlock records blocking structured review outcomes. |
| 171 | func (a *Audit) RecordReviewBlock(security bool) { |
| 172 | if a == nil { |
| 173 | return |
| 174 | } |
| 175 | a.mu.Lock() |
| 176 | defer a.mu.Unlock() |
| 177 | if security { |
| 178 | a.SecurityReviewBlocks++ |
| 179 | } else { |
| 180 | a.ReviewBlocks++ |
| 181 | } |
| 182 | } |
| 183 | |
| 184 | // Snapshot returns a copy of counters for metrics export. |
| 185 | func (a *Audit) Snapshot() Audit { |
| 186 | if a == nil { |
| 187 | return Audit{} |
| 188 | } |
| 189 | a.mu.Lock() |
| 190 | defer a.mu.Unlock() |
| 191 | return Audit{ |
| 192 | Routes: a.Routes, |
| 193 | RoutedCandidates: a.RoutedCandidates, |
| 194 | RoutedRequire: a.RoutedRequire, |
| 195 | RoutedPrefer: a.RoutedPrefer, |
| 196 | RoutedSuggest: a.RoutedSuggest, |
| 197 | Declines: a.Declines, |
| 198 | SemanticRoutes: a.SemanticRoutes, |
| 199 | SemanticFallbacks: a.SemanticFallbacks, |
| 200 | RequireMissing: a.RequireMissing, |
| 201 | RequireRecovered: a.RequireRecovered, |
| 202 | PreferMissing: a.PreferMissing, |
| 203 | PreferRecovered: a.PreferRecovered, |
| 204 | SkillInvocations: a.SkillInvocations, |
| 205 | SkillFailures: a.SkillFailures, |
| 206 | SkillUnavailable: a.SkillUnavailable, |
| 207 | MCPInspect: a.MCPInspect, |
| 208 | MCPCall: a.MCPCall, |
| 209 | MCPCallFailures: a.MCPCallFailures, |
| 210 | ReviewBlocks: a.ReviewBlocks, |
| 211 | SecurityReviewBlocks: a.SecurityReviewBlocks, |
| 212 | RouterPromptTokens: a.RouterPromptTokens, |
| 213 | RouterCompletionTokens: a.RouterCompletionTokens, |
| 214 | RouterCost: a.RouterCost, |
| 215 | RouterLatencyMs: a.RouterLatencyMs, |
| 216 | } |
| 217 | } |
| 218 |