| 1 | package cli |
| 2 | |
| 3 | import ( |
| 4 | "encoding/json" |
| 5 | "os" |
| 6 | "path/filepath" |
| 7 | "testing" |
| 8 | |
| 9 | "reasonix/internal/event" |
| 10 | "reasonix/internal/evidence" |
| 11 | "reasonix/internal/provider" |
| 12 | ) |
| 13 | |
| 14 | func TestMetricsSinkForwardsEachEventOnce(t *testing.T) { |
| 15 | forwarded := 0 |
| 16 | s := &metricsSink{inner: event.FuncSink(func(event.Event) { forwarded++ })} |
| 17 | s.Emit(event.Event{Kind: event.Text, Text: "x"}) |
| 18 | if forwarded != 1 { |
| 19 | t.Fatalf("inner sink saw %d emissions for one event, want 1", forwarded) |
| 20 | } |
| 21 | } |
| 22 | |
| 23 | type metricsCapabilityProbe struct { |
| 24 | event.Sink |
| 25 | readiness, delegation, workspace, runBudget int |
| 26 | } |
| 27 | |
| 28 | func (p *metricsCapabilityProbe) RecordReadinessAudit(evidence.ReadinessAudit) { p.readiness++ } |
| 29 | func (p *metricsCapabilityProbe) RecordDelegationAudit(evidence.DelegationAudit) { |
| 30 | p.delegation++ |
| 31 | } |
| 32 | func (p *metricsCapabilityProbe) RecordWorkspaceMutation(event.WorkspaceMutation) { |
| 33 | p.workspace++ |
| 34 | } |
| 35 | func (p *metricsCapabilityProbe) RecordRunBudget(event.RunBudgetSample) { p.runBudget++ } |
| 36 | |
| 37 | func TestMetricsSinkForwardsObservedCapabilities(t *testing.T) { |
| 38 | inner := &metricsCapabilityProbe{Sink: event.Discard} |
| 39 | s := &metricsSink{inner: inner} |
| 40 | |
| 41 | s.RecordReadinessAudit(evidence.ReadinessAudit{}) |
| 42 | s.RecordDelegationAudit(evidence.DelegationAudit{}) |
| 43 | event.RecordWorkspaceMutation(s, event.WorkspaceMutation{ToolName: "write_file"}) |
| 44 | event.RecordRunBudget(s, event.RunBudgetSample{Currency: "USD"}) |
| 45 | |
| 46 | if inner.readiness != 1 || inner.delegation != 1 || inner.workspace != 1 || inner.runBudget != 1 { |
| 47 | t.Fatalf("observed capabilities not forwarded: %+v", inner) |
| 48 | } |
| 49 | } |
| 50 | |
| 51 | func TestMetricsSinkUsesProviderCacheWriteCost(t *testing.T) { |
| 52 | s := &metricsSink{inner: event.Discard} |
| 53 | s.Emit(event.Event{ |
| 54 | Kind: event.Usage, |
| 55 | Usage: &provider.Usage{ |
| 56 | CacheMissTokens: 500_000, |
| 57 | CacheWriteTokens: 100_000, |
| 58 | CacheWriteBilledTokens: 200_000, |
| 59 | }, |
| 60 | Pricing: &provider.Pricing{Input: 2}, |
| 61 | }) |
| 62 | if got := s.Snapshot().Cost; got != 1.2 { |
| 63 | t.Fatalf("metrics cost = %f, want 1.2", got) |
| 64 | } |
| 65 | } |
| 66 | |
| 67 | func TestMetricsSinkAccumulatesReadinessAudit(t *testing.T) { |
| 68 | s := &metricsSink{inner: event.Discard} |
| 69 | |
| 70 | s.RecordReadinessAudit(evidence.ReadinessAudit{ |
| 71 | Result: evidence.ReadinessBlocked, |
| 72 | MissingProjectChecks: 2, |
| 73 | IncompleteTodos: 3, |
| 74 | CommandMismatchMissing: 2, |
| 75 | MissingAcceptanceCriteria: 1, |
| 76 | MissingVerification: 1, |
| 77 | MissingReview: 1, |
| 78 | MissingSignoff: 1, |
| 79 | MissingActionEvidence: 1, |
| 80 | MissingMutation: 1, |
| 81 | }) |
| 82 | s.RecordReadinessAudit(evidence.ReadinessAudit{ |
| 83 | Result: evidence.ReadinessAllowed, |
| 84 | Recovered: true, |
| 85 | }) |
| 86 | s.RecordReadinessAudit(evidence.ReadinessAudit{ |
| 87 | Result: evidence.ReadinessErrored, |
| 88 | }) |
| 89 | |
| 90 | if s.m.ReadinessChecks != 3 { |
| 91 | t.Fatalf("readiness checks = %d, want 3", s.m.ReadinessChecks) |
| 92 | } |
| 93 | if s.m.ReadinessAllowed != 1 { |
| 94 | t.Fatalf("readiness allowed = %d, want 1", s.m.ReadinessAllowed) |
| 95 | } |
| 96 | if s.m.ReadinessBlocks != 1 { |
| 97 | t.Fatalf("readiness blocks = %d, want 1", s.m.ReadinessBlocks) |
| 98 | } |
| 99 | if s.m.ReadinessRecoveries != 1 { |
| 100 | t.Fatalf("readiness recoveries = %d, want 1", s.m.ReadinessRecoveries) |
| 101 | } |
| 102 | if s.m.ReadinessErrors != 1 { |
| 103 | t.Fatalf("readiness errors = %d, want 1", s.m.ReadinessErrors) |
| 104 | } |
| 105 | if s.m.ReadinessMissingProjectChecks != 2 { |
| 106 | t.Fatalf("missing project checks = %d, want 2", s.m.ReadinessMissingProjectChecks) |
| 107 | } |
| 108 | if s.m.ReadinessIncompleteTodos != 3 { |
| 109 | t.Fatalf("incomplete todos = %d, want 3", s.m.ReadinessIncompleteTodos) |
| 110 | } |
| 111 | if s.m.ReadinessCommandMismatches != 2 { |
| 112 | t.Fatalf("command mismatches = %d, want 2", s.m.ReadinessCommandMismatches) |
| 113 | } |
| 114 | if s.m.ReadinessMissingAcceptance != 1 || s.m.ReadinessMissingVerification != 1 || s.m.ReadinessMissingReview != 1 || s.m.ReadinessMissingSignoff != 1 { |
| 115 | t.Fatalf("delivery readiness misses = acceptance %d verification %d review %d signoff %d, want 1/1/1/1", |
| 116 | s.m.ReadinessMissingAcceptance, s.m.ReadinessMissingVerification, s.m.ReadinessMissingReview, s.m.ReadinessMissingSignoff) |
| 117 | } |
| 118 | if s.m.ReadinessMissingActionEvidence != 1 || s.m.ReadinessMissingMutation != 1 { |
| 119 | t.Fatalf("delivery work misses = action evidence %d mutation %d, want 1/1", s.m.ReadinessMissingActionEvidence, s.m.ReadinessMissingMutation) |
| 120 | } |
| 121 | } |
| 122 | |
| 123 | func TestMetricsSinkAccumulatesSilentReasoningRecovery(t *testing.T) { |
| 124 | s := &metricsSink{inner: event.Discard} |
| 125 | s.RecordProtocolRecovery(event.ProtocolRecoveryAudit{Kind: event.ProtocolRecoveryMissingReasoningDetected}) |
| 126 | s.RecordProtocolRecovery(event.ProtocolRecoveryAudit{Kind: event.ProtocolRecoveryMissingReasoningRetryAttempted}) |
| 127 | s.RecordProtocolRecovery(event.ProtocolRecoveryAudit{Kind: event.ProtocolRecoveryMissingReasoningRetryRecovered}) |
| 128 | s.RecordProtocolRecovery(event.ProtocolRecoveryAudit{Kind: event.ProtocolRecoveryMissingReasoningRetryReplaced}) |
| 129 | s.RecordProtocolRecovery(event.ProtocolRecoveryAudit{Kind: event.ProtocolRecoveryMissingReasoningRetrySuppressed}) |
| 130 | s.RecordProtocolRecovery(event.ProtocolRecoveryAudit{Kind: event.ProtocolRecoveryMissingReasoningFallback}) |
| 131 | |
| 132 | if s.m.MissingReasoningDetected != 1 || s.m.MissingReasoningRetries != 1 || s.m.MissingReasoningRecovered != 1 || s.m.MissingReasoningReplaced != 1 || s.m.MissingReasoningSuppressed != 1 || s.m.MissingReasoningFallbacks != 1 { |
| 133 | t.Fatalf("reasoning recovery metrics = %+v", s.m) |
| 134 | } |
| 135 | if s.m.Steps != 1 { |
| 136 | t.Fatalf("retry model-call step = %d, want 1", s.m.Steps) |
| 137 | } |
| 138 | } |
| 139 | |
| 140 | func TestMetricsSinkAccountsToolCallsAndRetries(t *testing.T) { |
| 141 | s := &metricsSink{inner: event.Discard} |
| 142 | |
| 143 | s.Emit(event.Event{Kind: event.ToolDispatch, Tool: event.Tool{Name: "read_file"}}) |
| 144 | s.Emit(event.Event{Kind: event.ToolResult, Tool: event.Tool{Name: "read_file", DurationMs: 12}}) |
| 145 | s.Emit(event.Event{Kind: event.ToolResult, Tool: event.Tool{Name: "read_file", DurationMs: 8}}) |
| 146 | s.Emit(event.Event{Kind: event.ToolResult, Tool: event.Tool{Name: "bash", Err: "exit status 1", DurationMs: 30}}) |
| 147 | s.Emit(event.Event{Kind: event.ToolResult, Tool: event.Tool{Name: "grep", ParentID: "task-1", DurationMs: 5}}) |
| 148 | s.Emit(event.Event{Kind: event.Retrying, RetryAttempt: 1, RetryMax: 3}) |
| 149 | |
| 150 | if s.m.ToolCalls != 4 { |
| 151 | t.Fatalf("tool calls = %d, want 4 (dispatch must not count)", s.m.ToolCalls) |
| 152 | } |
| 153 | if s.m.ToolFailures != 1 { |
| 154 | t.Fatalf("tool failures = %d, want 1", s.m.ToolFailures) |
| 155 | } |
| 156 | if s.m.ToolDurationMs != 55 { |
| 157 | t.Fatalf("tool duration = %dms, want 55", s.m.ToolDurationMs) |
| 158 | } |
| 159 | if s.m.SubagentToolCalls != 1 { |
| 160 | t.Fatalf("subagent tool calls = %d, want 1", s.m.SubagentToolCalls) |
| 161 | } |
| 162 | if s.m.Retries != 1 { |
| 163 | t.Fatalf("retries = %d, want 1", s.m.Retries) |
| 164 | } |
| 165 | if s.m.ToolCallsByName["read_file"] != 2 || s.m.ToolCallsByName["bash"] != 1 { |
| 166 | t.Fatalf("calls by name = %v", s.m.ToolCallsByName) |
| 167 | } |
| 168 | if s.m.ToolFailuresByName["bash"] != 1 { |
| 169 | t.Fatalf("failures by name = %v, want bash 1", s.m.ToolFailuresByName) |
| 170 | } |
| 171 | if _, ok := s.m.ToolFailuresByName["read_file"]; ok { |
| 172 | t.Fatalf("a successful tool must not appear in failures: %v", s.m.ToolFailuresByName) |
| 173 | } |
| 174 | } |
| 175 | |
| 176 | func TestWriteMetricsIncludesReadinessFields(t *testing.T) { |
| 177 | path := filepath.Join(t.TempDir(), "metrics.json") |
| 178 | if err := writeMetrics(path, RunMetrics{ |
| 179 | PromptTokens: 10, |
| 180 | CompletionTokens: 3, |
| 181 | CacheHitTokens: 7, |
| 182 | CacheMissTokens: 3, |
| 183 | Steps: 2, |
| 184 | ReadinessChecks: 1, |
| 185 | ReadinessAllowed: 1, |
| 186 | ReadinessBlocks: 0, |
| 187 | ReadinessRecoveries: 1, |
| 188 | ReadinessErrors: 0, |
| 189 | ReadinessMissingProjectChecks: 0, |
| 190 | ReadinessIncompleteTodos: 0, |
| 191 | ReadinessCommandMismatches: 0, |
| 192 | ReadinessMissingAcceptance: 0, |
| 193 | ReadinessMissingVerification: 0, |
| 194 | ReadinessMissingReview: 0, |
| 195 | ReadinessMissingSignoff: 0, |
| 196 | ReadinessMissingActionEvidence: 0, |
| 197 | ReadinessMissingMutation: 0, |
| 198 | }); err != nil { |
| 199 | t.Fatalf("writeMetrics: %v", err) |
| 200 | } |
| 201 | |
| 202 | b, err := os.ReadFile(path) |
| 203 | if err != nil { |
| 204 | t.Fatalf("ReadFile: %v", err) |
| 205 | } |
| 206 | var got map[string]any |
| 207 | if err := json.Unmarshal(b, &got); err != nil { |
| 208 | t.Fatalf("Unmarshal: %v", err) |
| 209 | } |
| 210 | for _, key := range []string{ |
| 211 | "readiness_checks", |
| 212 | "readiness_allowed", |
| 213 | "readiness_blocks", |
| 214 | "readiness_recoveries", |
| 215 | "readiness_errors", |
| 216 | "readiness_missing_project_checks", |
| 217 | "readiness_incomplete_todos", |
| 218 | "readiness_command_mismatches", |
| 219 | "readiness_missing_acceptance_criteria", |
| 220 | "readiness_missing_verification", |
| 221 | "readiness_missing_review", |
| 222 | "readiness_missing_signoff", |
| 223 | "readiness_missing_action_evidence", |
| 224 | "readiness_missing_mutation", |
| 225 | } { |
| 226 | if _, ok := got[key]; !ok { |
| 227 | t.Fatalf("metrics JSON missing %q: %s", key, string(b)) |
| 228 | } |
| 229 | } |
| 230 | } |
| 231 |