| 1 | package cli |
| 2 | |
| 3 | import ( |
| 4 | "testing" |
| 5 | |
| 6 | "reasonix/internal/evidence" |
| 7 | ) |
| 8 | |
| 9 | // The instrument must be able to say, from one run, whether delegation |
| 10 | // produced verified work or just more tokens. |
| 11 | func TestDelegationMetricsAggregateAcrossChildren(t *testing.T) { |
| 12 | s := &metricsSink{} |
| 13 | s.RecordDelegationAudit(evidence.DelegationAudit{ |
| 14 | Depth: 1, ToolCalls: 6, Mutations: 2, |
| 15 | MutationPaths: []string{"api/handler.go", "api/handler_test.go"}, |
| 16 | HasReport: true, |
| 17 | AdjudicatedStatus: string(evidence.CompletionComplete), |
| 18 | }) |
| 19 | s.RecordDelegationAudit(evidence.DelegationAudit{ |
| 20 | Depth: 2, ToolCalls: 4, Mutations: 1, |
| 21 | MutationPaths: []string{"api/handler.go"}, |
| 22 | ClaimViolations: 1, |
| 23 | HasReport: true, |
| 24 | AdjudicatedStatus: string(evidence.CompletionPartial), |
| 25 | Downgrades: 2, |
| 26 | }) |
| 27 | s.RecordDelegationAudit(evidence.DelegationAudit{Depth: 1, ToolCalls: 3}) |
| 28 | |
| 29 | m := s.m |
| 30 | if m.SubagentRuns != 3 || m.SubagentNestedRuns != 1 { |
| 31 | t.Fatalf("runs = %d nested = %d, want 3/1", m.SubagentRuns, m.SubagentNestedRuns) |
| 32 | } |
| 33 | if m.SubagentMutations != 3 { |
| 34 | t.Fatalf("mutations = %d, want 3", m.SubagentMutations) |
| 35 | } |
| 36 | if m.CompletionReports != 2 || m.CompletionsProsedOnly != 1 { |
| 37 | t.Fatalf("reports = %d prose-only = %d, want 2/1", m.CompletionReports, m.CompletionsProsedOnly) |
| 38 | } |
| 39 | // One child claimed criteria the host refused: that is the false-completion |
| 40 | // signal an orchestration benchmark exists to surface. |
| 41 | if m.FalseCompletions != 1 || m.CriterionDowngrades != 2 { |
| 42 | t.Fatalf("false completions = %d downgrades = %d, want 1/2", m.FalseCompletions, m.CriterionDowngrades) |
| 43 | } |
| 44 | if m.WriteScopeViolations != 1 { |
| 45 | t.Fatalf("write scope violations = %d, want 1", m.WriteScopeViolations) |
| 46 | } |
| 47 | // Two children mutated api/handler.go: duplicated work, counted once. |
| 48 | if m.DuplicateWorkPaths != 1 { |
| 49 | t.Fatalf("duplicate work paths = %d, want 1", m.DuplicateWorkPaths) |
| 50 | } |
| 51 | } |
| 52 | |
| 53 | // An independence rate is a ratio of summed paths, never a mean of per-child |
| 54 | // rates: a child that opened one file must not weigh the same as one that |
| 55 | // swept twenty. Summing here is what makes the published rate that ratio. |
| 56 | func TestDelegationMetricsSumEvidenceOriginForARatioOfTotals(t *testing.T) { |
| 57 | s := &metricsSink{} |
| 58 | s.RecordDelegationAudit(evidence.DelegationAudit{ |
| 59 | Depth: 1, ParentNamedFiles: 1, EvidencePaths: 20, DiscoveredPaths: 19, |
| 60 | }) |
| 61 | s.RecordDelegationAudit(evidence.DelegationAudit{ |
| 62 | Depth: 1, ParentNamedFiles: 2, EvidencePaths: 1, DiscoveredPaths: 0, |
| 63 | }) |
| 64 | |
| 65 | m := s.m |
| 66 | if m.ParentNamedFiles != 3 { |
| 67 | t.Fatalf("parent named files = %d, want 3", m.ParentNamedFiles) |
| 68 | } |
| 69 | // 19/21, not the 50% a mean of 95% and 0% would report. |
| 70 | if m.ChildDiscoveredPaths != 19 || m.ChildEvidencePaths != 21 { |
| 71 | t.Fatalf("discovered %d/%d, want 19/21", m.ChildDiscoveredPaths, m.ChildEvidencePaths) |
| 72 | } |
| 73 | } |
| 74 | |
| 75 | // A run with no delegation must leave every delegation counter at zero, so the |
| 76 | // single-agent arm is a clean baseline rather than noise. |
| 77 | func TestDelegationMetricsStayZeroForSingleAgentArm(t *testing.T) { |
| 78 | s := &metricsSink{} |
| 79 | if m := s.m; m.SubagentRuns != 0 || m.CompletionReports != 0 || m.DuplicateWorkPaths != 0 { |
| 80 | t.Fatalf("single-agent baseline is not zero: %+v", m) |
| 81 | } |
| 82 | } |
| 83 |