| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "strings" |
| 5 | "testing" |
| 6 | ) |
| 7 | |
| 8 | func meteredRun(id string, meterPrompt, meterCompletion, selfPrompt, selfCompletion int) result { |
| 9 | r := result{task: task{ID: id}, Attempt: 1} |
| 10 | r.Meter = &meterUsage{ |
| 11 | Requests: 4, PromptTokens: meterPrompt, CompletionTokens: meterCompletion, |
| 12 | CacheHitTokens: meterPrompt / 2, CacheMissTokens: meterPrompt - meterPrompt/2, |
| 13 | } |
| 14 | r.PromptTokens = selfPrompt |
| 15 | r.CompletionTokens = selfCompletion |
| 16 | return r |
| 17 | } |
| 18 | |
| 19 | func TestMeterAccountingReportsAgreement(t *testing.T) { |
| 20 | got := renderMeterAccounting([]result{ |
| 21 | meteredRun("a", 1000, 200, 1000, 200), |
| 22 | meteredRun("b", 500, 100, 500, 100), |
| 23 | }) |
| 24 | for _, want := range []string{ |
| 25 | "**Metered at the boundary** (2 runs)", |
| 26 | "tokens 1,800", |
| 27 | "cache hit 50%", |
| 28 | "**self-report divergence** +0.0%", |
| 29 | } { |
| 30 | if !strings.Contains(got, want) { |
| 31 | t.Fatalf("meter line missing %q:\n%s", want, got) |
| 32 | } |
| 33 | } |
| 34 | } |
| 35 | |
| 36 | // The divergence is the publishability gate: if the proxy and the harness |
| 37 | // disagree about the same traffic, one of them is wrong. |
| 38 | func TestMeterAccountingSurfacesDisagreement(t *testing.T) { |
| 39 | got := renderMeterAccounting([]result{meteredRun("a", 1000, 0, 800, 0)}) |
| 40 | if !strings.Contains(got, "-20.0%") { |
| 41 | t.Fatalf("want the harness's under-report surfaced:\n%s", got) |
| 42 | } |
| 43 | } |
| 44 | |
| 45 | func TestMeterAccountingSurfacesUnmeasuredAndFaults(t *testing.T) { |
| 46 | r := meteredRun("a", 100, 10, 100, 10) |
| 47 | r.Meter.WithoutUsage = 2 |
| 48 | r.Meter.Injected = 1 |
| 49 | got := renderMeterAccounting([]result{r}) |
| 50 | if !strings.Contains(got, "**responses without usage** 2") { |
| 51 | t.Fatalf("unmeasured responses must be visible:\n%s", got) |
| 52 | } |
| 53 | if !strings.Contains(got, "injected faults 1") { |
| 54 | t.Fatalf("injected faults must be visible:\n%s", got) |
| 55 | } |
| 56 | } |
| 57 | |
| 58 | // An unaccounted run has no harness numbers to compare, so it must not drag the |
| 59 | // divergence toward zero by contributing a silent 0. |
| 60 | func TestMeterAccountingSkipsUnaccountedRunsInTheComparison(t *testing.T) { |
| 61 | lost := meteredRun("lost", 1000, 0, 0, 0) |
| 62 | lost.Unaccounted = true |
| 63 | got := renderMeterAccounting([]result{meteredRun("a", 1000, 0, 1000, 0), lost}) |
| 64 | if !strings.Contains(got, "+0.0% (harness 1,000 vs meter 1,000 over 1 runs)") { |
| 65 | t.Fatalf("the comparison must drop the unaccounted run from BOTH sides:\n%s", got) |
| 66 | } |
| 67 | if !strings.Contains(got, "tokens 2,000") { |
| 68 | t.Fatalf("real spend still includes the unaccounted run:\n%s", got) |
| 69 | } |
| 70 | } |
| 71 | |
| 72 | func TestMeterAccountingRendersNothingUnmetered(t *testing.T) { |
| 73 | if got := renderMeterAccounting([]result{{task: task{ID: "a"}, Attempt: 1}}); got != "" { |
| 74 | t.Fatalf("want no section when nothing was metered, got:\n%s", got) |
| 75 | } |
| 76 | } |
| 77 | |
| 78 | func TestStartTaskMeterIsOffByDefault(t *testing.T) { |
| 79 | env, m, stop, err := startTaskMeter(suiteConfig{}) |
| 80 | if err != nil || env != nil || m != nil || stop != nil { |
| 81 | t.Fatalf("metering must be opt-in: env=%v meter=%v err=%v", env, m, err) |
| 82 | } |
| 83 | } |
| 84 | |
| 85 | func TestStartTaskMeterPointsTheChildAtTheProxy(t *testing.T) { |
| 86 | cfg := suiteConfig{meterConfig: writeConfig(t, twoProviderConfig), model: "kimi-k2"} |
| 87 | env, m, stop, err := startTaskMeter(cfg) |
| 88 | if err != nil { |
| 89 | t.Fatalf("startTaskMeter: %v", err) |
| 90 | } |
| 91 | defer stop() |
| 92 | if m == nil || len(env) != 1 || !strings.HasPrefix(env[0], "REASONIX_HOME=") { |
| 93 | t.Fatalf("env = %v, want a redirected REASONIX_HOME", env) |
| 94 | } |
| 95 | home := strings.TrimPrefix(env[0], "REASONIX_HOME=") |
| 96 | providers := readProviders(t, home) |
| 97 | for _, p := range providers { |
| 98 | if name, _ := p["name"].(string); name == "kimi" { |
| 99 | base, _ := p["base_url"].(string) |
| 100 | if !strings.HasPrefix(base, "http://127.0.0.1:") { |
| 101 | t.Fatalf("kimi base_url = %q, want the loopback meter", base) |
| 102 | } |
| 103 | return |
| 104 | } |
| 105 | } |
| 106 | t.Fatalf("kimi provider missing from the metered home: %v", providers) |
| 107 | } |
| 108 |