| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "fmt" |
| 5 | "os" |
| 6 | "strings" |
| 7 | ) |
| 8 | |
| 9 | // meterSettings validates the metering flags together: a fault script has |
| 10 | // nowhere to be injected without the proxy, so asking for one without -meter |
| 11 | // is a mistake worth stopping for rather than silently ignoring. |
| 12 | func meterSettings(configPath, faultSpec string) (string, faultScript) { |
| 13 | faults, err := parseFaultScript(faultSpec) |
| 14 | if err != nil { |
| 15 | fmt.Fprintln(os.Stderr, "faults:", err) |
| 16 | os.Exit(2) |
| 17 | } |
| 18 | if !faults.empty() && strings.TrimSpace(configPath) == "" { |
| 19 | fmt.Fprintln(os.Stderr, "faults: -faults needs -meter; the proxy is where a fault can be injected") |
| 20 | os.Exit(2) |
| 21 | } |
| 22 | return configPath, faults |
| 23 | } |
| 24 | |
| 25 | // runMeter is one run's metering: the environment that points the child at the |
| 26 | // proxy, and the proxy itself. The zero value is a working no-op, so an |
| 27 | // unmetered run needs no branch at the call site. |
| 28 | type runMeter struct { |
| 29 | env []string |
| 30 | m *meter |
| 31 | stop func() |
| 32 | } |
| 33 | |
| 34 | // attachMeter starts the run's meter, folding a setup failure into the run's |
| 35 | // note instead of aborting the suite: an unmetered run is still a data point. |
| 36 | func attachMeter(cfg suiteConfig, r *result) runMeter { |
| 37 | env, m, stop, err := startTaskMeter(cfg) |
| 38 | if err != nil { |
| 39 | r.Note = strings.TrimSpace(r.Note + " meter: " + err.Error()) |
| 40 | } |
| 41 | return runMeter{env: env, m: m, stop: stop} |
| 42 | } |
| 43 | |
| 44 | func (rm runMeter) close() { |
| 45 | if rm.stop != nil { |
| 46 | rm.stop() |
| 47 | } |
| 48 | } |
| 49 | |
| 50 | // record attaches what the proxy observed. It stays separate from close so the |
| 51 | // snapshot is taken while the run's numbers are still being assembled. |
| 52 | func (rm runMeter) record(r *result) { |
| 53 | if rm.m == nil { |
| 54 | return |
| 55 | } |
| 56 | observed := rm.m.snapshot() |
| 57 | r.Meter = &observed |
| 58 | } |
| 59 | |
| 60 | // startTaskMeter brings up a per-task meter and returns the environment that |
| 61 | // points the child at it. One meter per task keeps spend attributable to the |
| 62 | // task that incurred it. A nil meter means metering is off, which is the |
| 63 | // default: the proxy carries provider credentials and must be opt-in. |
| 64 | func startTaskMeter(cfg suiteConfig) (env []string, m *meter, stop func(), err error) { |
| 65 | if strings.TrimSpace(cfg.meterConfig) == "" { |
| 66 | return nil, nil, nil, nil |
| 67 | } |
| 68 | upstream, err := meterUpstream(cfg.meterConfig, cfg.model) |
| 69 | if err != nil { |
| 70 | return nil, nil, nil, err |
| 71 | } |
| 72 | m, err = newMeter(upstream, cfg.meterFaults) |
| 73 | if err != nil { |
| 74 | return nil, nil, nil, err |
| 75 | } |
| 76 | base, stopServer, err := m.serve() |
| 77 | if err != nil { |
| 78 | return nil, nil, nil, err |
| 79 | } |
| 80 | home, err := os.MkdirTemp("", "e2ebench-meter-home-") |
| 81 | if err != nil { |
| 82 | stopServer() |
| 83 | return nil, nil, nil, err |
| 84 | } |
| 85 | if err := writeMeteredConfig(cfg.meterConfig, home, cfg.model, base); err != nil { |
| 86 | stopServer() |
| 87 | _ = os.RemoveAll(home) |
| 88 | return nil, nil, nil, err |
| 89 | } |
| 90 | return []string{"REASONIX_HOME=" + home}, m, func() { |
| 91 | stopServer() |
| 92 | _ = os.RemoveAll(home) |
| 93 | }, nil |
| 94 | } |
| 95 | |
| 96 | // renderMeterAccounting reports what the neutral proxy measured and how far the |
| 97 | // harness's own accounting drifted from it. Divergence is the number that |
| 98 | // decides whether a cross-harness comparison is publishable at all: if the |
| 99 | // proxy and the harness disagree about the same run, one of them is wrong. |
| 100 | func renderMeterAccounting(results []result) string { |
| 101 | metered, selfReported, unmeasured, injected := 0, 0, 0, 0 |
| 102 | var meterTokens, harnessTokens, comparableTokens int |
| 103 | var hit, miss int |
| 104 | for _, r := range results { |
| 105 | if r.Skipped || r.Meter == nil { |
| 106 | continue |
| 107 | } |
| 108 | metered++ |
| 109 | meterTokens += r.Meter.PromptTokens + r.Meter.CompletionTokens |
| 110 | hit += r.Meter.CacheHitTokens |
| 111 | miss += r.Meter.CacheMissTokens |
| 112 | unmeasured += r.Meter.WithoutUsage |
| 113 | injected += r.Meter.Injected |
| 114 | // A run whose metrics file never landed has no harness number, so it |
| 115 | // stays out of both sides of the comparison; folding its meter tokens |
| 116 | // against a silent zero would invent a divergence. |
| 117 | if !r.Unaccounted { |
| 118 | selfReported++ |
| 119 | harnessTokens += r.PromptTokens + r.CompletionTokens |
| 120 | comparableTokens += r.Meter.PromptTokens + r.Meter.CompletionTokens |
| 121 | } |
| 122 | } |
| 123 | if metered == 0 { |
| 124 | return "" |
| 125 | } |
| 126 | var b strings.Builder |
| 127 | fmt.Fprintf(&b, "**Metered at the boundary** (%d runs): tokens %s · cache hit %s", |
| 128 | metered, comma(meterTokens), pct(hit, hit+miss)) |
| 129 | if injected > 0 { |
| 130 | fmt.Fprintf(&b, " · injected faults %d", injected) |
| 131 | } |
| 132 | if unmeasured > 0 { |
| 133 | fmt.Fprintf(&b, " · **responses without usage** %d", unmeasured) |
| 134 | } |
| 135 | if selfReported > 0 && comparableTokens > 0 { |
| 136 | fmt.Fprintf(&b, " · **self-report divergence** %s (harness %s vs meter %s over %d runs)", |
| 137 | divergence(harnessTokens, comparableTokens), comma(harnessTokens), comma(comparableTokens), selfReported) |
| 138 | } |
| 139 | return b.String() + "\n\n" |
| 140 | } |
| 141 | |
| 142 | // divergence is the harness's own token count as a signed deviation from the |
| 143 | // meter's. Anything but ~0 means the two disagree about the same traffic. |
| 144 | func divergence(harness, meter int) string { |
| 145 | if meter == 0 { |
| 146 | return "—" |
| 147 | } |
| 148 | delta := float64(harness-meter) / float64(meter) * 100 |
| 149 | return fmt.Sprintf("%+.1f%%", delta) |
| 150 | } |
| 151 |