| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "encoding/json" |
| 6 | "io" |
| 7 | "net/http" |
| 8 | "net/http/httptest" |
| 9 | "strings" |
| 10 | "testing" |
| 11 | ) |
| 12 | |
| 13 | func meterAgainst(t *testing.T, upstream http.Handler, faults faultScript) (*meter, string, func()) { |
| 14 | t.Helper() |
| 15 | up := httptest.NewServer(upstream) |
| 16 | m, err := newMeter(up.URL, faults) |
| 17 | if err != nil { |
| 18 | t.Fatalf("newMeter: %v", err) |
| 19 | } |
| 20 | base, stop, err := m.serve() |
| 21 | if err != nil { |
| 22 | t.Fatalf("serve: %v", err) |
| 23 | } |
| 24 | return m, base, func() { stop(); up.Close() } |
| 25 | } |
| 26 | |
| 27 | func post(t *testing.T, base, path, body string) *http.Response { |
| 28 | t.Helper() |
| 29 | resp, err := http.Post(base+path, "application/json", strings.NewReader(body)) |
| 30 | if err != nil { |
| 31 | t.Fatalf("post: %v", err) |
| 32 | } |
| 33 | return resp |
| 34 | } |
| 35 | |
| 36 | func TestMeterCountsNonStreamingUsage(t *testing.T) { |
| 37 | upstream := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 38 | w.Header().Set("Content-Type", "application/json") |
| 39 | io.WriteString(w, `{"choices":[{"message":{"content":"hi"}}],"usage":{"prompt_tokens":100,"completion_tokens":20,"prompt_cache_hit_tokens":64,"prompt_cache_miss_tokens":36}}`) |
| 40 | }) |
| 41 | m, base, stop := meterAgainst(t, upstream, faultScript{}) |
| 42 | defer stop() |
| 43 | |
| 44 | resp := post(t, base, "/chat/completions", `{"model":"x"}`) |
| 45 | body, _ := io.ReadAll(resp.Body) |
| 46 | resp.Body.Close() |
| 47 | if !bytes.Contains(body, []byte(`"content":"hi"`)) { |
| 48 | t.Fatalf("response not forwarded: %s", body) |
| 49 | } |
| 50 | got := m.snapshot() |
| 51 | if got.Requests != 1 || got.PromptTokens != 100 || got.CompletionTokens != 20 { |
| 52 | t.Fatalf("usage = %+v", got) |
| 53 | } |
| 54 | if got.CacheHitTokens != 64 || got.CacheMissTokens != 36 { |
| 55 | t.Fatalf("cache split = %d/%d, want 64/36", got.CacheHitTokens, got.CacheMissTokens) |
| 56 | } |
| 57 | if got.WithoutUsage != 0 { |
| 58 | t.Fatalf("usage was present; WithoutUsage = %d", got.WithoutUsage) |
| 59 | } |
| 60 | } |
| 61 | |
| 62 | func TestMeterReadsOpenAICachedTokensSpelling(t *testing.T) { |
| 63 | upstream := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 64 | w.Header().Set("Content-Type", "application/json") |
| 65 | io.WriteString(w, `{"usage":{"prompt_tokens":90,"completion_tokens":5,"prompt_tokens_details":{"cached_tokens":30}}}`) |
| 66 | }) |
| 67 | m, base, stop := meterAgainst(t, upstream, faultScript{}) |
| 68 | defer stop() |
| 69 | post(t, base, "/chat/completions", `{"model":"x"}`).Body.Close() |
| 70 | |
| 71 | got := m.snapshot() |
| 72 | if got.CacheHitTokens != 30 || got.CacheMissTokens != 60 { |
| 73 | t.Fatalf("cache split = %d/%d, want 30/60 derived from prompt_tokens", got.CacheHitTokens, got.CacheMissTokens) |
| 74 | } |
| 75 | } |
| 76 | |
| 77 | func TestMeterCountsStreamedUsageAndForwardsFrames(t *testing.T) { |
| 78 | upstream := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 79 | w.Header().Set("Content-Type", "text/event-stream") |
| 80 | io.WriteString(w, "data: {\"choices\":[{\"delta\":{\"content\":\"a\"}}]}\n\n") |
| 81 | io.WriteString(w, "data: {\"usage\":{\"prompt_tokens\":7,\"completion_tokens\":3}}\n\n") |
| 82 | io.WriteString(w, "data: [DONE]\n\n") |
| 83 | }) |
| 84 | m, base, stop := meterAgainst(t, upstream, faultScript{}) |
| 85 | defer stop() |
| 86 | |
| 87 | resp := post(t, base, "/chat/completions", `{"model":"x","stream":true}`) |
| 88 | body, _ := io.ReadAll(resp.Body) |
| 89 | resp.Body.Close() |
| 90 | if !strings.Contains(string(body), "[DONE]") || !strings.Contains(string(body), `"content":"a"`) { |
| 91 | t.Fatalf("frames not forwarded verbatim: %q", body) |
| 92 | } |
| 93 | got := m.snapshot() |
| 94 | if got.PromptTokens != 7 || got.CompletionTokens != 3 || got.WithoutUsage != 0 { |
| 95 | t.Fatalf("streamed usage = %+v", got) |
| 96 | } |
| 97 | } |
| 98 | |
| 99 | // A harness that never asks for usage would otherwise measure as free. |
| 100 | func TestMeterOptsStreamedRequestsIntoUsage(t *testing.T) { |
| 101 | var seen []byte |
| 102 | upstream := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 103 | seen, _ = io.ReadAll(r.Body) |
| 104 | w.Header().Set("Content-Type", "text/event-stream") |
| 105 | io.WriteString(w, "data: [DONE]\n\n") |
| 106 | }) |
| 107 | _, base, stop := meterAgainst(t, upstream, faultScript{}) |
| 108 | defer stop() |
| 109 | post(t, base, "/chat/completions", `{"model":"x","stream":true}`).Body.Close() |
| 110 | |
| 111 | var payload map[string]any |
| 112 | if err := json.Unmarshal(seen, &payload); err != nil { |
| 113 | t.Fatalf("upstream body: %v", err) |
| 114 | } |
| 115 | opts, ok := payload["stream_options"].(map[string]any) |
| 116 | if !ok || opts["include_usage"] != true { |
| 117 | t.Fatalf("stream_options not injected: %s", seen) |
| 118 | } |
| 119 | } |
| 120 | |
| 121 | func TestMeterLeavesNonStreamedRequestsAlone(t *testing.T) { |
| 122 | var seen []byte |
| 123 | upstream := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 124 | seen, _ = io.ReadAll(r.Body) |
| 125 | w.Header().Set("Content-Type", "application/json") |
| 126 | io.WriteString(w, `{"usage":{"prompt_tokens":1,"completion_tokens":1}}`) |
| 127 | }) |
| 128 | _, base, stop := meterAgainst(t, upstream, faultScript{}) |
| 129 | defer stop() |
| 130 | post(t, base, "/chat/completions", `{"model":"x"}`).Body.Close() |
| 131 | |
| 132 | if strings.Contains(string(seen), "stream_options") { |
| 133 | t.Fatalf("non-streamed request was rewritten: %s", seen) |
| 134 | } |
| 135 | } |
| 136 | |
| 137 | func TestMeterReportsResponsesWithoutUsage(t *testing.T) { |
| 138 | upstream := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 139 | w.Header().Set("Content-Type", "application/json") |
| 140 | io.WriteString(w, `{"choices":[]}`) |
| 141 | }) |
| 142 | m, base, stop := meterAgainst(t, upstream, faultScript{}) |
| 143 | defer stop() |
| 144 | post(t, base, "/chat/completions", `{"model":"x"}`).Body.Close() |
| 145 | |
| 146 | if got := m.snapshot(); got.WithoutUsage != 1 || got.PromptTokens != 0 { |
| 147 | t.Fatalf("unmeasured response must be reported, not zeroed: %+v", got) |
| 148 | } |
| 149 | } |
| 150 | |
| 151 | func TestMeterInjectsFaultsByRequestIndex(t *testing.T) { |
| 152 | reached := 0 |
| 153 | upstream := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 154 | reached++ |
| 155 | w.Header().Set("Content-Type", "application/json") |
| 156 | io.WriteString(w, `{"usage":{"prompt_tokens":1,"completion_tokens":1}}`) |
| 157 | }) |
| 158 | m, base, stop := meterAgainst(t, upstream, faultScript{at: map[int]int{2: 429}}) |
| 159 | defer stop() |
| 160 | |
| 161 | for i := range 3 { |
| 162 | resp := post(t, base, "/chat/completions", `{"model":"x"}`) |
| 163 | want := http.StatusOK |
| 164 | if i == 1 { |
| 165 | want = http.StatusTooManyRequests |
| 166 | } |
| 167 | if resp.StatusCode != want { |
| 168 | t.Fatalf("request %d status = %d, want %d", i+1, resp.StatusCode, want) |
| 169 | } |
| 170 | resp.Body.Close() |
| 171 | } |
| 172 | if reached != 2 { |
| 173 | t.Fatalf("upstream saw %d requests, want 2 — the faulted one must not be forwarded", reached) |
| 174 | } |
| 175 | if got := m.snapshot(); got.Injected != 1 || got.Requests != 3 { |
| 176 | t.Fatalf("meter = %+v, want 3 requests with 1 injected", got) |
| 177 | } |
| 178 | } |
| 179 | |
| 180 | func TestParseFaultScript(t *testing.T) { |
| 181 | got, err := parseFaultScript(" 3:429 , 7:500 ") |
| 182 | if err != nil { |
| 183 | t.Fatalf("parse: %v", err) |
| 184 | } |
| 185 | if got.at[3] != 429 || got.at[7] != 500 || len(got.at) != 2 { |
| 186 | t.Fatalf("faults = %v", got) |
| 187 | } |
| 188 | if got, err := parseFaultScript(""); err != nil || !got.empty() { |
| 189 | t.Fatalf("empty spec = %v, %v", got, err) |
| 190 | } |
| 191 | for _, bad := range []string{"3", "0:429", "3:200", "x:429", "3:999"} { |
| 192 | if _, err := parseFaultScript(bad); err == nil { |
| 193 | t.Fatalf("%q must be rejected", bad) |
| 194 | } |
| 195 | } |
| 196 | } |
| 197 | |
| 198 | func TestNewMeterRejectsRelativeUpstream(t *testing.T) { |
| 199 | if _, err := newMeter("/v1", faultScript{}); err == nil { |
| 200 | t.Fatal("a relative upstream must be rejected") |
| 201 | } |
| 202 | } |
| 203 | |
| 204 | // okUpstream is a minimal usage-reporting upstream for fault tests. |
| 205 | func okUpstream() http.Handler { |
| 206 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 207 | w.Header().Set("Content-Type", "application/json") |
| 208 | io.WriteString(w, `{"usage":{"prompt_tokens":1,"completion_tokens":1}}`) |
| 209 | }) |
| 210 | } |
| 211 |