| 1 | // Package replay computes paired-run medians for use_capability eval harnesses. |
| 2 | package replay |
| 3 | |
| 4 | import ( |
| 5 | "encoding/json" |
| 6 | "fmt" |
| 7 | "os" |
| 8 | "sort" |
| 9 | ) |
| 10 | |
| 11 | // Pair is one proxy-vs-baseline observation. |
| 12 | type Pair struct { |
| 13 | Name string `json:"name"` |
| 14 | ProxyListCount int `json:"proxy_list_count"` |
| 15 | BaselineListCount int `json:"baseline_list_count"` |
| 16 | ProxyLatencyMs float64 `json:"proxy_latency_ms"` |
| 17 | BaselineLatencyMs float64 `json:"baseline_latency_ms"` |
| 18 | } |
| 19 | |
| 20 | // Report is the median of five (or more) paired runs. |
| 21 | type Report struct { |
| 22 | Pairs int `json:"pairs"` |
| 23 | MedianListDelta float64 `json:"median_list_delta"` |
| 24 | MedianLatencyDelta float64 `json:"median_latency_delta"` |
| 25 | } |
| 26 | |
| 27 | // LoadPairs reads a JSON array of paired runs. |
| 28 | func LoadPairs(path string) ([]Pair, error) { |
| 29 | raw, err := os.ReadFile(path) |
| 30 | if err != nil { |
| 31 | return nil, err |
| 32 | } |
| 33 | var pairs []Pair |
| 34 | if err := json.Unmarshal(raw, &pairs); err != nil { |
| 35 | return nil, fmt.Errorf("decode paired runs: %w", err) |
| 36 | } |
| 37 | return pairs, nil |
| 38 | } |
| 39 | |
| 40 | // MedianReport returns median(proxy-baseline) for list count and latency. |
| 41 | func MedianReport(pairs []Pair) Report { |
| 42 | list := make([]float64, 0, len(pairs)) |
| 43 | lat := make([]float64, 0, len(pairs)) |
| 44 | for _, p := range pairs { |
| 45 | list = append(list, float64(p.ProxyListCount-p.BaselineListCount)) |
| 46 | lat = append(lat, p.ProxyLatencyMs-p.BaselineLatencyMs) |
| 47 | } |
| 48 | return Report{ |
| 49 | Pairs: len(pairs), |
| 50 | MedianListDelta: median(list), |
| 51 | MedianLatencyDelta: median(lat), |
| 52 | } |
| 53 | } |
| 54 | |
| 55 | func median(values []float64) float64 { |
| 56 | if len(values) == 0 { |
| 57 | return 0 |
| 58 | } |
| 59 | sorted := append([]float64(nil), values...) |
| 60 | sort.Float64s(sorted) |
| 61 | mid := len(sorted) / 2 |
| 62 | if len(sorted)%2 == 1 { |
| 63 | return sorted[mid] |
| 64 | } |
| 65 | return (sorted[mid-1] + sorted[mid]) / 2 |
| 66 | } |
| 67 |