返回 CodeWhale
tests.rs
根目录 / crates / config / src / pricing / tests.rs
1 //! Behavior tests for the offering pricing projection (#3085).
2
3 use super::*;
4 use crate::catalog::{CatalogOffering, CatalogSource, bundled_offerings_from_models_dev};
5 use crate::models_dev::{ModelsDevCatalog, ModelsDevCost};
6 use crate::route::PricingSku;
7
8 /// A DeepSeek-shaped priced offering (input/output/cache-read known,
9 /// cache-write deliberately unknown) tagged with the given provenance source.
10 fn priced(source: CatalogSource) -> CatalogOffering {
11 CatalogOffering {
12 provider: "deepseek".into(),
13 wire_model_id: "deepseek-v4-pro".into(),
14 canonical_model: Some("deepseek-v4-pro".into()),
15 endpoint_key: "chat".into(),
16 cost: Some(ModelsDevCost {
17 input: Some(0.28),
18 output: Some(0.42),
19 cache_read: Some(0.028),
20 cache_write: None,
21 }),
22 source,
23 ..Default::default()
24 }
25 }
26
27 #[test]
28 fn maps_models_dev_cost_with_bundled_provenance_in_usd() {
29 let p =
30 OfferingPricing::from_catalog_offering(&priced(CatalogSource::Bundled)).expect("priced");
31 assert_eq!(p.currency, Currency::Usd);
32 assert_eq!(p.input_per_million, Some(0.28));
33 assert_eq!(p.output_per_million, Some(0.42));
34 assert_eq!(p.cache_read_per_million, Some(0.028));
35 assert_eq!(p.cache_write_per_million, None);
36 assert_eq!(p.provenance, PricingProvenance::ModelsDevBundled);
37 assert_eq!(p.effective_at, None);
38 assert!(p.has_any_price());
39 }
40
41 #[test]
42 fn malformed_catalog_prices_fail_closed_at_every_projection() {
43 for invalid in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY, -0.01] {
44 for field in 0..4 {
45 let mut offering = priced(CatalogSource::Bundled);
46 offering.provider = "openrouter".to_string();
47 offering.wire_model_id = "openai/gpt-5.5".to_string();
48 let cost = offering.cost.as_mut().expect("cost fixture");
49 match field {
50 0 => cost.input = Some(invalid),
51 1 => cost.output = Some(invalid),
52 2 => cost.cache_read = Some(invalid),
53 3 => cost.cache_write = Some(invalid),
54 _ => unreachable!(),
55 }
56 assert!(
57 OfferingPricing::from_catalog_offering(&offering).is_none(),
58 "field {field} accepted {invalid:?}"
59 );
60 assert_eq!(
61 route_pricing_sku(&offering),
62 PricingSku::UnknownOrStale,
63 "route projection accepted field {field} = {invalid:?}"
64 );
65 }
66 }
67
68 let zero = priced(CatalogSource::Bundled);
69 let mut zero = zero;
70 let cost = zero.cost.as_mut().expect("cost fixture");
71 cost.input = Some(0.0);
72 cost.output = Some(0.0);
73 cost.cache_read = Some(0.0);
74 cost.cache_write = Some(0.0);
75 assert!(OfferingPricing::from_catalog_offering(&zero).is_some());
76 }
77
78 /// A price of an impossible magnitude is a unit error, not an expensive model.
79 ///
80 /// The two ways this happens in the wild are a per-token price parsed as
81 /// per-million (10^6 too large) and a minor-unit integer read as a major unit
82 /// (10^2 too large). Both would bill the user orders of magnitude over, so the
83 /// row is rejected at the same boundary that rejects NaN and negatives — and
84 /// rejected *whole*, so the surviving fields cannot produce a quietly
85 /// under-counted estimate instead.
86 #[test]
87 fn absurd_catalog_prices_fail_closed_at_every_projection() {
88 let absurd = [
89 // $3/token, mistakenly published as a per-million rate.
90 3_000_000.0,
91 // The declared bound itself is out, one ulp above is far out.
92 MAX_PLAUSIBLE_PRICE_PER_MILLION + 1.0,
93 f64::MAX,
94 ];
95 for price in absurd {
96 for field in 0..4 {
97 let mut offering = priced(CatalogSource::Bundled);
98 let cost = offering.cost.as_mut().expect("cost fixture");
99 match field {
100 0 => cost.input = Some(price),
101 1 => cost.output = Some(price),
102 2 => cost.cache_read = Some(price),
103 3 => cost.cache_write = Some(price),
104 _ => unreachable!(),
105 }
106 assert!(
107 !catalog_cost_is_valid(cost),
108 "field {field} accepted absurd price {price}"
109 );
110 assert!(
111 OfferingPricing::from_catalog_offering(&offering).is_none(),
112 "field {field} produced pricing from absurd price {price}"
113 );
114 assert_eq!(
115 route_pricing_sku(&offering),
116 PricingSku::UnknownOrStale,
117 "route projection accepted field {field} = {price}"
118 );
119 }
120 }
121
122 // The bound is generous on purpose: a genuinely expensive published rate
123 // stays priced. Rejecting a real price would be its own kind of lie.
124 let mut expensive = priced(CatalogSource::Bundled);
125 let cost = expensive.cost.as_mut().expect("cost fixture");
126 cost.input = Some(600.0);
127 cost.output = Some(2_400.0);
128 assert!(
129 OfferingPricing::from_catalog_offering(&expensive).is_some(),
130 "a plausible frontier rate must survive the magnitude bound"
131 );
132
133 // Exactly at the bound is still accepted; only values above it are not.
134 let mut boundary = priced(CatalogSource::Bundled);
135 let cost = boundary.cost.as_mut().expect("cost fixture");
136 cost.input = Some(MAX_PLAUSIBLE_PRICE_PER_MILLION);
137 assert!(OfferingPricing::from_catalog_offering(&boundary).is_some());
138 }
139
140 #[test]
141 fn live_source_carries_provider_live_provenance_and_effective_at() {
142 let src = CatalogSource::Live {
143 base_url_fingerprint: "fp".into(),
144 fetched_at: 1_700,
145 };
146 let p = OfferingPricing::from_catalog_offering(&priced(src)).expect("priced");
147 assert_eq!(p.provenance, PricingProvenance::ProviderLive);
148 assert_eq!(p.effective_at, Some(1_700));
149 assert_eq!(p.endpoint_fingerprint.as_deref(), Some("fp"));
150 }
151
152 /// A live row is only authoritative while it is *both* fresh and fetched from
153 /// the endpoint the turn was served on. Every other combination is a defect the
154 /// caller must fail closed on rather than billing against.
155 #[test]
156 fn live_pricing_defect_gates_stale_and_mismatched_rows() {
157 let live = OfferingPricing::from_catalog_offering(&priced(CatalogSource::Live {
158 base_url_fingerprint: "route-fp".into(),
159 fetched_at: 1_000,
160 }))
161 .expect("priced");
162
163 // Fresh + fingerprint-matched: authoritative.
164 assert_eq!(
165 live.live_pricing_defect(Some("route-fp"), Some(1_500), 1_000),
166 None
167 );
168
169 // Same age, at the window boundary: stale is inclusive, matching
170 // `is_stale`, so a row exactly at the TTL is not authoritative.
171 assert_eq!(
172 live.live_pricing_defect(Some("route-fp"), Some(2_000), 1_000),
173 Some(LivePricingDefect::Stale {
174 age_secs: 1_000,
175 max_age_secs: 1_000,
176 })
177 );
178
179 // A row fetched from a different endpoint prices a different billing
180 // surface; it is never a "fresher price" for this route.
181 assert_eq!(
182 live.live_pricing_defect(Some("other-fp"), Some(1_500), 1_000),
183 Some(LivePricingDefect::EndpointMismatch {
184 row_fingerprint: "route-fp".into(),
185 route_fingerprint: "other-fp".into(),
186 })
187 );
188
189 // An unknown route endpoint cannot confirm any live row.
190 assert_eq!(
191 live.live_pricing_defect(None, Some(1_500), 1_000),
192 Some(LivePricingDefect::UnknownRouteEndpoint)
193 );
194
195 // No clock means the age is unknowable, so the row stays unproven rather
196 // than being assumed fresh.
197 assert_eq!(
198 live.live_pricing_defect(Some("route-fp"), None, 1_000),
199 Some(LivePricingDefect::MissingTimestamp)
200 );
201
202 // Non-live provenances carry no fetch clock and are not age-gated here.
203 for source in [CatalogSource::Bundled, CatalogSource::UserOverride] {
204 let row = OfferingPricing::from_catalog_offering(&priced(source)).expect("priced");
205 assert_eq!(row.live_pricing_defect(None, Some(u64::MAX), 1), None);
206 assert!(row.provenance.is_authoritative_without_freshness_check());
207 }
208 assert!(!PricingProvenance::ProviderLive.is_authoritative_without_freshness_check());
209 }
210
211 /// A live row that claims live provenance but lost its fingerprint (hand-built
212 /// or migrated from an older schema) cannot be matched to a route.
213 #[test]
214 fn live_row_without_a_fingerprint_is_never_authoritative() {
215 let mut live = OfferingPricing::from_catalog_offering(&priced(CatalogSource::Live {
216 base_url_fingerprint: "fp".into(),
217 fetched_at: 1_000,
218 }))
219 .expect("priced");
220 live.endpoint_fingerprint = None;
221 assert_eq!(
222 live.live_pricing_defect(Some("fp"), Some(1_001), 1_000),
223 Some(LivePricingDefect::MissingEndpointFingerprint)
224 );
225
226 // Defect labels are stable, non-localized, and carry no URL — only the
227 // non-secret FNV digests the catalog already scopes caches on.
228 let mismatch = LivePricingDefect::EndpointMismatch {
229 row_fingerprint: "a".into(),
230 route_fingerprint: "b".into(),
231 };
232 assert_eq!(mismatch.label(), "live_pricing_endpoint_mismatch");
233 let json = serde_json::to_string(&mismatch).expect("serialize defect");
234 assert!(!json.contains("http"), "{json}");
235 }
236
237 #[test]
238 fn no_cost_or_empty_cost_object_is_unknown() {
239 let mut offering = priced(CatalogSource::Bundled);
240 offering.cost = None;
241 assert!(
242 OfferingPricing::from_catalog_offering(&offering).is_none(),
243 "absent cost is unknown, not free"
244 );
245
246 // A cost object present but with no concrete price is still unknown.
247 offering.cost = Some(ModelsDevCost::default());
248 assert!(OfferingPricing::from_catalog_offering(&offering).is_none());
249 }
250
251 #[test]
252 fn estimate_cost_sums_priced_classes() {
253 let p = OfferingPricing::from_catalog_offering(&priced(CatalogSource::Bundled)).unwrap();
254 // 1M input @0.28 + 0.5M output @0.42 + 2M cache_read @0.028 = 0.546
255 let usage = TokenUsage {
256 input: 1_000_000,
257 output: 500_000,
258 cache_read: 2_000_000,
259 cache_write: 0,
260 };
261 let cost = p.estimate_cost(&usage).expect("priced classes estimate");
262 assert!((cost - 0.546).abs() < 1e-9, "got {cost}");
263 }
264
265 #[test]
266 fn estimate_cost_is_none_when_a_used_class_is_unpriced() {
267 // cache_write price is unknown; charging cache-write tokens cannot be
268 // estimated honestly, so the whole estimate is None rather than under-reported.
269 let p = OfferingPricing::from_catalog_offering(&priced(CatalogSource::Bundled)).unwrap();
270 let usage = TokenUsage {
271 input: 100,
272 output: 0,
273 cache_read: 0,
274 cache_write: 10,
275 };
276 assert!(p.estimate_cost(&usage).is_none());
277 }
278
279 #[test]
280 fn estimate_cost_with_zero_usage_is_zero() {
281 let p = OfferingPricing::from_catalog_offering(&priced(CatalogSource::Bundled)).unwrap();
282 assert_eq!(p.estimate_cost(&TokenUsage::default()), Some(0.0));
283 }
284
285 #[test]
286 fn finite_rates_that_overflow_the_computed_total_fail_closed() {
287 // Constructed directly rather than through `from_catalog_offering`: the
288 // magnitude bound now rejects a rate this large at the catalog boundary, so
289 // the only way to reach the estimator with one is to bypass that boundary.
290 // The estimator keeps its own overflow guard regardless — it is the last
291 // check before a number becomes money, and it must not depend on an earlier
292 // layer having run.
293 let pricing = OfferingPricing {
294 provider: "deepseek".to_string(),
295 wire_model_id: "deepseek-v4-pro".to_string(),
296 canonical_model: Some("deepseek-v4-pro".to_string()),
297 currency: Currency::Usd,
298 input_per_million: Some(f64::MAX),
299 output_per_million: None,
300 cache_read_per_million: None,
301 cache_write_per_million: None,
302 provenance: PricingProvenance::ModelsDevBundled,
303 effective_at: None,
304 endpoint_fingerprint: None,
305 };
306 let usage = TokenUsage {
307 input: u64::MAX,
308 ..TokenUsage::default()
309 };
310 assert_eq!(pricing.estimate_cost(&usage), None);
311
312 // And the boundary itself refuses to hand such a row over in the first
313 // place, so the guard above is defence in depth, not the only defence.
314 let mut offering = priced(CatalogSource::Bundled);
315 offering.cost.as_mut().expect("cost fixture").input = Some(f64::MAX);
316 assert!(OfferingPricing::from_catalog_offering(&offering).is_none());
317 }
318
319 #[test]
320 fn route_pricing_sku_is_token_when_priced_and_unknown_otherwise() {
321 match route_pricing_sku(&priced(CatalogSource::Bundled)) {
322 PricingSku::Token {
323 input_per_mtok,
324 output_per_mtok,
325 } => {
326 assert_eq!(input_per_mtok, Some(0.28));
327 assert_eq!(output_per_mtok, Some(0.42));
328 }
329 other => panic!("expected Token, got {other:?}"),
330 }
331
332 // No cost → honest UnknownOrStale, never a fabricated zero price.
333 let mut unpriced = priced(CatalogSource::Bundled);
334 unpriced.cost = None;
335 assert!(matches!(
336 route_pricing_sku(&unpriced),
337 PricingSku::UnknownOrStale
338 ));
339 }
340
341 #[test]
342 fn currency_round_trips_including_other() {
343 for currency in [Currency::Usd, Currency::Cny, Currency::Other("eur".into())] {
344 let json = serde_json::to_string(&currency).expect("serialize");
345 let back: Currency = serde_json::from_str(&json).expect("deserialize");
346 assert_eq!(currency, back);
347 }
348 }
349
350 #[test]
351 fn user_override_pricing_round_trips_and_carries_no_secrets() {
352 let pricing = OfferingPricing {
353 provider: "custom".into(),
354 wire_model_id: "house-model".into(),
355 canonical_model: None,
356 currency: Currency::Cny,
357 input_per_million: Some(8.0),
358 output_per_million: Some(16.0),
359 cache_read_per_million: None,
360 cache_write_per_million: None,
361 provenance: PricingProvenance::UserOverride,
362 effective_at: None,
363 endpoint_fingerprint: None,
364 };
365 let json = serde_json::to_string_pretty(&pricing).expect("serialize");
366 let back: OfferingPricing = serde_json::from_str(&json).expect("round-trip");
367 assert_eq!(pricing, back);
368
369 let lower = json.to_lowercase();
370 for needle in [
371 "api_key",
372 "apikey",
373 "authorization",
374 "secret",
375 "password",
376 "bearer",
377 "access_token",
378 ] {
379 assert!(!lower.contains(needle), "pricing JSON contains `{needle}`");
380 }
381 }
382
383 #[test]
384 fn staleness_applies_to_live_rows_only() {
385 let live = CatalogSource::Live {
386 base_url_fingerprint: "fp".into(),
387 fetched_at: 1_000,
388 };
389 let live_price = OfferingPricing::from_catalog_offering(&priced(live)).unwrap();
390 assert!(!live_price.is_stale(1_500, 3_600), "within TTL");
391 assert!(live_price.is_stale(5_000, 3_600), "past TTL");
392
393 // A bundled price has no fetch clock and is not age-stale.
394 let bundled = OfferingPricing::from_catalog_offering(&priced(CatalogSource::Bundled)).unwrap();
395 assert!(!bundled.is_stale(u64::MAX, 1));
396 }
397
398 #[test]
399 fn pricing_flows_from_the_models_dev_parser() {
400 let raw = r#"{
401 "providers": {
402 "zai": {
403 "models": {
404 "glm-5.2": {
405 "id": "glm-5.2",
406 "modalities": { "input": ["text"], "output": ["text"] },
407 "cost": { "input": 1.4, "output": 4.4, "cache_read": 0.26 }
408 }
409 }
410 }
411 }
412 }"#;
413 let catalog = ModelsDevCatalog::parse_json(raw).expect("fixture parses");
414 let rows = bundled_offerings_from_models_dev(&catalog);
415 let pricing = OfferingPricing::from_catalog_offering(&rows[0]).expect("zai glm-5.2 is priced");
416
417 assert_eq!(pricing.provider, "zai");
418 assert_eq!(pricing.wire_model_id, "glm-5.2");
419 assert_eq!(pricing.input_per_million, Some(1.4));
420 assert_eq!(pricing.output_per_million, Some(4.4));
421 assert_eq!(pricing.cache_read_per_million, Some(0.26));
422 assert_eq!(pricing.provenance, PricingProvenance::ModelsDevBundled);
423 }
424
425 #[test]
426 fn cache_only_offering_is_unknown_at_the_route_layer() {
427 // Priced only on cache classes (no input/output): the route Token badge
428 // would have no visible rates, so route_pricing_sku degrades to
429 // UnknownOrStale — yet the cache rate is still usable for estimate_cost.
430 let mut offering = priced(CatalogSource::Bundled);
431 offering.cost = Some(ModelsDevCost {
432 input: None,
433 output: None,
434 cache_read: Some(0.028),
435 cache_write: None,
436 });
437
438 assert!(matches!(
439 route_pricing_sku(&offering),
440 PricingSku::UnknownOrStale
441 ));
442
443 let pricing =
444 OfferingPricing::from_catalog_offering(&offering).expect("cache-only row is still priced");
445 assert!(pricing.has_any_price());
446 let usage = TokenUsage {
447 cache_read: 1_000_000,
448 ..Default::default()
449 };
450 assert_eq!(pricing.estimate_cost(&usage), Some(0.028));
451 }
452
453 #[test]
454 fn user_override_source_maps_through_from_catalog_offering() {
455 // Exercises provenance_from_source / effective_at_from_source for the
456 // override arm via the hydration path (not direct construction).
457 let pricing = OfferingPricing::from_catalog_offering(&priced(CatalogSource::UserOverride))
458 .expect("priced");
459 assert_eq!(pricing.provenance, PricingProvenance::UserOverride);
460 assert_eq!(pricing.effective_at, None);
461 }
462
463 #[test]
464 fn staleness_is_inclusive_at_the_ttl_boundary() {
465 let live = CatalogSource::Live {
466 base_url_fingerprint: "fp".into(),
467 fetched_at: 1_000,
468 };
469 let p = OfferingPricing::from_catalog_offering(&priced(live)).unwrap();
470 // age == max_age_secs counts as stale (`>=` semantics)...
471 assert!(p.is_stale(1_100, 100));
472 // ...one second younger is still fresh.
473 assert!(!p.is_stale(1_099, 100));
474 }
475
476 #[test]
477 fn unpriced_used_classes_names_exactly_what_makes_an_estimate_fail_closed() {
478 // The DeepSeek-shaped row publishes input/output/cache-read but no
479 // cache-write rate.
480 let pricing = OfferingPricing::from_catalog_offering(&priced(CatalogSource::Bundled))
481 .expect("priced row");
482
483 // A turn that never wrote to cache is fully priced.
484 let no_write = TokenUsage {
485 input: 1_000_000,
486 output: 1_000_000,
487 cache_read: 1_000_000,
488 cache_write: 0,
489 };
490 assert!(pricing.unpriced_used_classes(&no_write).is_empty());
491 assert_eq!(pricing.estimate_cost(&no_write), Some(0.28 + 0.42 + 0.028));
492
493 // The moment cache-write tokens appear, the estimate fails closed and the
494 // audit names the single class responsible.
495 let with_write = TokenUsage {
496 cache_write: 1,
497 ..no_write
498 };
499 assert_eq!(
500 pricing.unpriced_used_classes(&with_write),
501 vec![TokenClass::CacheWrite]
502 );
503 assert_eq!(pricing.estimate_cost(&with_write), None);
504
505 // Zero-token classes never count as unpriced.
506 let empty = TokenUsage::default();
507 assert!(pricing.unpriced_used_classes(&empty).is_empty());
508 assert_eq!(pricing.estimate_cost(&empty), Some(0.0));
509 }
510
511 #[test]
512 fn token_class_labels_and_counts_stay_aligned_with_token_usage() {
513 let usage = TokenUsage {
514 input: 1,
515 output: 2,
516 cache_read: 3,
517 cache_write: 4,
518 };
519 let seen: Vec<(&str, u64)> = TokenClass::ALL
520 .into_iter()
521 .map(|class| (class.label(), class.tokens(&usage)))
522 .collect();
523 assert_eq!(
524 seen,
525 vec![
526 ("input", 1),
527 ("output", 2),
528 ("cache_read", 3),
529 ("cache_write", 4),
530 ]
531 );
532 }
533
533 lines RUST