scorecard.rs - CodeWhale

scorecard.rs

1	//! Token / cache / cost scorecard (#3388).
2	//!
3	//! A release-gate view of an agent run's token economics: per-turn input /
4	//! output / cache-read tokens and cost, aggregate totals + cache-hit ratio, and
5	//! regression detection against a committed baseline. This is the measurement
6	//! layer the "token, cache, and context discipline" EPIC asks for — it makes a
7	//! cost/token regression visible instead of silently shipping.
8	//!
9	//! The core here is pure and offline: it turns already-recorded per-turn
10	//! [`Usage`] (captured on every turn, persisted in `TurnRecord`) into a
11	//! scorecard, reusing the existing pricing layer rather than reinventing cost
12	//! math. The `scorecard` subcommand is a thin I/O wrapper over this module.
13
14	use chrono::{DateTime, Utc};
15	use serde::{Deserialize, Serialize};
16
17	use crate::config::ApiProvider;
18	#[cfg(test)]
19	use crate::config::{DEEPSEEK_ALIAS_REPLACEMENT, DEEPSEEK_ALIAS_RETIREMENT_UTC};
20	use crate::pricing::{
21	CostEstimate, TurnCostAudit, audit_turn_cost_for_route_at, token_usage_for_pricing,
22	};
23	use codewhale_models::Usage;
24
25	/// One turn's normalized token economics.
26	#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
27	pub struct TurnScore {
28	pub turn_id: String,
29	/// Timestamp used for historical/time-window pricing. `None` means the
30	/// recorder did not preserve when the turn occurred.
31	#[serde(default, skip_serializing_if = "Option::is_none")]
32	pub created_at: Option<DateTime<Utc>>,
33	/// Effective provider recorded for this turn. `None` means legacy or
34	/// otherwise unknown provenance, so cost must remain unpriced.
35	#[serde(default)]
36	pub provider: Option<String>,
37	/// Non-secret discriminator when one provider/model pair spans multiple
38	/// billing systems. Missing provenance keeps ambiguous routes unpriced.
39	#[serde(default, skip_serializing_if = "Option::is_none")]
40	pub billing_surface: Option<String>,
41	pub model: String,
42	/// Non-cached (billable) input tokens.
43	pub input_tokens: u64,
44	/// Output tokens, including reasoning output.
45	pub output_tokens: u64,
46	/// Cache-read (cache-hit) input tokens.
47	pub cache_read_tokens: u64,
48	/// Cache-write (cache-creation) input tokens. Billed at a premium on the
49	/// providers that publish one, so it is audited as its own class rather
50	/// than folded into input. Defaults to 0 for legacy records.
51	#[serde(default)]
52	pub cache_write_tokens: u64,
53	/// Reasoning tokens reported for the turn. Informational only — every
54	/// provider counts these inside `output_tokens`, so adding them here would
55	/// double-bill. Kept so a reasoning-heavy run can still be inspected.
56	#[serde(default)]
57	pub reasoning_tokens: u64,
58	pub cost_usd: f64,
59	pub cost_cny: f64,
60	/// True when provider provenance is missing/unknown or no authoritative USD
61	/// pricing row exists: numeric cost stays 0 for compatibility, while this
62	/// flag prevents it from being represented as a real zero-dollar charge.
63	pub cost_unpriced: bool,
64	/// Same availability marker for CNY. Most catalog offerings publish only
65	/// USD, so their CNY value is unavailable rather than a real zero.
66	#[serde(default)]
67	pub cost_cny_unpriced: bool,
68	/// Why USD cost is unavailable, when it is (`no_pricing_row`,
69	/// `missing_class_price`, `not_money_metered`, …). `None` for priced turns.
70	#[serde(default, skip_serializing_if = "Option::is_none")]
71	pub cost_unpriced_reason: Option<String>,
72	/// Token classes this turn used that carry no published price. Non-empty
73	/// means the estimate failed closed on purpose.
74	#[serde(default, skip_serializing_if = "Vec::is_empty")]
75	pub unpriced_classes: Vec<String>,
76	/// Provenance of the pricing row that was applied or attempted
77	/// (`models_dev_bundled`, `provider_live`, `provider_docs`,
78	/// `user_override`). `None` when no row was found at all.
79	#[serde(default, skip_serializing_if = "Option::is_none")]
80	pub pricing_provenance: Option<String>,
81	/// Live-pricing downgrade receipt: the live catalog row for this route could
82	/// not be verified (stale, or fetched from a different endpoint), so the
83	/// bundled published rates were used. Present even on priced turns, because
84	/// it explains which row the number came from.
85	#[serde(default, skip_serializing_if = "Option::is_none")]
86	pub live_pricing_defect: Option<String>,
87	/// Whether this turn is inside the money-metered coverage denominator.
88	///
89	/// False only for routes exactly identified as non-metered. Serialized so a
90	/// re-read scorecard can reproduce the coverage split without re-deriving it
91	/// from `provider` + `billing_surface`, which are also preserved above.
92	#[serde(default)]
93	pub money_metered: bool,
94	}
95
96	/// Aggregate metrics for a run. Serializes/deserializes as the baseline file.
97	#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
98	pub struct ScorecardMetrics {
99	pub turns: usize,
100	/// Turns whose route meters money, or whose billing basis could not be
101	/// established. This — not `turns` — is the denominator the USD total is
102	/// meant to cover: a local or subscription turn owes no dollars, so counting
103	/// it would understate coverage, while an unknown-basis turn must stay in
104	/// (#4318). Defaults to zero so existing baseline JSON stays readable.
105	#[serde(default)]
106	pub money_metered_turns: usize,
107	/// Money-metered turns that could not be priced authoritatively in USD.
108	/// Defaults to zero so existing baseline JSON remains readable.
109	#[serde(default)]
110	pub unpriced_turns: usize,
111	/// Turns without authoritative CNY pricing.
112	#[serde(default)]
113	pub cny_unpriced_turns: usize,
114	/// Whether every turn contributed authoritative USD pricing. Legacy
115	/// baselines lack this field and therefore default to `false`, preventing
116	/// comparisons against totals that may have been inferred from model ids
117	/// alone.
118	#[serde(default)]
119	pub cost_complete: bool,
120	/// Whether every turn contributed authoritative CNY pricing.
121	#[serde(default)]
122	pub cny_cost_complete: bool,
123	/// Token classes used somewhere in the run that had no published price, in
124	/// stable order. Non-empty means `cost_complete` is false because of a
125	/// class-level pricing gap, not merely an unknown route.
126	#[serde(default)]
127	pub unpriced_classes: Vec<String>,
128	pub total_input_tokens: u64,
129	pub total_output_tokens: u64,
130	pub total_cache_read_tokens: u64,
131	/// Cache-write (cache-creation) tokens across the run. Defaults to zero so
132	/// existing baseline JSON stays readable.
133	#[serde(default)]
134	pub total_cache_write_tokens: u64,
135	/// Reasoning tokens across the run. Informational: already inside
136	/// `total_output_tokens`, never added to it.
137	#[serde(default)]
138	pub total_reasoning_tokens: u64,
139	pub total_cost_usd: f64,
140	pub total_cost_cny: f64,
141	/// `cache_read / (input + cache_read)`; `0.0` when there are no input
142	/// tokens. Higher is better (more of the prompt was served from cache).
143	pub cache_hit_ratio: f64,
144	}
145
146	/// A metric that grew beyond the allowed threshold versus the baseline.
147	#[derive(Debug, Clone, Serialize, PartialEq)]
148	pub struct Regression {
149	pub metric: String,
150	pub baseline: f64,
151	pub current: f64,
152	/// Percent increase over baseline. `f64::INFINITY` when baseline was 0.
153	pub pct_increase: f64,
154	}
155
156	fn cacheable_token_total(input: u64, cache_read: u64, cache_write: u64) -> u64 {
157	input.saturating_add(cache_read).saturating_add(cache_write)
158	}
159
160	/// Full scorecard: per-turn breakdown plus aggregates.
161	#[derive(Debug, Clone, Serialize)]
162	pub struct Scorecard {
163	pub per_turn: Vec<TurnScore>,
164	pub metrics: ScorecardMetrics,
165	}
166
167	/// One row of input to the scorecard: a turn id, the model that served it, and
168	/// the turn's recorded usage.
169	///
170	/// `billing_surface` is explicit and has no default. The scorecard has two
171	/// entry modes, and they must agree: if this fixture mode could silently supply
172	/// an official first-party surface, every scorecard test would be asserting
173	/// against a route classification that `from_recorded_turns` never invents, and
174	/// the fail-closed path would go unexercised in the mode the tests use.
175	#[cfg(test)]
176	pub struct TurnInput<'a> {
177	pub turn_id: String,
178	pub created_at: Option<&'a DateTime<Utc>>,
179	pub provider: Option<&'a str>,
180	/// The route's recorded billing surface, or `None` when the recording did
181	/// not establish one. `None` must price exactly as it does for a recorded
182	/// turn: unknown, never official.
183	pub billing_surface: Option<&'a str>,
184	pub model: String,
185	pub usage: &'a Usage,
186	}
187
188	#[derive(Debug, Clone, Copy)]
189	struct ScorecardTurnRef<'a> {
190	turn_id: &'a str,
191	created_at: Option<&'a DateTime<Utc>>,
192	provider: Option<&'a str>,
193	billing_surface: Option<&'a str>,
194	model: &'a str,
195	usage: &'a Usage,
196	}
197
198	/// A recorded turn as read from a scorecard input file (a JSON array of these).
199	/// The base shape matches the per-turn data a `TurnEnd` hook emits. Recorders
200	/// and persisted runtime exports can add `provider` / `effective_provider` plus
201	/// non-secret billing-surface provenance. Legacy model-only recordings remain
202	/// readable but deliberately unpriced.
203	#[derive(Debug, Clone, Deserialize)]
204	pub struct RecordedTurn {
205	#[serde(default, alias = "id")]
206	pub turn_id: String,
207	#[serde(default)]
208	pub created_at: Option<DateTime<Utc>>,
209	/// New `turn_end` hooks mark shell-only lifecycle records false so the
210	/// model-cost scorecard can ignore them. Missing stays compatible with
211	/// legacy hook rows and persisted runtime turns, which are model-backed.
212	#[serde(default)]
213	pub model_backed: Option<bool>,
214	#[serde(default, alias = "effective_provider")]
215	pub provider: Option<String>,
216	#[serde(default, alias = "effective_billing_surface")]
217	pub billing_surface: Option<String>,
218	#[serde(default, alias = "effective_model")]
219	pub model: String,
220	#[serde(default)]
221	pub usage: Option<Usage>,
222	}
223
224	impl RecordedTurn {
225	#[must_use]
226	pub fn contributes_to_scorecard(&self) -> bool {
227	self.model_backed.unwrap_or(true) && self.usage.is_some() && !self.model.trim().is_empty()
228	}
229	}
230
231	#[derive(Debug, Clone, Default)]
232	struct AvailableCost {
233	usd: Option<f64>,
234	cny: Option<f64>,
235	unpriced_reason: Option<String>,
236	unpriced_classes: Vec<String>,
237	provenance: Option<String>,
238	/// Live-pricing downgrade receipt, when the row used was a bundled fallback
239	/// for an unverifiable live row.
240	live_pricing_defect: Option<String>,
241	/// Whether this turn belongs in the money-metered coverage denominator.
242	/// False only for routes exactly identified as non-metered.
243	counts_toward_money_coverage: bool,
244	}
245
246	impl AvailableCost {
247	/// Legacy/unknown provenance: no route to price against at all.
248	///
249	/// This still counts toward money coverage. A recording whose provider text
250	/// CodeWhale cannot parse is a turn whose spend is unknown, not a turn that
251	/// cost nothing — excusing it would let a legacy input file report a complete
252	/// total (#4318).
253	fn unknown_route() -> Self {
254	Self {
255	unpriced_reason: Some("unknown_route".to_string()),
256	counts_toward_money_coverage: true,
257	..Self::default()
258	}
259	}
260
261	/// Fails closed with an explicit reason, still inside money coverage.
262	fn failed_closed(reason: &str) -> Self {
263	Self {
264	unpriced_reason: Some(reason.to_string()),
265	counts_toward_money_coverage: true,
266	..Self::default()
267	}
268	}
269
270	fn from_audit(audit: &TurnCostAudit) -> Self {
271	Self {
272	usd: audit
273	.estimate
274	.and_then(\|cost\| audit.usd_priced.then_some(cost.usd)),
275	cny: audit
276	.estimate
277	.and_then(\|cost\| audit.cny_priced.then_some(cost.cny)),
278	unpriced_reason: audit
279	.unpriced_reason
280	.map(\|reason\| reason.label().to_string()),
281	unpriced_classes: audit
282	.unpriced_classes
283	.iter()
284	.map(\|class\| class.label().to_string())
285	.collect(),
286	provenance: audit
287	.provenance
288	.as_ref()
289	.map(\|provenance\| provenance.label().to_string()),
290	live_pricing_defect: audit
291	.live_pricing_defect
292	.as_ref()
293	.map(\|defect\| defect.label().to_string()),
294	counts_toward_money_coverage: audit.counts_toward_money_coverage(),
295	}
296	}
297	}
298
299	fn provider_scoped_cost(
300	provider: ApiProvider,
301	model: &str,
302	usage: &Usage,
303	created_at: Option<&DateTime<Utc>>,
304	billing_surface: Option<&str>,
305	) -> AvailableCost {
306	// These provider identities are themselves exact billing provenance and
307	// override stale/junk recorded surfaces: they cannot become PAYG merely
308	// because an older recorder wrote a bogus endpoint classification.
309	let intrinsic_surface = match provider {
310	ApiProvider::Ollama \| ApiProvider::Sglang \| ApiProvider::Vllm => {
311	Some(crate::pricing::LOCAL_BILLING_SURFACE)
312	}
313	ApiProvider::OpenaiCodex \| ApiProvider::OpencodeGo => {
314	Some(crate::pricing::OAUTH_SUBSCRIPTION_BILLING_SURFACE)
315	}
316	_ => None,
317	};
318	let billing_surface = intrinsic_surface.or(billing_surface);
319	// Every provider that supports both PAYG and plan/OAuth routes needs the
320	// recorded surface to choose between them. A model id or provider name is
321	// not sufficient evidence in an offline scorecard.
322	if billing_surface.is_none()
323	&& matches!(
324	provider,
325	ApiProvider::Zai
326	\| ApiProvider::Moonshot
327	\| ApiProvider::Anthropic
328	\| ApiProvider::XiaomiMimo
329	\| ApiProvider::Xai
330	\| ApiProvider::Minimax
331	\| ApiProvider::MinimaxAnthropic
332	\| ApiProvider::Stepfun
333	\| ApiProvider::Custom
334	)
335	{
336	return AvailableCost::failed_closed("missing_billing_surface");
337	}
338	let direct_deepseek = matches!(
339	provider,
340	ApiProvider::Deepseek \| ApiProvider::DeepseekCN \| ApiProvider::DeepseekAnthropic
341	);
342	let normalized_model = model.trim();
343	let model_lower = normalized_model.to_ascii_lowercase();
344	// Every direct DeepSeek first-party rate is time-windowed now — the
345	// V4 flash/pro rows carry peak/off-peak tiers (01:00–04:00 and
346	// 06:00–10:00 UTC on weekdays, with the whole of a Beijing-time Saturday
347	// and Sunday billing off-peak from 2026-08-23), and the retired
348	// `deepseek-chat` / `deepseek-reasoner` aliases price through them — so an
349	// undated DeepSeek turn cannot be resolved to one price. The weekend is
350	// bounded in Beijing time, which is why the window it covers is not the
351	// one a UTC weekday would give. `claude-sonnet-5` keeps the same recorded-time
352	// contract it had during its introductory window (Anthropic later made
353	// that $2/$10 rate permanent; the row still prices at the turn's own time
354	// rather than the wall clock, and undated turns still fail closed).
355	let needs_recorded_time =
356	direct_deepseek \|\| (provider == ApiProvider::Anthropic && model_lower == "claude-sonnet-5");
357	let recorded_at = match (created_at, needs_recorded_time) {
358	(Some(recorded_at), _) => recorded_at.to_owned(),
359	// A time-windowed rate without a recorded time cannot be resolved to a
360	// single price; fail closed rather than guess a window.
361	(None, true) => return AvailableCost::failed_closed("missing_recorded_time"),
362	(None, false) => Utc::now(),
363	};
364
365	// The billing surface recorded with the turn is authoritative over any
366	// provider-level assumption, and it now covers every classification a route
367	// can carry — Z.ai Coding Plan, Kimi Code, MiniMax Token Plan, MiMo token
368	// plan, OAuth brokers, local runtimes, aggregators, first-party PAYG, and
369	// "unclassified" — not just StepFun's two surfaces (#4318).
370	match crate::pricing::endpoint_metering_for_billing_surface(billing_surface) {
371	// Exactly identified as non-metered: no dollar figure is owed, and the
372	// turn leaves the money-coverage denominator.
373	crate::pricing::EndpointMetering::ExactSubscription
374	\| crate::pricing::EndpointMetering::LocalNoBill => {
375	return AvailableCost {
376	unpriced_reason: Some(
377	crate::pricing::UnpricedReason::NotMoneyMetered
378	.label()
379	.to_string(),
380	),
381	counts_toward_money_coverage: false,
382	..AvailableCost::default()
383	};
384	}
385	// A recorded surface CodeWhale cannot place must not inherit the
386	// provider's default rates.
387	crate::pricing::EndpointMetering::Unknown if billing_surface.is_some() => {
388	return AvailableCost::failed_closed(
389	crate::pricing::UnpricedReason::UnknownBillingBasis.label(),
390	);
391	}
392	crate::pricing::EndpointMetering::Unknown \| crate::pricing::EndpointMetering::Money => {}
393	}
394
395	// The pricing layer owns the exact provider/model catalog gate, explicit
396	// first-party hand-price allowlist, cache-class completeness checks, and
397	// endpoint-derived billing surfaces. Keeping one route-aware path prevents
398	// the scorecard from drifting back to model-only pricing.
399	let audit = audit_turn_cost_for_route_at(
400	provider,
401	normalized_model,
402	billing_surface,
403	usage,
404	recorded_at,
405	);
406	AvailableCost::from_audit(&audit)
407	}
408
409	impl Scorecard {
410	/// Build a scorecard from recorded per-turn usage. Pure + offline; cost is
411	/// computed via the shared pricing layer (`None` pricing → unpriced, 0 cost).
412	#[must_use]
413	#[cfg(test)]
414	pub fn from_turns(turns: &[TurnInput<'_>]) -> Self {
415	Self::from_turn_refs(turns.iter().map(\|turn\| ScorecardTurnRef {
416	turn_id: &turn.turn_id,
417	created_at: turn.created_at,
418	provider: turn.provider,
419	billing_surface: turn.billing_surface,
420	model: &turn.model,
421	usage: turn.usage,
422	}))
423	}
424
425	/// Build directly from hook/runtime records, retaining billing provenance
426	/// while excluding explicitly non-model lifecycle rows.
427	#[must_use]
428	pub fn from_recorded_turns(turns: &[RecordedTurn]) -> Self {
429	Self::from_turn_refs(turns.iter().filter_map(\|turn\| {
430	if !turn.contributes_to_scorecard() {
431	return None;
432	}
433	let usage = turn.usage.as_ref()?;
434	Some(ScorecardTurnRef {
435	turn_id: &turn.turn_id,
436	created_at: turn.created_at.as_ref(),
437	provider: turn.provider.as_deref(),
438	billing_surface: turn.billing_surface.as_deref(),
439	model: &turn.model,
440	usage,
441	})
442	}))
443	}
444
445	fn from_turn_refs<'a>(turns: impl IntoIterator<Item = ScorecardTurnRef<'a>>) -> Self {
446	let turns = turns.into_iter();
447	let mut per_turn = Vec::with_capacity(turns.size_hint().0);
448	let mut metrics = ScorecardMetrics::default();
449	let mut unpriced_classes = std::collections::BTreeSet::new();
450
451	for turn in turns {
452	// Normalize provider usage into canonical billable classes once.
453	let classes = token_usage_for_pricing(turn.usage);
454	let provider = turn
455	.provider
456	.map(str::trim)
457	.filter(\|value\| !value.is_empty());
458	let cost = provider.and_then(ApiProvider::parse).map_or_else(
459	AvailableCost::unknown_route,
460	\|provider\| {
461	provider_scoped_cost(
462	provider,
463	turn.model,
464	turn.usage,
465	turn.created_at,
466	turn.billing_surface,
467	)
468	},
469	);
470	let cost_unpriced = cost.usd.is_none();
471	let cost_cny_unpriced = cost.cny.is_none();
472	let cost_usd = cost.usd.unwrap_or(0.0);
473	let cost_cny = cost.cny.unwrap_or(0.0);
474	let reasoning_tokens = u64::from(turn.usage.reasoning_tokens.unwrap_or(0));
475	unpriced_classes.extend(cost.unpriced_classes.iter().cloned());
476
477	metrics.turns = metrics.turns.saturating_add(1);
478	// Only money-metered turns can make a dollar total incomplete. A
479	// local or plan turn is not an unpriced dollar; an unknown one is.
480	if cost.counts_toward_money_coverage {
481	metrics.money_metered_turns = metrics.money_metered_turns.saturating_add(1);
482	metrics.unpriced_turns = metrics
483	.unpriced_turns
484	.saturating_add(usize::from(cost_unpriced));
485	metrics.cny_unpriced_turns = metrics
486	.cny_unpriced_turns
487	.saturating_add(usize::from(cost_cny_unpriced));
488	}
489	metrics.total_input_tokens = metrics.total_input_tokens.saturating_add(classes.input);
490	metrics.total_output_tokens =
491	metrics.total_output_tokens.saturating_add(classes.output);
492	metrics.total_cache_read_tokens = metrics
493	.total_cache_read_tokens
494	.saturating_add(classes.cache_read);
495	metrics.total_cache_write_tokens = metrics
496	.total_cache_write_tokens
497	.saturating_add(classes.cache_write);
498	metrics.total_reasoning_tokens = metrics
499	.total_reasoning_tokens
500	.saturating_add(reasoning_tokens);
501	metrics.total_cost_usd = CostEstimate::usd_only(metrics.total_cost_usd)
502	.saturating_add(CostEstimate::usd_only(cost_usd))
503	.usd;
504	metrics.total_cost_cny = CostEstimate {
505	usd: 0.0,
506	cny: metrics.total_cost_cny,
507	}
508	.saturating_add(CostEstimate {
509	usd: 0.0,
510	cny: cost_cny,
511	})
512	.cny;
513
514	per_turn.push(TurnScore {
515	turn_id: turn.turn_id.to_string(),
516	created_at: turn.created_at.cloned(),
517	provider: provider.map(str::to_string),
518	billing_surface: turn.billing_surface.map(str::to_string),
519	model: turn.model.to_string(),
520	input_tokens: classes.input,
521	output_tokens: classes.output,
522	cache_read_tokens: classes.cache_read,
523	cache_write_tokens: classes.cache_write,
524	reasoning_tokens,
525	cost_usd,
526	cost_cny,
527	cost_unpriced,
528	cost_cny_unpriced,
529	cost_unpriced_reason: cost.unpriced_reason,
530	unpriced_classes: cost.unpriced_classes,
531	pricing_provenance: cost.provenance,
532	live_pricing_defect: cost.live_pricing_defect,
533	money_metered: cost.counts_toward_money_coverage,
534	});
535	}
536	metrics.unpriced_classes = unpriced_classes.into_iter().collect();
537
538	// Canonical denominator: hit / (non-cached input + hit + write).
539	// `total_input_tokens` here is already the non-cached input, because
540	// `token_usage_for_pricing` splits hits and writes out of the reported
541	// prompt total. Cache-write tokens were previously missing from the
542	// denominator, which reported a better hit ratio on precisely the turns
543	// that paid to populate the cache (#4318). Write stays a separate
544	// reported total so the premium is not hidden inside the ratio.
545	let cacheable = cacheable_token_total(
546	metrics.total_input_tokens,
547	metrics.total_cache_read_tokens,
548	metrics.total_cache_write_tokens,
549	);
550	metrics.cache_hit_ratio = if cacheable > 0 {
551	metrics.total_cache_read_tokens as f64 / cacheable as f64
552	} else {
553	0.0
554	};
555	metrics.cost_complete = metrics.unpriced_turns == 0;
556	metrics.cny_cost_complete = metrics.cny_unpriced_turns == 0;
557
558	Self { per_turn, metrics }
559	}
560
561	/// Render a compact human-readable summary (used for non-JSON output).
562	#[must_use]
563	pub fn to_summary(&self) -> String {
564	let m = &self.metrics;
565	let mut out = String::new();
566	out.push_str("Token / cache / cost scorecard\n");
567	out.push_str(&format!(
568	"turns: {} money_metered_turns: {}\n",
569	m.turns, m.money_metered_turns
570	));
571	out.push_str(&format!(
572	"input_tokens: {} output_tokens: {} cache_read_tokens: {} cache_write_tokens: {}\n",
573	m.total_input_tokens,
574	m.total_output_tokens,
575	m.total_cache_read_tokens,
576	m.total_cache_write_tokens
577	));
578	out.push_str(&format!(
579	"reasoning_tokens: {} (informational; already inside output_tokens)\n",
580	m.total_reasoning_tokens
581	));
582	out.push_str(&format!(
583	"cache_hit_ratio: {:.1}%\n",
584	m.cache_hit_ratio * 100.0
585	));
586	append_currency_summary(
587	&mut out,
588	"cost_usd",
589	"priced_cost_subtotal_usd",
590	"$",
591	m.total_cost_usd,
592	m.unpriced_turns,
593	// Coverage is reported against the money-metered turns, not every
594	// turn: a local or plan turn owes no dollars, so including it in the
595	// denominator would understate how complete the figure is.
596	m.money_metered_turns,
597	);
598	append_currency_summary(
599	&mut out,
600	"cost_cny",
601	"priced_cost_subtotal_cny",
602	"¥",
603	m.total_cost_cny,
604	m.cny_unpriced_turns,
605	m.money_metered_turns,
606	);
607	if m.unpriced_turns > 0 {
608	out.push_str(&format!(
609	"note: {} turn(s) had missing/unknown provider provenance or no authoritative USD pricing row; their USD cost is unavailable and excluded.\n",
610	m.unpriced_turns
611	));
612	}
613	if m.cny_unpriced_turns > 0 {
614	out.push_str(&format!(
615	"note: {} turn(s) had no authoritative CNY pricing row; their CNY cost is unavailable and excluded.\n",
616	m.cny_unpriced_turns
617	));
618	}
619	if !m.unpriced_classes.is_empty() {
620	out.push_str(&format!(
621	"note: token class(es) with no published price on a used route: {}. Those turns fail closed rather than under-report.\n",
622	m.unpriced_classes.join(", ")
623	));
624	}
625	out
626	}
627	}
628
629	fn append_currency_summary(
630	out: &mut String,
631	complete_label: &str,
632	subtotal_label: &str,
633	symbol: &str,
634	total: f64,
635	unpriced_turns: usize,
636	turns: usize,
637	) {
638	if unpriced_turns == 0 {
639	out.push_str(&format!("{complete_label}: {symbol}{total:.4}\n"));
640	} else if unpriced_turns == turns {
641	out.push_str(&format!("{complete_label}: unavailable\n"));
642	} else {
643	out.push_str(&format!("{subtotal_label}: {symbol}{total:.4}\n"));
644	}
645	}
646
647	impl ScorecardMetrics {
648	/// Flag metrics that grew more than `threshold_pct` over `baseline`. Cost
649	/// and token counts are "lower is better", so only increases are
650	/// regressions. (Cache-hit ratio is the opposite, reported separately.)
651	#[must_use]
652	pub fn regressions_against(
653	&self,
654	baseline: &ScorecardMetrics,
655	threshold_pct: f64,
656	) -> Vec<Regression> {
657	let mut out = Vec::new();
658	// A partial/unknown subtotal is not comparable to a complete baseline,
659	// but losing completeness is itself a regression. Otherwise removing
660	// provider provenance could turn real spend into a smaller subtotal
661	// and silently bypass the release gate.
662	if baseline.cost_complete && !self.cost_complete {
663	out.push(Regression {
664	metric: "cost_completeness_drop".to_string(),
665	baseline: 1.0,
666	current: 0.0,
667	pct_increase: 100.0,
668	});
669	} else if self.cost_complete && baseline.cost_complete {
670	push_regression(
671	&mut out,
672	"total_cost_usd",
673	baseline.total_cost_usd,
674	self.total_cost_usd,
675	threshold_pct,
676	);
677	}
678	if baseline.cny_cost_complete && !self.cny_cost_complete {
679	out.push(Regression {
680	metric: "cny_cost_completeness_drop".to_string(),
681	baseline: 1.0,
682	current: 0.0,
683	pct_increase: 100.0,
684	});
685	} else if self.cny_cost_complete && baseline.cny_cost_complete {
686	push_regression(
687	&mut out,
688	"total_cost_cny",
689	baseline.total_cost_cny,
690	self.total_cost_cny,
691	threshold_pct,
692	);
693	}
694	push_regression(
695	&mut out,
696	"total_input_tokens",
697	baseline.total_input_tokens as f64,
698	self.total_input_tokens as f64,
699	threshold_pct,
700	);
701	push_regression(
702	&mut out,
703	"total_output_tokens",
704	baseline.total_output_tokens as f64,
705	self.total_output_tokens as f64,
706	threshold_pct,
707	);
708	// Cache-hit ratio regresses when it drops; express the drop as a
709	// positive percentage so it reads like the others.
710	if baseline.cache_hit_ratio > 0.0 {
711	let drop_pct = (baseline.cache_hit_ratio - self.cache_hit_ratio)
712	/ baseline.cache_hit_ratio
713	* 100.0;
714	if drop_pct > threshold_pct {
715	out.push(Regression {
716	metric: "cache_hit_ratio_drop".to_string(),
717	baseline: baseline.cache_hit_ratio,
718	current: self.cache_hit_ratio,
719	pct_increase: drop_pct,
720	});
721	}
722	}
723	out
724	}
725	}
726
727	fn push_regression(
728	out: &mut Vec<Regression>,
729	metric: &str,
730	base: f64,
731	cur: f64,
732	threshold_pct: f64,
733	) {
734	if base > 0.0 {
735	let pct = (cur - base) / base * 100.0;
736	if pct > threshold_pct {
737	out.push(Regression {
738	metric: metric.to_string(),
739	baseline: base,
740	current: cur,
741	pct_increase: pct,
742	});
743	}
744	} else if cur > 0.0 {
745	out.push(Regression {
746	metric: metric.to_string(),
747	baseline: base,
748	current: cur,
749	pct_increase: f64::INFINITY,
750	});
751	}
752	}
753
754	#[cfg(test)]
755	mod tests {
756	use super::*;
757
758	fn usage(input: u32, output: u32, cache_hit: u32) -> Usage {
759	Usage {
760	input_tokens: input,
761	output_tokens: output,
762	prompt_cache_hit_tokens: Some(cache_hit),
763	..Default::default()
764	}
765	}
766
767	/// The scorecard has two entry modes and they must classify identically.
768	///
769	/// `from_turns` is the fixture mode used by this test module;
770	/// `from_recorded_turns` is the mode that reads real recordings. The
771	/// fixture mode used to inject `first-party-payg` for every row, which
772	/// meant the whole suite was asserting against a route classification the
773	/// real mode never produces — the fail-closed path was untested precisely
774	/// where it mattered. Neither mode may invent an official surface.
775	#[test]
776	fn both_scorecard_entry_modes_agree_on_an_unestablished_billing_surface() {
777	let sample = usage(10_000, 1_000, 0);
778
779	let fixture = Scorecard::from_turns(&[TurnInput {
780	turn_id: "t1".into(),
781	created_at: None,
782	provider: Some("anthropic"),
783	billing_surface: None,
784	model: "claude-haiku-4-5".into(),
785	usage: &sample,
786	}]);
787	let recorded = Scorecard::from_recorded_turns(&[RecordedTurn {
788	turn_id: "t1".to_string(),
789	created_at: None,
790	model_backed: None,
791	provider: Some("anthropic".to_string()),
792	billing_surface: None,
793	model: "claude-haiku-4-5".to_string(),
794	usage: Some(sample.clone()),
795	}]);
796
797	assert_eq!(fixture.per_turn, recorded.per_turn);
798	assert_eq!(fixture.metrics, recorded.metrics);
799	assert!(
800	fixture.per_turn[0].cost_unpriced,
801	"a route with no established surface must not be priced"
802	);
803	assert_eq!(fixture.per_turn[0].cost_usd, 0.0);
804	assert!(!fixture.metrics.cost_complete);
805	assert_eq!(fixture.metrics.money_metered_turns, 1);
806	assert_eq!(fixture.metrics.unpriced_turns, 1);
807	let summary = fixture.to_summary();
808	assert!(summary.contains("cost_usd: unavailable"), "{summary}");
809	assert!(summary.contains("cost_cny: unavailable"), "{summary}");
810	assert!(
811	!summary.contains('$') && !summary.contains('¥'),
812	"an unpriced-only run must name no amount at all: {summary}"
813	);
814
815	// With the surface actually established, both modes price it — and
816	// still agree.
817	let priced_fixture = Scorecard::from_turns(&[TurnInput {
818	turn_id: "t1".into(),
819	created_at: None,
820	provider: Some("anthropic"),
821	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
822	model: "claude-haiku-4-5".into(),
823	usage: &sample,
824	}]);
825	let priced_recorded = Scorecard::from_recorded_turns(&[RecordedTurn {
826	turn_id: "t1".to_string(),
827	created_at: None,
828	model_backed: None,
829	provider: Some("anthropic".to_string()),
830	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE.to_string()),
831	model: "claude-haiku-4-5".to_string(),
832	usage: Some(sample),
833	}]);
834	assert_eq!(priced_fixture.per_turn, priced_recorded.per_turn);
835	assert!(!priced_fixture.per_turn[0].cost_unpriced);
836	assert!(priced_fixture.metrics.cost_complete);
837	}
838
839	#[test]
840	fn dual_mode_routes_require_surface_but_intrinsic_routes_override_junk() {
841	let usage = usage(10_000, 1_000, 0);
842	for (provider, model) in [
843	(ApiProvider::Anthropic, "claude-haiku-4-5"),
844	(ApiProvider::Moonshot, "kimi-k2.7-code"),
845	(ApiProvider::Zai, "glm-5.2"),
846	(ApiProvider::Minimax, "minimax-m3"),
847	] {
848	let cost = provider_scoped_cost(provider, model, &usage, None, None);
849	assert_eq!(
850	cost.unpriced_reason.as_deref(),
851	Some("missing_billing_surface"),
852	"{provider:?}"
853	);
854	assert!(cost.usd.is_none(), "{provider:?}");
855	}
856
857	for provider in [ApiProvider::OpenaiCodex, ApiProvider::OpencodeGo] {
858	let cost = provider_scoped_cost(
859	provider,
860	"gpt-5.5",
861	&usage,
862	None,
863	Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
864	);
865	assert_eq!(cost.unpriced_reason.as_deref(), Some("not_money_metered"));
866	assert!(!cost.counts_toward_money_coverage);
867	}
868	let local = provider_scoped_cost(
869	ApiProvider::Ollama,
870	"llama3.2",
871	&usage,
872	None,
873	Some(crate::pricing::UNCLASSIFIED_BILLING_SURFACE),
874	);
875	assert_eq!(local.unpriced_reason.as_deref(), Some("not_money_metered"));
876	assert!(!local.counts_toward_money_coverage);
877
878	let cloud = provider_scoped_cost(
879	ApiProvider::OllamaCloud,
880	crate::config::DEFAULT_OLLAMA_CLOUD_MODEL,
881	&usage,
882	None,
883	Some(crate::pricing::UNCLASSIFIED_BILLING_SURFACE),
884	);
885	assert_eq!(
886	cloud.unpriced_reason.as_deref(),
887	Some("unknown_billing_basis")
888	);
889	assert!(
890	cloud.counts_toward_money_coverage,
891	"hosted Cloud usage must never disappear as local/free"
892	);
893	}
894
895	fn cache_write_usage(input: u32, output: u32, cache_hit: u32, cache_write: u32) -> Usage {
896	Usage {
897	input_tokens: input,
898	output_tokens: output,
899	prompt_cache_hit_tokens: Some(cache_hit),
900	prompt_cache_write_tokens: Some(cache_write),
901	reasoning_tokens: Some(output / 2),
902	..Default::default()
903	}
904	}
905
906	/// A mixed-route run: one fully-priced cache-write turn, one turn whose
907	/// route publishes no cache-write rate, and one non-metered OAuth turn.
908	/// The priced subtotal stays honest, `cost_complete` fails closed, and the
909	/// audit names the class and provenance behind each gap.
910	#[test]
911	fn mixed_route_run_audits_cache_write_classes_and_fails_closed() {
912	// 1M input of which 200k is a cache read, 100k is a cache write.
913	let priced = cache_write_usage(1_000_000, 100_000, 200_000, 100_000);
914	let unpriced_write = cache_write_usage(1_000_000, 100_000, 200_000, 100_000);
915	let oauth = cache_write_usage(1_000_000, 100_000, 200_000, 100_000);
916	let turns = [
917	TurnInput {
918	turn_id: "anthropic".into(),
919	created_at: None,
920	provider: Some("anthropic"),
921	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
922	model: "claude-haiku-4-5".into(),
923	usage: &priced,
924	},
925	TurnInput {
926	turn_id: "moonshot".into(),
927	created_at: None,
928	provider: Some("moonshot"),
929	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
930	model: "kimi-k2.7-code".into(),
931	usage: &unpriced_write,
932	},
933	TurnInput {
934	turn_id: "oauth".into(),
935	created_at: None,
936	provider: Some("openai-codex"),
937	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
938	model: "gpt-5.5".into(),
939	usage: &oauth,
940	},
941	];
942
943	let card = Scorecard::from_turns(&turns);
944
945	// Cache-write tokens are their own audited class on every turn.
946	for turn in &card.per_turn {
947	assert_eq!(turn.cache_write_tokens, 100_000, "{}", turn.turn_id);
948	assert_eq!(turn.input_tokens, 700_000, "{}", turn.turn_id);
949	assert_eq!(turn.cache_read_tokens, 200_000, "{}", turn.turn_id);
950	// Reasoning stays informational: never added to billable output.
951	assert_eq!(turn.output_tokens, 100_000, "{}", turn.turn_id);
952	assert_eq!(turn.reasoning_tokens, 50_000, "{}", turn.turn_id);
953	}
954	assert_eq!(card.metrics.total_cache_write_tokens, 300_000);
955	assert_eq!(card.metrics.total_output_tokens, 300_000);
956	assert_eq!(card.metrics.total_reasoning_tokens, 150_000);
957
958	// Anthropic publishes a 1.25/M cache-write rate, so the write premium
959	// is billed rather than silently charged at the input rate.
960	let anthropic = &card.per_turn[0];
961	assert!(!anthropic.cost_unpriced);
962	assert_eq!(anthropic.cost_unpriced_reason, None);
963	assert!(anthropic.unpriced_classes.is_empty());
964	// Provenance is recorded (bundled snapshot offline, live after a
965	// catalog refresh); the point is that it is never absent for a
966	// priced turn.
967	assert!(anthropic.pricing_provenance.is_some());
968	let expected = 0.7 * 1.0 + 0.1 * 5.0 + 0.2 * 0.1 + 0.1 * 1.25;
969	assert!((anthropic.cost_usd - expected).abs() < 1e-9);
970
971	// Moonshot's row has no published cache-write rate: the whole turn
972	// fails closed instead of under-reporting the write tokens.
973	let moonshot = &card.per_turn[1];
974	assert!(moonshot.cost_unpriced);
975	assert_eq!(moonshot.cost_usd, 0.0);
976	assert_eq!(
977	moonshot.cost_unpriced_reason.as_deref(),
978	Some("missing_class_price")
979	);
980	assert_eq!(moonshot.unpriced_classes, vec!["cache_write".to_string()]);
981	assert!(moonshot.pricing_provenance.is_some());
982
983	// A subscription route is not "free"; it is not money-metered.
984	let oauth = &card.per_turn[2];
985	assert!(oauth.cost_unpriced);
986	assert_eq!(
987	oauth.cost_unpriced_reason.as_deref(),
988	Some("not_money_metered")
989	);
990	assert!(oauth.unpriced_classes.is_empty());
991	assert_eq!(oauth.pricing_provenance, None);
992
993	// Aggregates stay honest about what the number covers. Two of the three
994	// turns owe money (Anthropic and Moonshot); the OAuth turn does not, so
995	// it is outside the denominator rather than counted as an unpriced dollar.
996	assert_eq!(card.metrics.turns, 3);
997	assert_eq!(card.metrics.money_metered_turns, 2);
998	assert_eq!(card.metrics.unpriced_turns, 1);
999	assert!(!card.metrics.cost_complete);
1000	assert_eq!(
1001	card.metrics.unpriced_classes,
1002	vec!["cache_write".to_string()]
1003	);
1004	assert!((card.metrics.total_cost_usd - expected).abs() < 1e-9);
1005	assert!(card.per_turn[0].money_metered);
1006	assert!(card.per_turn[1].money_metered);
1007	assert!(!card.per_turn[2].money_metered);
1008
1009	let summary = card.to_summary();
1010	assert!(summary.contains("priced_cost_subtotal_usd"));
1011	assert!(summary.contains("cache_write_tokens: 300000"));
1012	assert!(summary.contains("no published price"));
1013	// Coverage reads against the money-metered turns, not all three.
1014	assert!(summary.contains("money_metered_turns: 2"), "{summary}");
1015
1016	let json = serde_json::to_value(&card).expect("serialize scorecard");
1017	assert_eq!(json["per_turn"][1]["unpriced_classes"][0], "cache_write");
1018	assert_eq!(json["metrics"]["total_cache_write_tokens"], 300_000);
1019	assert_eq!(json["metrics"]["cost_complete"], false);
1020	assert_eq!(json["metrics"]["money_metered_turns"], 2);
1021	// Route identity survives serialization, so a re-read scorecard can be
1022	// re-explained without the original input file.
1023	assert_eq!(json["per_turn"][1]["provider"], "moonshot");
1024	assert_eq!(json["per_turn"][2]["money_metered"], false);
1025	}
1026
1027	/// Legacy baselines and per-turn records that predate the class audit must
1028	/// still deserialize; the new fields default rather than fail.
1029	#[test]
1030	fn legacy_turn_score_json_defaults_the_new_audit_fields() {
1031	let score: TurnScore = serde_json::from_value(serde_json::json!({
1032	"turn_id": "t1",
1033	"model": "gpt-5.5",
1034	"input_tokens": 10,
1035	"output_tokens": 5,
1036	"cache_read_tokens": 0,
1037	"cost_usd": 0.1,
1038	"cost_cny": 0.0,
1039	"cost_unpriced": false
1040	}))
1041	.expect("legacy per-turn record stays readable");
1042	assert_eq!(score.cache_write_tokens, 0);
1043	assert_eq!(score.reasoning_tokens, 0);
1044	assert!(score.unpriced_classes.is_empty());
1045	assert_eq!(score.pricing_provenance, None);
1046	assert_eq!(score.cost_unpriced_reason, None);
1047	assert_eq!(score.live_pricing_defect, None);
1048	// A legacy row carries no coverage evidence, so `money_metered` defaults
1049	// to false rather than asserting the row was inside a complete total.
1050	assert!(!score.money_metered);
1051
1052	// Legacy aggregate baselines stay readable too, defaulting the new
1053	// coverage denominator rather than failing the parse.
1054	let metrics: ScorecardMetrics = serde_json::from_value(serde_json::json!({
1055	"turns": 3,
1056	"total_input_tokens": 10,
1057	"total_output_tokens": 5,
1058	"total_cache_read_tokens": 0,
1059	"total_cost_usd": 0.1,
1060	"total_cost_cny": 0.0,
1061	"cache_hit_ratio": 0.0
1062	}))
1063	.expect("legacy baseline stays readable");
1064	assert_eq!(metrics.money_metered_turns, 0);
1065	assert_eq!(metrics.total_cache_write_tokens, 0);
1066	}
1067
1068	#[test]
1069	fn aggregates_tokens_and_cache_hit_ratio_independent_of_pricing() {
1070	// input_tokens includes cache hits; token_usage_for_pricing splits them:
1071	// non-cached input = 1000-200 = 800, cache_read = 200.
1072	let u1 = usage(1000, 500, 200);
1073	let u2 = usage(2000, 100, 800); // non-cached = 1200, cache_read = 800
1074	let turns = [
1075	TurnInput {
1076	turn_id: "t1".into(),
1077	created_at: None,
1078	provider: None,
1079	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1080	model: "unpriced-x".into(),
1081	usage: &u1,
1082	},
1083	TurnInput {
1084	turn_id: "t2".into(),
1085	created_at: None,
1086	provider: None,
1087	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1088	model: "unpriced-x".into(),
1089	usage: &u2,
1090	},
1091	];
1092	let card = Scorecard::from_turns(&turns);
1093
1094	assert_eq!(card.metrics.turns, 2);
1095	assert_eq!(card.metrics.total_input_tokens, 800 + 1200);
1096	assert_eq!(card.metrics.total_output_tokens, 600); // 500 + 100
1097	assert_eq!(card.metrics.total_cache_read_tokens, 1000); // 200 + 800
1098	assert_eq!(card.metrics.unpriced_turns, 2);
1099	// cache_read / (input + cache_read) = 1000 / (2000 + 1000)
1100	let expected = 1000.0 / 3000.0;
1101	assert!((card.metrics.cache_hit_ratio - expected).abs() < 1e-9);
1102	}
1103
1104	/// The canonical cache-efficiency denominator is
1105	/// `hit / (non-cached input + hit + write)`. Cache-write tokens are prompt
1106	/// tokens that were not served from cache, so omitting them reported a
1107	/// flattering ratio on exactly the turns that paid to populate the cache.
1108	#[test]
1109	fn cache_hit_ratio_counts_cache_write_in_the_denominator() {
1110	fn card_for(usage: &Usage) -> Scorecard {
1111	Scorecard::from_turns(&[TurnInput {
1112	turn_id: "t1".into(),
1113	created_at: None,
1114	provider: None,
1115	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1116	model: "unpriced-x".into(),
1117	usage,
1118	}])
1119	}
1120
1121	// Zero everything: a ratio is undefined, reported as 0.0 rather than NaN.
1122	let empty = Usage::default();
1123	let card = card_for(&empty);
1124	assert_eq!(card.metrics.cache_hit_ratio, 0.0);
1125	assert_eq!(card.metrics.total_cache_write_tokens, 0);
1126
1127	// Write-only: a turn that populated the cache and read nothing from it
1128	// has a 0% hit ratio, not an undefined-and-therefore-zero one that a
1129	// write-blind denominator would produce by accident.
1130	let write_only = Usage {
1131	input_tokens: 1_000,
1132	output_tokens: 10,
1133	prompt_cache_hit_tokens: Some(0),
1134	prompt_cache_write_tokens: Some(1_000),
1135	..Default::default()
1136	};
1137	let card = card_for(&write_only);
1138	assert_eq!(card.metrics.total_cache_write_tokens, 1_000);
1139	assert_eq!(card.metrics.total_cache_read_tokens, 0);
1140	assert_eq!(card.metrics.cache_hit_ratio, 0.0);
1141
1142	// Mixed: 1000 prompt tokens = 200 read + 300 write + 500 non-cached.
1143	let mixed = Usage {
1144	input_tokens: 1_000,
1145	output_tokens: 10,
1146	prompt_cache_hit_tokens: Some(200),
1147	prompt_cache_write_tokens: Some(300),
1148	..Default::default()
1149	};
1150	let card = card_for(&mixed);
1151	assert_eq!(card.metrics.total_input_tokens, 500);
1152	assert_eq!(card.metrics.total_cache_read_tokens, 200);
1153	assert_eq!(card.metrics.total_cache_write_tokens, 300);
1154	let expected = 200.0 / (500.0 + 200.0 + 300.0);
1155	assert!(
1156	(card.metrics.cache_hit_ratio - expected).abs() < 1e-9,
1157	"got {}, want {expected}",
1158	card.metrics.cache_hit_ratio
1159	);
1160	// The write-blind denominator would have said 200/700 — assert the two
1161	// are distinguishable so a regression is unambiguous.
1162	let write_blind = 200.0 / 700.0;
1163	assert!((card.metrics.cache_hit_ratio - write_blind).abs() > 1e-6);
1164	}
1165
1166	#[test]
1167	fn unknown_model_is_marked_unpriced_with_zero_cost() {
1168	let u = usage(1000, 500, 0);
1169	let turns = [TurnInput {
1170	turn_id: "t1".into(),
1171	created_at: None,
1172	provider: Some("openai"),
1173	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1174	model: "definitely-not-a-real-model".into(),
1175	usage: &u,
1176	}];
1177	let card = Scorecard::from_turns(&turns);
1178	assert!(card.per_turn[0].cost_unpriced);
1179	assert_eq!(card.per_turn[0].cost_usd, 0.0);
1180	assert_eq!(card.metrics.total_cost_usd, 0.0);
1181	assert!(card.to_summary().contains("cost_usd: unavailable"));
1182	}
1183
1184	#[test]
1185	fn same_model_is_priced_only_for_its_authoritative_provider_route() {
1186	let u = usage(1000, 500, 0);
1187	let turns = [
1188	TurnInput {
1189	turn_id: "api".into(),
1190	created_at: None,
1191	provider: Some("openai"),
1192	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1193	model: "gpt-5.5".into(),
1194	usage: &u,
1195	},
1196	TurnInput {
1197	turn_id: "oauth".into(),
1198	created_at: None,
1199	provider: Some("openai-codex"),
1200	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1201	model: "gpt-5.5".into(),
1202	usage: &u,
1203	},
1204	TurnInput {
1205	turn_id: "local".into(),
1206	created_at: None,
1207	provider: Some("ollama"),
1208	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1209	model: "gpt-5.5".into(),
1210	usage: &u,
1211	},
1212	];
1213
1214	let card = Scorecard::from_turns(&turns);
1215
1216	assert!(!card.per_turn[0].cost_unpriced);
1217	assert!(card.per_turn[0].cost_usd > 0.0);
1218	assert!(card.per_turn[1].cost_unpriced);
1219	assert_eq!(card.per_turn[1].cost_usd, 0.0);
1220	assert!(card.per_turn[2].cost_unpriced);
1221	assert_eq!(card.per_turn[2].cost_usd, 0.0);
1222	// Codex OAuth and Ollama are exactly non-metered, so they leave the
1223	// money-coverage denominator entirely rather than counting as unpriced
1224	// dollars: only the OpenAI turn owes money, and it is priced. The USD
1225	// total is therefore genuinely complete for the spend it covers (#4318).
1226	assert_eq!(card.metrics.money_metered_turns, 1);
1227	assert_eq!(card.metrics.unpriced_turns, 0);
1228	assert!(card.metrics.cost_complete);
1229	for (index, expected) in [(1_usize, false), (2_usize, false)] {
1230	assert_eq!(
1231	card.per_turn[index].money_metered, expected,
1232	"turn {index} money-metered"
1233	);
1234	assert_eq!(
1235	card.per_turn[index].cost_unpriced_reason.as_deref(),
1236	Some("not_money_metered"),
1237	"turn {index} reason"
1238	);
1239	}
1240	assert!(card.per_turn[0].money_metered);
1241	// CNY is only published by direct DeepSeek, so the single metered turn
1242	// still has no authoritative CNY figure.
1243	assert_eq!(card.metrics.cny_unpriced_turns, 1);
1244	assert!(!card.metrics.cny_cost_complete);
1245	assert!(card.to_summary().contains("money_metered_turns: 1"));
1246	assert!(card.to_summary().contains("cost_cny: unavailable"));
1247
1248	let json = serde_json::to_value(&card).expect("serialize scorecard");
1249	assert_eq!(json["per_turn"][0]["provider"], "openai");
1250	assert_eq!(json["per_turn"][1]["provider"], "openai-codex");
1251	assert_eq!(json["per_turn"][2]["provider"], "ollama");
1252	assert_eq!(json["metrics"]["money_metered_turns"], 1);
1253	assert_eq!(json["metrics"]["unpriced_turns"], 0);
1254	assert_eq!(json["metrics"]["cost_complete"], true);
1255	assert_eq!(json["metrics"]["cny_cost_complete"], false);
1256	}
1257
1258	#[test]
1259	fn first_party_hand_price_survives_a_missing_catalog_offering() {
1260	let u = usage(1_000_000, 0, 0);
1261	let turns = [
1262	TurnInput {
1263	turn_id: "openai-api".into(),
1264	created_at: None,
1265	provider: Some("openai"),
1266	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1267	model: "gpt-5-codex".into(),
1268	usage: &u,
1269	},
1270	TurnInput {
1271	turn_id: "foreign-route".into(),
1272	created_at: None,
1273	provider: Some("ollama"),
1274	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1275	model: "gpt-5-codex".into(),
1276	usage: &u,
1277	},
1278	];
1279
1280	let card = Scorecard::from_turns(&turns);
1281
1282	assert!(!card.per_turn[0].cost_unpriced);
1283	assert!((card.per_turn[0].cost_usd - 1.25).abs() < f64::EPSILON);
1284	assert!(card.per_turn[1].cost_unpriced);
1285	}
1286
1287	#[test]
1288	fn documented_no_cache_discount_uses_input_without_generalizing_missing_rates() {
1289	let u = Usage {
1290	input_tokens: 1_000_000,
1291	output_tokens: 0,
1292	prompt_cache_hit_tokens: Some(250_000),
1293	prompt_cache_write_tokens: Some(100_000),
1294	..Default::default()
1295	};
1296	let turns = [
1297	TurnInput {
1298	turn_id: "documented-no-discount".into(),
1299	created_at: None,
1300	provider: Some("openai"),
1301	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1302	model: "gpt-5.5-pro".into(),
1303	usage: &u,
1304	},
1305	TurnInput {
1306	turn_id: "missing-cache-rate".into(),
1307	created_at: None,
1308	provider: Some("meta"),
1309	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1310	model: "muse-spark-1.1".into(),
1311	usage: &u,
1312	},
1313	];
1314
1315	let card = Scorecard::from_turns(&turns);
1316
1317	assert!(!card.per_turn[0].cost_unpriced);
1318	assert!((card.per_turn[0].cost_usd - 30.0).abs() < f64::EPSILON);
1319	assert!(card.per_turn[1].cost_unpriced);
1320	assert!(!card.metrics.cost_complete);
1321	}
1322
1323	#[test]
1324	fn anthropic_sonnet_5_uses_the_recorded_turn_time() {
1325	let u = Usage {
1326	input_tokens: 1_000_000,
1327	output_tokens: 500_000,
1328	prompt_cache_hit_tokens: Some(250_000),
1329	prompt_cache_write_tokens: Some(100_000),
1330	..Default::default()
1331	};
1332	// Sonnet 5's $2/$10 launch rate became the standard price (Anthropic
1333	// pricing page, 2026-08-17: the 2026-09-01 increase "will not
1334	// occur"), so both sides of the former boundary price identically:
1335	// 650K miss * 2.00 + 250K hit * 0.20 + 100K write * 2.50 + 500K out
1336	// * 10.00 = 1.30 + 0.05 + 0.25 + 5.00 = 6.60. A turn with no recorded
1337	// time still fails closed rather than guessing a window.
1338	let before_boundary: DateTime<Utc> = "2026-08-31T23:59:59Z".parse().expect("time");
1339	let after_boundary: DateTime<Utc> = "2026-09-01T00:00:00Z".parse().expect("time");
1340	let turns = [
1341	TurnInput {
1342	turn_id: "sonnet-before".into(),
1343	created_at: Some(&before_boundary),
1344	provider: Some("anthropic"),
1345	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1346	model: " claude-sonnet-5 ".into(),
1347	usage: &u,
1348	},
1349	TurnInput {
1350	turn_id: "sonnet-after".into(),
1351	created_at: Some(&after_boundary),
1352	provider: Some("anthropic"),
1353	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1354	model: "claude-sonnet-5".into(),
1355	usage: &u,
1356	},
1357	TurnInput {
1358	turn_id: "sonnet-missing-time".into(),
1359	created_at: None,
1360	provider: Some("anthropic"),
1361	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1362	model: "claude-sonnet-5".into(),
1363	usage: &u,
1364	},
1365	];
1366
1367	let card = Scorecard::from_turns(&turns);
1368
1369	assert!(!card.per_turn[0].cost_unpriced);
1370	assert!((card.per_turn[0].cost_usd - 6.60).abs() < 1e-12);
1371	assert_eq!(card.per_turn[0].created_at.as_ref(), Some(&before_boundary));
1372	assert!(card.per_turn[0].cost_cny_unpriced);
1373	assert!(!card.per_turn[1].cost_unpriced);
1374	assert!((card.per_turn[1].cost_usd - 6.60).abs() < 1e-12);
1375	assert!(card.per_turn[1].cost_cny_unpriced);
1376	assert!(card.per_turn[2].cost_unpriced);
1377	}
1378
1379	#[test]
1380	fn known_zero_usage_is_zero_cost_not_unavailable() {
1381	let u = usage(0, 0, 0);
1382	let turns = [TurnInput {
1383	turn_id: "zero".into(),
1384	created_at: None,
1385	provider: Some("openai"),
1386	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1387	model: "gpt-5.5".into(),
1388	usage: &u,
1389	}];
1390
1391	let card = Scorecard::from_turns(&turns);
1392
1393	assert!(!card.per_turn[0].cost_unpriced);
1394	assert_eq!(card.per_turn[0].cost_usd, 0.0);
1395	assert!(card.per_turn[0].cost_cny_unpriced);
1396	assert_eq!(card.metrics.unpriced_turns, 0);
1397	assert_eq!(card.metrics.cny_unpriced_turns, 1);
1398	assert!(card.metrics.cost_complete);
1399	assert!(!card.metrics.cny_cost_complete);
1400	assert!(card.to_summary().contains("cost_usd: $0.0000"));
1401	assert!(card.to_summary().contains("cost_cny: unavailable"));
1402	}
1403
1404	#[test]
1405	fn direct_deepseek_route_keeps_authoritative_dual_currency_pricing() {
1406	let u = usage(1000, 500, 0);
1407	let recorded_at: DateTime<Utc> = "2026-08-17T15:00:00Z".parse().expect("recorded time");
1408	let turns = [TurnInput {
1409	turn_id: "deepseek".into(),
1410	created_at: Some(&recorded_at),
1411	provider: Some("deepseek"),
1412	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1413	model: "deepseek-v4-pro".into(),
1414	usage: &u,
1415	}];
1416
1417	let card = Scorecard::from_turns(&turns);
1418
1419	assert!(!card.per_turn[0].cost_unpriced);
1420	assert!(!card.per_turn[0].cost_cny_unpriced);
1421	assert!(card.per_turn[0].cost_usd > 0.0);
1422	assert!(card.per_turn[0].cost_cny > 0.0);
1423	assert!(card.metrics.cost_complete);
1424	assert!(card.metrics.cny_cost_complete);
1425	}
1426
1427	#[test]
1428	fn undated_direct_deepseek_v4_turns_fail_closed_on_the_time_window() {
1429	// V4 flash/pro are peak/off-peak tiered by the turn's recorded time; a
1430	// turn the recorder did not date cannot be resolved to one price and
1431	// must not be silently priced at whatever tier `now` happens to be.
1432	let u = usage(1000, 500, 0);
1433	let turns = [
1434	TurnInput {
1435	turn_id: "undated-pro".into(),
1436	created_at: None,
1437	provider: Some("deepseek"),
1438	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1439	model: "deepseek-v4-pro".into(),
1440	usage: &u,
1441	},
1442	TurnInput {
1443	turn_id: "undated-flash".into(),
1444	created_at: None,
1445	provider: Some("deepseek"),
1446	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1447	model: "deepseek-v4-flash".into(),
1448	usage: &u,
1449	},
1450	];
1451
1452	let card = Scorecard::from_turns(&turns);
1453
1454	assert!(card.per_turn.iter().all(\|turn\| turn.cost_unpriced));
1455	assert!(card.per_turn.iter().all(\|turn\| turn.cost_cny_unpriced));
1456	assert!(!card.metrics.cost_complete);
1457	}
1458
1459	#[test]
1460	fn direct_deepseek_compact_aliases_use_canonical_pricing() {
1461	let u = usage(1000, 500, 100);
1462	let recorded_at: DateTime<Utc> = "2026-08-17T15:00:00Z".parse().expect("recorded time");
1463	let models = [
1464	"deepseek-v4-pro",
1465	"pro",
1466	" DeepSeek-V4Pro ",
1467	"deepseek-v4-flash",
1468	"flash",
1469	"DEEPSEEK-V4FLASH",
1470	];
1471	let turns: Vec<_> = models
1472	.iter()
1473	.map(\|model\| TurnInput {
1474	turn_id: (*model).into(),
1475	created_at: Some(&recorded_at),
1476	provider: Some("deepseek"),
1477	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1478	model: (*model).into(),
1479	usage: &u,
1480	})
1481	.collect();
1482
1483	let card = Scorecard::from_turns(&turns);
1484
1485	for alias in [1, 2] {
1486	assert_eq!(card.per_turn[alias].cost_usd, card.per_turn[0].cost_usd);
1487	assert_eq!(card.per_turn[alias].cost_cny, card.per_turn[0].cost_cny);
1488	}
1489	for alias in [4, 5] {
1490	assert_eq!(card.per_turn[alias].cost_usd, card.per_turn[3].cost_usd);
1491	assert_eq!(card.per_turn[alias].cost_cny, card.per_turn[3].cost_cny);
1492	}
1493	assert!(card.per_turn.iter().all(\|turn\| !turn.cost_unpriced));
1494	assert!(card.per_turn.iter().all(\|turn\| !turn.cost_cny_unpriced));
1495	}
1496
1497	#[test]
1498	fn direct_deepseek_compatibility_aliases_use_the_flash_route() {
1499	let u = usage(1000, 500, 100);
1500	let before_retirement: DateTime<Utc> =
1501	"2026-07-24T15:58:59Z".parse().expect("pre-retirement time");
1502	let at_retirement: DateTime<Utc> = DEEPSEEK_ALIAS_RETIREMENT_UTC
1503	.parse()
1504	.expect("retirement time");
1505	let turns = [
1506	TurnInput {
1507	turn_id: "chat-alias".into(),
1508	created_at: Some(&before_retirement),
1509	provider: Some("deepseek"),
1510	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1511	model: "deepseek-chat".into(),
1512	usage: &u,
1513	},
1514	TurnInput {
1515	turn_id: "reasoner-alias".into(),
1516	created_at: Some(&before_retirement),
1517	provider: Some("deepseek"),
1518	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1519	model: "deepseek-reasoner".into(),
1520	usage: &u,
1521	},
1522	TurnInput {
1523	turn_id: "canonical".into(),
1524	created_at: Some(&before_retirement),
1525	provider: Some("deepseek"),
1526	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1527	model: DEEPSEEK_ALIAS_REPLACEMENT.into(),
1528	usage: &u,
1529	},
1530	TurnInput {
1531	turn_id: "retired-alias".into(),
1532	created_at: Some(&at_retirement),
1533	provider: Some("deepseek"),
1534	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1535	model: "deepseek-chat".into(),
1536	usage: &u,
1537	},
1538	TurnInput {
1539	turn_id: "undated-alias".into(),
1540	created_at: None,
1541	provider: Some("deepseek"),
1542	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1543	model: "deepseek-reasoner".into(),
1544	usage: &u,
1545	},
1546	];
1547
1548	let card = Scorecard::from_turns(&turns);
1549
1550	assert_eq!(card.per_turn[0].cost_usd, card.per_turn[2].cost_usd);
1551	assert_eq!(card.per_turn[1].cost_usd, card.per_turn[2].cost_usd);
1552	assert_eq!(card.per_turn[0].cost_cny, card.per_turn[2].cost_cny);
1553	assert_eq!(card.per_turn[1].cost_cny, card.per_turn[2].cost_cny);
1554	assert!(card.per_turn[..3].iter().all(\|turn\| !turn.cost_unpriced));
1555	assert!(
1556	card.per_turn[..3]
1557	.iter()
1558	.all(\|turn\| !turn.cost_cny_unpriced)
1559	);
1560	assert!(card.per_turn[3].cost_unpriced);
1561	assert!(card.per_turn[4].cost_unpriced);
1562	}
1563
1564	#[test]
1565	fn direct_arcee_aliases_do_not_cross_the_openrouter_namespace() {
1566	let u = Usage {
1567	input_tokens: 1_000_000,
1568	output_tokens: 500_000,
1569	prompt_cache_hit_tokens: Some(250_000),
1570	prompt_cache_write_tokens: Some(100_000),
1571	..Default::default()
1572	};
1573	let turns = [
1574	TurnInput {
1575	turn_id: "canonical-direct".into(),
1576	created_at: None,
1577	provider: Some("arcee"),
1578	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1579	model: "trinity-large-thinking".into(),
1580	usage: &u,
1581	},
1582	TurnInput {
1583	turn_id: "direct-alias".into(),
1584	created_at: None,
1585	provider: Some("arcee"),
1586	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1587	model: "arcee-trinity-large-thinking".into(),
1588	usage: &u,
1589	},
1590	TurnInput {
1591	turn_id: "openrouter-namespace".into(),
1592	created_at: None,
1593	provider: Some("arcee"),
1594	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1595	model: "arcee-ai/trinity-large-thinking".into(),
1596	usage: &u,
1597	},
1598	];
1599
1600	let card = Scorecard::from_turns(&turns);
1601
1602	assert!(!card.per_turn[0].cost_unpriced);
1603	assert!((card.per_turn[0].cost_usd - 0.65).abs() < f64::EPSILON);
1604	assert_eq!(card.per_turn[1].cost_usd, card.per_turn[0].cost_usd);
1605	assert!(!card.per_turn[1].cost_unpriced);
1606	assert!(card.per_turn[2].cost_unpriced);
1607	}
1608
1609	#[test]
1610	fn costless_catalog_rows_fall_back_only_to_verified_provider_prices() {
1611	let u = Usage {
1612	input_tokens: 1_000_000,
1613	output_tokens: 500_000,
1614	prompt_cache_hit_tokens: Some(250_000),
1615	prompt_cache_write_tokens: Some(100_000),
1616	..Default::default()
1617	};
1618	let turns = [
1619	TurnInput {
1620	turn_id: "arcee-mini".into(),
1621	created_at: None,
1622	provider: Some("arcee"),
1623	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1624	model: "trinity-mini".into(),
1625	usage: &u,
1626	},
1627	TurnInput {
1628	turn_id: "minimax-m2.7".into(),
1629	created_at: None,
1630	provider: Some("minimax"),
1631	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1632	model: "minimax-m2.7".into(),
1633	usage: &u,
1634	},
1635	TurnInput {
1636	turn_id: "foreign-route".into(),
1637	created_at: None,
1638	provider: Some("ollama"),
1639	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1640	model: "trinity-mini".into(),
1641	usage: &u,
1642	},
1643	TurnInput {
1644	turn_id: "openai-hosted-deepseek".into(),
1645	created_at: None,
1646	provider: Some("openai"),
1647	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1648	model: "deepseek-v4-pro".into(),
1649	usage: &u,
1650	},
1651	TurnInput {
1652	turn_id: "openrouter-hosted-zai".into(),
1653	created_at: None,
1654	provider: Some("openrouter"),
1655	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1656	model: "z-ai/glm-5.2".into(),
1657	usage: &u,
1658	},
1659	];
1660
1661	let card = Scorecard::from_turns(&turns);
1662
1663	// Trinity Mini has no verified provider rate in the release metadata;
1664	// a removed hand-written estimate must stay unknown, not become zero
1665	// or leak through from a similarly named route.
1666	assert_eq!(card.per_turn[0].cost_usd, 0.0);
1667	assert!(card.per_turn[0].cost_unpriced);
1668	// MiniMax-M2.7 publishes a distinct cache-write rate (0.375/M),
1669	// retained by the provider-owned fallback even without a priced
1670	// catalog offering.
1671	assert!((card.per_turn[1].cost_usd - 0.8475).abs() < f64::EPSILON);
1672	assert!(!card.per_turn[1].cost_unpriced);
1673	assert!(card.per_turn[..2].iter().all(\|turn\| turn.cost_cny_unpriced));
1674	assert!(card.per_turn[2..].iter().all(\|turn\| turn.cost_unpriced));
1675	}
1676
1677	#[test]
1678	fn stepfun_legacy_route_keeps_pricing_without_a_catalog_row() {
1679	let u = usage(1000, 500, 250);
1680	let recorded = \|turn_id: &str,
1681	provider: &str,
1682	model: &str,
1683	billing_surface: Option<&str>\| RecordedTurn {
1684	turn_id: turn_id.to_string(),
1685	created_at: None,
1686	model_backed: Some(true),
1687	provider: Some(provider.to_string()),
1688	billing_surface: billing_surface.map(str::to_string),
1689	model: model.to_string(),
1690	usage: Some(u.clone()),
1691	};
1692	let turns = [
1693	recorded(
1694	"stepfun-default",
1695	"stepfun",
1696	" STEP-3.7-FLASH ",
1697	Some(crate::pricing::STEPFUN_PAYG_BILLING_SURFACE),
1698	),
1699	recorded(
1700	"stepfun-plan",
1701	"stepfun",
1702	"step-3.7-flash",
1703	Some(crate::pricing::STEPFUN_PLAN_BILLING_SURFACE),
1704	),
1705	recorded("stepfun-missing-surface", "stepfun", "step-3.7-flash", None),
1706	recorded("stepfun-unknown-model", "stepfun", "step-3.5-flash", None),
1707	recorded(
1708	"openrouter-stepfun-name",
1709	"openrouter",
1710	"step-3.7-flash",
1711	None,
1712	),
1713	recorded("local-stepfun-name", "ollama", "step-3.7-flash", None),
1714	recorded(
1715	"sakana-incomplete-tier-price",
1716	"sakana",
1717	"fugu-ultra-20260615",
1718	None,
1719	),
1720	recorded(
1721	"foreign-deepseek-name",
1722	"openmodel",
1723	"deepseek-v4-flash",
1724	None,
1725	),
1726	];
1727
1728	let card = Scorecard::from_recorded_turns(&turns);
1729
1730	assert!((card.per_turn[0].cost_usd - 0.000_735).abs() < 1e-12);
1731	assert!(!card.per_turn[0].cost_unpriced);
1732	assert!(card.per_turn[0].cost_cny_unpriced);
1733	assert_eq!(
1734	card.per_turn[0].billing_surface.as_deref(),
1735	Some(crate::pricing::STEPFUN_PAYG_BILLING_SURFACE)
1736	);
1737	assert!(card.per_turn[1..].iter().all(\|turn\| turn.cost_unpriced));
1738	}
1739
1740	#[test]
1741	fn legacy_model_only_record_is_readable_but_unpriced() {
1742	let recorded: RecordedTurn = serde_json::from_value(serde_json::json!({
1743	"turn_id": "legacy",
1744	"model": "gpt-5.5",
1745	"usage": {
1746	"input_tokens": 0,
1747	"output_tokens": 0
1748	}
1749	}))
1750	.expect("parse legacy scorecard turn");
1751	assert_eq!(recorded.provider, None);
1752	assert_eq!(recorded.billing_surface, None);
1753
1754	let card = Scorecard::from_recorded_turns(&[recorded]);
1755
1756	assert!(card.per_turn[0].cost_unpriced);
1757	assert_eq!(card.per_turn[0].cost_usd, 0.0);
1758	assert_eq!(card.metrics.unpriced_turns, 1);
1759	assert!(card.to_summary().contains("cost_usd: unavailable"));
1760	}
1761
1762	#[test]
1763	fn recorded_turn_accepts_runtime_route_aliases() {
1764	let recorded: RecordedTurn = serde_json::from_value(serde_json::json!({
1765	"schema_version": 1,
1766	"id": "runtime-turn",
1767	"thread_id": "thread-1",
1768	"status": "completed",
1769	"input_summary": "score this turn",
1770	"created_at": "2026-07-12T10:30:00Z",
1771	"effective_provider": "openai-codex",
1772	"effective_billing_surface": "account-subscription",
1773	"effective_model": "gpt-5.5",
1774	"usage": {
1775	"input_tokens": 1,
1776	"output_tokens": 1
1777	}
1778	}))
1779	.expect("parse runtime scorecard turn");
1780
1781	assert_eq!(recorded.turn_id, "runtime-turn");
1782	assert_eq!(
1783	recorded.created_at.as_ref().map(DateTime::to_rfc3339),
1784	Some("2026-07-12T10:30:00+00:00".to_string())
1785	);
1786	assert_eq!(recorded.provider.as_deref(), Some("openai-codex"));
1787	assert_eq!(
1788	recorded.billing_surface.as_deref(),
1789	Some("account-subscription")
1790	);
1791	assert_eq!(recorded.model, "gpt-5.5");
1792	assert!(recorded.contributes_to_scorecard());
1793	}
1794
1795	#[test]
1796	fn runtime_turn_without_usage_is_readable_and_filtered() {
1797	let recorded: RecordedTurn = serde_json::from_value(serde_json::json!({
1798	"schema_version": 1,
1799	"id": "queued-runtime-turn",
1800	"thread_id": "thread-1",
1801	"status": "queued",
1802	"input_summary": "waiting to run",
1803	"created_at": "2026-07-12T10:30:00Z",
1804	"effective_provider": "openai",
1805	"effective_model": "gpt-5.5"
1806	}))
1807	.expect("parse runtime row before usage is recorded");
1808
1809	assert!(recorded.usage.is_none());
1810	assert!(!recorded.contributes_to_scorecard());
1811	let card = Scorecard::from_recorded_turns(&[recorded]);
1812	assert_eq!(card.metrics.turns, 0);
1813	assert!(card.per_turn.is_empty());
1814	}
1815
1816	#[test]
1817	fn recorded_non_model_hook_turn_is_excluded_from_model_scorecard() {
1818	let recorded: RecordedTurn = serde_json::from_value(serde_json::json!({
1819	"turn_id": "shell-turn",
1820	"created_at": "2026-07-12T10:30:00Z",
1821	"model_backed": false,
1822	"provider": null,
1823	"model": "gpt-5.5",
1824	"usage": {
1825	"input_tokens": 0,
1826	"output_tokens": 0
1827	}
1828	}))
1829	.expect("parse non-model turn_end record");
1830
1831	assert!(!recorded.contributes_to_scorecard());
1832	}
1833
1834	#[test]
1835	fn blank_unknown_and_custom_providers_fail_closed_as_unpriced() {
1836	let u = usage(1000, 500, 0);
1837	let turns = [
1838	TurnInput {
1839	turn_id: "blank".into(),
1840	created_at: None,
1841	provider: Some(" "),
1842	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1843	model: "gpt-5.5".into(),
1844	usage: &u,
1845	},
1846	TurnInput {
1847	turn_id: "named-custom".into(),
1848	created_at: None,
1849	provider: Some("my-openai-proxy"),
1850	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1851	model: "gpt-5.5".into(),
1852	usage: &u,
1853	},
1854	TurnInput {
1855	turn_id: "generic-custom".into(),
1856	created_at: None,
1857	provider: Some("custom"),
1858	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1859	model: "gpt-5.5".into(),
1860	usage: &u,
1861	},
1862	];
1863
1864	let card = Scorecard::from_turns(&turns);
1865
1866	assert_eq!(card.per_turn[0].provider, None);
1867	assert_eq!(
1868	card.per_turn[1].provider.as_deref(),
1869	Some("my-openai-proxy")
1870	);
1871	assert_eq!(card.per_turn[2].provider.as_deref(), Some("custom"));
1872	assert!(card.per_turn.iter().all(\|turn\| turn.cost_unpriced));
1873	assert_eq!(card.metrics.unpriced_turns, 3);
1874	assert!(!card.metrics.cost_complete);
1875	assert!(card.to_summary().contains("cost_usd: unavailable"));
1876	}
1877
1878	#[test]
1879	fn regression_flags_cost_and_token_increases_over_threshold() {
1880	let baseline = ScorecardMetrics {
1881	turns: 1,
1882	money_metered_turns: 1,
1883	unpriced_turns: 0,
1884	cny_unpriced_turns: 0,
1885	cost_complete: true,
1886	cny_cost_complete: true,
1887	unpriced_classes: Vec::new(),
1888	total_input_tokens: 1000,
1889	total_output_tokens: 1000,
1890	total_cache_read_tokens: 0,
1891	total_cache_write_tokens: 0,
1892	total_reasoning_tokens: 0,
1893	total_cost_usd: 0.10,
1894	total_cost_cny: 0.7,
1895	cache_hit_ratio: 0.5,
1896	};
1897	let current = ScorecardMetrics {
1898	total_cost_usd: 0.20, // +100% → regression
1899	total_input_tokens: 1010, // +1% → under 5% threshold, no regression
1900	total_output_tokens: 2000, // +100% → regression
1901	cache_hit_ratio: 0.5, // unchanged
1902	..baseline.clone()
1903	};
1904	let regs = current.regressions_against(&baseline, 5.0);
1905	let names: Vec<&str> = regs.iter().map(\|r\| r.metric.as_str()).collect();
1906	assert!(names.contains(&"total_cost_usd"));
1907	assert!(names.contains(&"total_output_tokens"));
1908	assert!(!names.contains(&"total_input_tokens")); // under threshold
1909	}
1910
1911	#[test]
1912	fn regression_flags_loss_of_cost_completeness_without_comparing_subtotals() {
1913	let baseline = ScorecardMetrics {
1914	cost_complete: true,
1915	total_cost_usd: 0.10,
1916	..Default::default()
1917	};
1918	let current = ScorecardMetrics {
1919	turns: 1,
1920	unpriced_turns: 1,
1921	total_cost_usd: 0.20,
1922	..Default::default()
1923	};
1924
1925	let regs = current.regressions_against(&baseline, 5.0);
1926	assert!(!regs.iter().any(\|r\| r.metric == "total_cost_usd"));
1927	assert!(regs.iter().any(\|r\| r.metric == "cost_completeness_drop"));
1928	}
1929
1930	#[test]
1931	fn regression_flags_loss_of_cny_cost_completeness() {
1932	let baseline = ScorecardMetrics {
1933	cny_cost_complete: true,
1934	total_cost_cny: 0.70,
1935	..Default::default()
1936	};
1937	let current = ScorecardMetrics {
1938	turns: 1,
1939	cny_unpriced_turns: 1,
1940	total_cost_cny: 0.0,
1941	..Default::default()
1942	};
1943
1944	let regs = current.regressions_against(&baseline, 5.0);
1945	assert!(
1946	regs.iter()
1947	.any(\|r\| r.metric == "cny_cost_completeness_drop")
1948	);
1949	}
1950
1951	#[test]
1952	fn regression_flags_complete_cny_cost_increase() {
1953	let baseline = ScorecardMetrics {
1954	cny_cost_complete: true,
1955	total_cost_cny: 0.70,
1956	..Default::default()
1957	};
1958	let current = ScorecardMetrics {
1959	total_cost_cny: 1.40,
1960	..baseline.clone()
1961	};
1962
1963	let regs = current.regressions_against(&baseline, 5.0);
1964	assert!(regs.iter().any(\|r\| r.metric == "total_cost_cny"));
1965	}
1966
1967	#[test]
1968	fn legacy_baseline_is_readable_but_cost_is_not_comparable() {
1969	let baseline: ScorecardMetrics = serde_json::from_value(serde_json::json!({
1970	"turns": 1,
1971	"total_input_tokens": 10,
1972	"total_output_tokens": 5,
1973	"total_cache_read_tokens": 0,
1974	"total_cost_usd": 0.10,
1975	"total_cost_cny": 0.0,
1976	"cache_hit_ratio": 0.0
1977	}))
1978	.expect("parse legacy scorecard baseline");
1979	assert!(!baseline.cost_complete);
1980
1981	let current = ScorecardMetrics {
1982	cost_complete: true,
1983	total_cost_usd: 0.20,
1984	total_input_tokens: 10,
1985	total_output_tokens: 5,
1986	..Default::default()
1987	};
1988	let regs = current.regressions_against(&baseline, 5.0);
1989	assert!(!regs.iter().any(\|r\| r.metric == "total_cost_usd"));
1990	}
1991
1992	#[test]
1993	fn regression_flags_cache_hit_ratio_drop() {
1994	let baseline = ScorecardMetrics {
1995	cache_hit_ratio: 0.80,
1996	..Default::default()
1997	};
1998	let current = ScorecardMetrics {
1999	cache_hit_ratio: 0.40,
2000	..Default::default()
2001	};
2002	let regs = current.regressions_against(&baseline, 10.0);
2003	assert!(regs.iter().any(\|r\| r.metric == "cache_hit_ratio_drop"));
2004	}
2005
2006	#[test]
2007	fn no_regressions_when_within_threshold() {
2008	let baseline = ScorecardMetrics {
2009	total_cost_usd: 1.0,
2010	total_input_tokens: 1000,
2011	total_output_tokens: 1000,
2012	cache_hit_ratio: 0.5,
2013	..Default::default()
2014	};
2015	let current = baseline.clone();
2016	assert!(current.regressions_against(&baseline, 5.0).is_empty());
2017	}
2018
2019	#[test]
2020	fn cache_hit_denominator_saturates_instead_of_wrapping() {
2021	assert_eq!(cacheable_token_total(u64::MAX, 1, 1), u64::MAX);
2022	assert_eq!(cacheable_token_total(1, u64::MAX, 1), u64::MAX);
2023	}
2024	}
2025

2025 lines RUST