| 1 | //! Live Models.dev catalog fetch + secret-free disk cache (#4187). |
| 2 | //! |
| 3 | //! OpenCode-style producer that: |
| 4 | //! - reads a stale/fresh disk cache on startup (never blocks model selection), |
| 5 | //! - fetches `https://models.dev/catalog.json` in the background with a bounded |
| 6 | //! timeout and explicit user-agent (no credentials), |
| 7 | //! - writes the cache atomically via temp file + rename, |
| 8 | //! - compiles parsed rows into `CatalogOffering`s and publishes them into |
| 9 | //! [`crate::provider_lake`], |
| 10 | //! - falls back to the prior cache or the bundled snapshot on any failure. |
| 11 | //! |
| 12 | //! Override knobs (tests / dogfood): |
| 13 | //! - `CODEWHALE_MODELS_DEV_URL` — base URL (appends `/catalog.json`) or a full |
| 14 | //! `*.json` URL. |
| 15 | //! - `CODEWHALE_MODELS_DEV_PATH` — local file path; skips the network. |
| 16 | //! - `CODEWHALE_DISABLE_MODELS_DEV_FETCH` — when truthy, never hits the network. |
| 17 | |
| 18 | use std::path::{Path, PathBuf}; |
| 19 | use std::sync::RwLock; |
| 20 | use std::time::Duration; |
| 21 | |
| 22 | use codewhale_config::catalog::{ |
| 23 | CatalogSnapshot, base_url_fingerprint, live_offerings_from_models_dev, now_unix, |
| 24 | }; |
| 25 | use codewhale_config::models_dev::{MODELS_DEV_CATALOG_URL, ModelsDevCatalog}; |
| 26 | use codewhale_config::persistence::atomic_write; |
| 27 | use serde::{Deserialize, Serialize}; |
| 28 | |
| 29 | /// Default TTL for a live Models.dev snapshot (24h, #4187 / #4114). |
| 30 | pub const DEFAULT_MODELS_DEV_TTL_SECS: u64 = 24 * 60 * 60; |
| 31 | |
| 32 | /// Bounded HTTP timeout for the Models.dev fetch. |
| 33 | pub const FETCH_TIMEOUT: Duration = Duration::from_secs(15); |
| 34 | |
| 35 | /// Explicit user-agent; no credentials, no session cookies. |
| 36 | pub const USER_AGENT: &str = concat!("CodeWhale/", env!("CARGO_PKG_VERSION"), " (+models-dev)"); |
| 37 | |
| 38 | /// Filename under the CodeWhale `catalog` state dir. |
| 39 | pub const CACHE_FILE: &str = "models-dev-catalog.json"; |
| 40 | |
| 41 | /// Env: override Models.dev base URL or full catalog URL. |
| 42 | pub const ENV_MODELS_DEV_URL: &str = "CODEWHALE_MODELS_DEV_URL"; |
| 43 | /// Env: load catalog JSON from a local path (skips network). |
| 44 | pub const ENV_MODELS_DEV_PATH: &str = "CODEWHALE_MODELS_DEV_PATH"; |
| 45 | /// Env: disable network fetch entirely (`1`/`true`/`yes`/`on`). |
| 46 | pub const ENV_DISABLE_FETCH: &str = "CODEWHALE_DISABLE_MODELS_DEV_FETCH"; |
| 47 | |
| 48 | const CACHE_SCHEMA_VERSION: u32 = 1; |
| 49 | |
| 50 | /// Provenance / freshness of the Models.dev live layer for UI chips (#4187). |
| 51 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] |
| 52 | #[serde(rename_all = "snake_case")] |
| 53 | pub enum ModelsDevFreshness { |
| 54 | /// No live/cache layer; pickers see bundled rows only. |
| 55 | #[default] |
| 56 | Bundled, |
| 57 | /// Live (or disk-cache) rows within TTL. |
| 58 | Live, |
| 59 | /// Disk-cache / prior live rows past TTL; still visible. |
| 60 | Stale, |
| 61 | /// Last refresh failed; prior/bundled rows remain available. |
| 62 | Failed, |
| 63 | } |
| 64 | |
| 65 | /// Quiet status snapshot for UI / `/model refresh` feedback. |
| 66 | #[derive(Debug, Clone, PartialEq, Eq, Default)] |
| 67 | pub struct ModelsDevStatus { |
| 68 | pub freshness: ModelsDevFreshness, |
| 69 | pub offering_count: usize, |
| 70 | pub fetched_at: Option<u64>, |
| 71 | pub source_label: String, |
| 72 | pub last_error: Option<String>, |
| 73 | } |
| 74 | |
| 75 | static STATUS: RwLock<ModelsDevStatus> = RwLock::new(ModelsDevStatus { |
| 76 | freshness: ModelsDevFreshness::Bundled, |
| 77 | offering_count: 0, |
| 78 | fetched_at: None, |
| 79 | source_label: String::new(), |
| 80 | last_error: None, |
| 81 | }); |
| 82 | |
| 83 | #[derive(Debug, Clone, Serialize, Deserialize)] |
| 84 | struct PersistedModelsDevCache { |
| 85 | schema_version: u32, |
| 86 | /// Unix seconds the payload was fetched (or loaded from an override path). |
| 87 | fetched_at: u64, |
| 88 | /// Fingerprint of the source URL/path this body was fetched from. It scopes |
| 89 | /// the on-disk cache; it is deliberately not carried on the published rows, |
| 90 | /// which describe a model rather than an endpoint (`ModelsDevLive`). |
| 91 | source_fingerprint: String, |
| 92 | /// Human-readable source label (URL or `file:…`); never a secret. |
| 93 | source_label: String, |
| 94 | /// Raw Models.dev catalog JSON body (secret-free by construction). |
| 95 | body: String, |
| 96 | } |
| 97 | |
| 98 | /// Metadata header for the v2 cache format. |
| 99 | /// |
| 100 | /// v1 serialized the whole cache as one JSON envelope with the catalog body |
| 101 | /// escaped inside it, so loading parsed ~5MB twice (envelope, then body) plus |
| 102 | /// a full-body copy on every interactive boot. v2 stores the metadata as a |
| 103 | /// single JSON header line followed by the raw catalog body bytes, so boot |
| 104 | /// performs exactly one catalog parse and zero body copies. |
| 105 | const CACHE_SCHEMA_VERSION_V2: u32 = 2; |
| 106 | |
| 107 | #[derive(Debug, Clone, Serialize, Deserialize)] |
| 108 | struct PersistedModelsDevCacheV2 { |
| 109 | schema_version: u32, |
| 110 | fetched_at: u64, |
| 111 | source_fingerprint: String, |
| 112 | source_label: String, |
| 113 | } |
| 114 | |
| 115 | /// Why a Models.dev refresh did not publish new rows. |
| 116 | #[derive(Debug, Clone, PartialEq, Eq)] |
| 117 | pub enum ModelsDevRefreshError { |
| 118 | Disabled, |
| 119 | Network(String), |
| 120 | HttpStatus(u16), |
| 121 | InvalidResponse(String), |
| 122 | EmptyCatalog, |
| 123 | Io(String), |
| 124 | } |
| 125 | |
| 126 | impl std::fmt::Display for ModelsDevRefreshError { |
| 127 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { |
| 128 | match self { |
| 129 | Self::Disabled => write!(f, "Models.dev fetch disabled"), |
| 130 | Self::Network(msg) => write!(f, "network: {msg}"), |
| 131 | Self::HttpStatus(code) => write!(f, "HTTP {code}"), |
| 132 | Self::InvalidResponse(msg) => write!(f, "invalid response: {msg}"), |
| 133 | Self::EmptyCatalog => write!(f, "empty catalog"), |
| 134 | Self::Io(msg) => write!(f, "io: {msg}"), |
| 135 | } |
| 136 | } |
| 137 | } |
| 138 | |
| 139 | /// Resolve the on-disk cache path under the CodeWhale `catalog` state dir. |
| 140 | /// |
| 141 | /// Under `cfg(test)` this is confined the same way settings and config paths |
| 142 | /// are: `resolve_state_dir` lives in `codewhale-config`, which is compiled as a |
| 143 | /// plain dependency here and so has no view of this crate's isolated test root. |
| 144 | /// Without this shield a test that never asked for the developer's catalog read |
| 145 | /// their real `~/.codewhale/catalog` and rendered against whatever models they |
| 146 | /// last fetched (#5359). |
| 147 | #[must_use] |
| 148 | pub fn cache_path() -> Option<PathBuf> { |
| 149 | #[cfg(test)] |
| 150 | { |
| 151 | if !crate::test_support::guarded_environment_provides_state_paths() { |
| 152 | return Some( |
| 153 | crate::test_support::unsealed_test_state_root() |
| 154 | .join("catalog") |
| 155 | .join(CACHE_FILE), |
| 156 | ); |
| 157 | } |
| 158 | } |
| 159 | codewhale_config::resolve_state_dir("catalog") |
| 160 | .ok() |
| 161 | .map(|dir| dir.join(CACHE_FILE)) |
| 162 | } |
| 163 | |
| 164 | /// Current quiet status (for UI / slash-command feedback). |
| 165 | #[must_use] |
| 166 | pub fn status() -> ModelsDevStatus { |
| 167 | let current = STATUS.read().map(|guard| guard.clone()).unwrap_or_default(); |
| 168 | honor_bundled_staleness( |
| 169 | current, |
| 170 | codewhale_models::model_catalog::bundled_catalog_is_stale(), |
| 171 | ) |
| 172 | } |
| 173 | |
| 174 | /// A Bundled-only report whose snapshot is itself past TTL reports `Stale`: |
| 175 | /// rows stay visible for offline use, but never as a current catalog (#A2). |
| 176 | fn honor_bundled_staleness(mut status: ModelsDevStatus, bundled_stale: bool) -> ModelsDevStatus { |
| 177 | if status.freshness == ModelsDevFreshness::Bundled && bundled_stale { |
| 178 | status.freshness = ModelsDevFreshness::Stale; |
| 179 | } |
| 180 | status |
| 181 | } |
| 182 | |
| 183 | fn set_status(next: ModelsDevStatus) { |
| 184 | if let Ok(mut guard) = STATUS.write() { |
| 185 | *guard = next; |
| 186 | } |
| 187 | } |
| 188 | |
| 189 | fn env_truthy(name: &str) -> bool { |
| 190 | std::env::var(name) |
| 191 | .map(|v| { |
| 192 | matches!( |
| 193 | v.trim().to_ascii_lowercase().as_str(), |
| 194 | "1" | "true" | "yes" | "on" |
| 195 | ) |
| 196 | }) |
| 197 | .unwrap_or(false) |
| 198 | } |
| 199 | |
| 200 | /// Resolve the catalog URL from env override or the Models.dev default. |
| 201 | #[must_use] |
| 202 | pub fn resolve_catalog_url() -> String { |
| 203 | match std::env::var(ENV_MODELS_DEV_URL) { |
| 204 | Ok(raw) => { |
| 205 | let trimmed = raw.trim(); |
| 206 | if trimmed.is_empty() { |
| 207 | MODELS_DEV_CATALOG_URL.to_string() |
| 208 | } else if trimmed.ends_with(".json") { |
| 209 | trimmed.to_string() |
| 210 | } else { |
| 211 | format!("{}/catalog.json", trimmed.trim_end_matches('/')) |
| 212 | } |
| 213 | } |
| 214 | Err(_) => MODELS_DEV_CATALOG_URL.to_string(), |
| 215 | } |
| 216 | } |
| 217 | |
| 218 | /// Seed ProviderLake from the on-disk Models.dev cache before any picker read. |
| 219 | /// |
| 220 | /// Missing / corrupt / empty caches are a no-op — bundled rows remain available. |
| 221 | /// Stale caches still publish (freshness = Stale) so offline startups keep the |
| 222 | /// last-known live rows. |
| 223 | pub fn maybe_load_persisted_cache() { |
| 224 | let Some(path) = cache_path() else { |
| 225 | return; |
| 226 | }; |
| 227 | let Some(cache) = load_cache_file(&path) else { |
| 228 | return; |
| 229 | }; |
| 230 | let age = now_unix().saturating_sub(cache.fetched_at); |
| 231 | let freshness = if age > DEFAULT_MODELS_DEV_TTL_SECS { |
| 232 | ModelsDevFreshness::Stale |
| 233 | } else { |
| 234 | ModelsDevFreshness::Live |
| 235 | }; |
| 236 | if let Err(err) = publish_from_body( |
| 237 | &cache.body, |
| 238 | cache.fetched_at, |
| 239 | cache.source_label.as_str(), |
| 240 | freshness, |
| 241 | ) { |
| 242 | tracing::debug!( |
| 243 | target: "models_dev_live", |
| 244 | error = %err, |
| 245 | "persisted Models.dev cache failed to publish; keeping bundled" |
| 246 | ); |
| 247 | } |
| 248 | } |
| 249 | |
| 250 | /// Force a refresh: prefer `CODEWHALE_MODELS_DEV_PATH`, else network fetch. |
| 251 | /// |
| 252 | /// On success, updates the disk cache and ProviderLake. On failure, keeps any |
| 253 | /// prior live/bundled rows and records a quiet Failed status. |
| 254 | pub async fn refresh(force_network: bool) -> Result<usize, ModelsDevRefreshError> { |
| 255 | if let Ok(path) = std::env::var(ENV_MODELS_DEV_PATH) { |
| 256 | let trimmed = path.trim(); |
| 257 | if !trimmed.is_empty() { |
| 258 | return refresh_from_path(Path::new(trimmed)).await; |
| 259 | } |
| 260 | } |
| 261 | |
| 262 | if env_truthy(ENV_DISABLE_FETCH) { |
| 263 | mark_failed(ModelsDevRefreshError::Disabled); |
| 264 | return Err(ModelsDevRefreshError::Disabled); |
| 265 | } |
| 266 | |
| 267 | if !force_network { |
| 268 | let current = status(); |
| 269 | if current.freshness == ModelsDevFreshness::Live |
| 270 | && current |
| 271 | .fetched_at |
| 272 | .is_some_and(|ts| now_unix().saturating_sub(ts) < DEFAULT_MODELS_DEV_TTL_SECS) |
| 273 | { |
| 274 | return Ok(current.offering_count); |
| 275 | } |
| 276 | } |
| 277 | |
| 278 | let url = resolve_catalog_url(); |
| 279 | let body = match fetch_catalog_body(&url).await { |
| 280 | Ok(body) => body, |
| 281 | Err(err) => { |
| 282 | mark_failed(err.clone()); |
| 283 | return Err(err); |
| 284 | } |
| 285 | }; |
| 286 | let fetched_at = now_unix(); |
| 287 | let fingerprint = base_url_fingerprint(&url); |
| 288 | let count = publish_from_body(&body, fetched_at, &url, ModelsDevFreshness::Live)?; |
| 289 | if let Some(path) = cache_path() { |
| 290 | save_cache_file( |
| 291 | &path, |
| 292 | &PersistedModelsDevCache { |
| 293 | schema_version: CACHE_SCHEMA_VERSION, |
| 294 | fetched_at, |
| 295 | source_fingerprint: fingerprint, |
| 296 | source_label: url, |
| 297 | body, |
| 298 | }, |
| 299 | ) |
| 300 | .inspect_err(|error| mark_failed(error.clone()))?; |
| 301 | } |
| 302 | Ok(count) |
| 303 | } |
| 304 | |
| 305 | /// Best-effort background refresh: never panics, never blocks callers. |
| 306 | pub fn spawn_background_refresh() { |
| 307 | if env_truthy(ENV_DISABLE_FETCH) && std::env::var(ENV_MODELS_DEV_PATH).is_err() { |
| 308 | return; |
| 309 | } |
| 310 | tokio::spawn(async { |
| 311 | match refresh(false).await { |
| 312 | Ok(count) => { |
| 313 | tracing::debug!( |
| 314 | target: "models_dev_live", |
| 315 | offering_count = count, |
| 316 | "Models.dev live catalog refreshed" |
| 317 | ); |
| 318 | } |
| 319 | Err(err) => { |
| 320 | tracing::debug!( |
| 321 | target: "models_dev_live", |
| 322 | error = %err, |
| 323 | "Models.dev live catalog refresh skipped" |
| 324 | ); |
| 325 | } |
| 326 | } |
| 327 | }); |
| 328 | } |
| 329 | |
| 330 | async fn refresh_from_path(path: &Path) -> Result<usize, ModelsDevRefreshError> { |
| 331 | let body = match tokio::fs::read_to_string(path).await { |
| 332 | Ok(body) => body, |
| 333 | Err(err) => { |
| 334 | let mapped = ModelsDevRefreshError::Io(err.to_string()); |
| 335 | mark_failed(mapped.clone()); |
| 336 | return Err(mapped); |
| 337 | } |
| 338 | }; |
| 339 | let fetched_at = now_unix(); |
| 340 | let label = format!("file:{}", path.display()); |
| 341 | let fingerprint = base_url_fingerprint(&label); |
| 342 | let count = publish_from_body(&body, fetched_at, &label, ModelsDevFreshness::Live)?; |
| 343 | if let Some(cache) = cache_path() { |
| 344 | save_cache_file( |
| 345 | &cache, |
| 346 | &PersistedModelsDevCache { |
| 347 | schema_version: CACHE_SCHEMA_VERSION, |
| 348 | fetched_at, |
| 349 | source_fingerprint: fingerprint, |
| 350 | source_label: label, |
| 351 | body, |
| 352 | }, |
| 353 | ) |
| 354 | .inspect_err(|error| mark_failed(error.clone()))?; |
| 355 | } |
| 356 | Ok(count) |
| 357 | } |
| 358 | |
| 359 | async fn fetch_catalog_body(url: &str) -> Result<String, ModelsDevRefreshError> { |
| 360 | let client = crate::tls::reqwest_client_builder() |
| 361 | .timeout(FETCH_TIMEOUT) |
| 362 | .connect_timeout(Duration::from_secs(10)) |
| 363 | .user_agent(USER_AGENT) |
| 364 | .build() |
| 365 | .map_err(|err| ModelsDevRefreshError::Network(err.to_string()))?; |
| 366 | |
| 367 | let response = client |
| 368 | .get(url) |
| 369 | .send() |
| 370 | .await |
| 371 | .map_err(|err| ModelsDevRefreshError::Network(err.to_string()))?; |
| 372 | |
| 373 | let status = response.status(); |
| 374 | if !status.is_success() { |
| 375 | return Err(ModelsDevRefreshError::HttpStatus(status.as_u16())); |
| 376 | } |
| 377 | |
| 378 | response |
| 379 | .text() |
| 380 | .await |
| 381 | .map_err(|err| ModelsDevRefreshError::Network(err.to_string())) |
| 382 | } |
| 383 | |
| 384 | fn publish_from_body( |
| 385 | body: &str, |
| 386 | fetched_at: u64, |
| 387 | source_label: &str, |
| 388 | freshness: ModelsDevFreshness, |
| 389 | ) -> Result<usize, ModelsDevRefreshError> { |
| 390 | let catalog = ModelsDevCatalog::parse_json(body).map_err(|err| { |
| 391 | let mapped = ModelsDevRefreshError::InvalidResponse(err.to_string()); |
| 392 | mark_failed(mapped.clone()); |
| 393 | mapped |
| 394 | })?; |
| 395 | // The source fingerprint scopes the *disk cache*, not the rows: a |
| 396 | // models.dev row describes a model, not an endpoint (`ModelsDevLive`). |
| 397 | let offerings = live_offerings_from_models_dev(&catalog, fetched_at); |
| 398 | if offerings.is_empty() { |
| 399 | let err = ModelsDevRefreshError::EmptyCatalog; |
| 400 | mark_failed(err.clone()); |
| 401 | return Err(err); |
| 402 | } |
| 403 | let count = offerings.len(); |
| 404 | crate::provider_lake::set_live_snapshot( |
| 405 | CatalogSnapshot { offerings }, |
| 406 | crate::provider_lake::LiveSource::ModelsDev, |
| 407 | ); |
| 408 | set_status(ModelsDevStatus { |
| 409 | freshness, |
| 410 | offering_count: count, |
| 411 | fetched_at: Some(fetched_at), |
| 412 | source_label: source_label.to_string(), |
| 413 | last_error: None, |
| 414 | }); |
| 415 | Ok(count) |
| 416 | } |
| 417 | |
| 418 | fn mark_failed(err: ModelsDevRefreshError) { |
| 419 | let mut next = status(); |
| 420 | // Keep prior offering_count / fetched_at so UI can still show the last |
| 421 | // rows, but mark the last refresh outcome distinctly from TTL staleness. |
| 422 | next.freshness = ModelsDevFreshness::Failed; |
| 423 | next.last_error = Some(err.to_string()); |
| 424 | set_status(next); |
| 425 | } |
| 426 | |
| 427 | /// Load the on-disk Models.dev cache. |
| 428 | /// |
| 429 | /// Reads v2 (single-parse: one header line + raw body) and v1 (JSON envelope |
| 430 | /// with an escaped body) formats. Returns metadata and the *unescaped* body |
| 431 | /// without copying it in the v2 path. |
| 432 | fn load_cache_file(path: &Path) -> Option<PersistedModelsDevCache> { |
| 433 | let bytes = std::fs::read(path).ok()?; |
| 434 | // v2: single-line JSON header terminated by a newline, then the verbatim |
| 435 | // catalog body. One small parse, zero body copies. |
| 436 | if bytes.first() == Some(&b'{') && bytes.contains(&b'\n') { |
| 437 | let split = bytes.iter().position(|b| *b == b'\n')?; |
| 438 | if let Ok(header) = serde_json::from_slice::<PersistedModelsDevCacheV2>(&bytes[..split]) |
| 439 | && header.schema_version == CACHE_SCHEMA_VERSION_V2 |
| 440 | && !bytes[split + 1..].is_empty() |
| 441 | { |
| 442 | let body = String::from_utf8(bytes[split + 1..].to_vec()).ok()?; |
| 443 | return Some(PersistedModelsDevCache { |
| 444 | schema_version: CACHE_SCHEMA_VERSION, |
| 445 | fetched_at: header.fetched_at, |
| 446 | source_fingerprint: header.source_fingerprint, |
| 447 | source_label: header.source_label, |
| 448 | body, |
| 449 | }); |
| 450 | } |
| 451 | } |
| 452 | // v1 fallback: whole-file JSON envelope with the body escaped inside. |
| 453 | let cache: PersistedModelsDevCache = serde_json::from_slice(&bytes).ok()?; |
| 454 | if cache.schema_version != CACHE_SCHEMA_VERSION { |
| 455 | return None; |
| 456 | } |
| 457 | if cache.body.trim().is_empty() { |
| 458 | return None; |
| 459 | } |
| 460 | Some(cache) |
| 461 | } |
| 462 | |
| 463 | fn save_cache_file( |
| 464 | path: &Path, |
| 465 | cache: &PersistedModelsDevCache, |
| 466 | ) -> Result<(), ModelsDevRefreshError> { |
| 467 | // Write the single-parse format so the next boot parses the catalog once. |
| 468 | let header = PersistedModelsDevCacheV2 { |
| 469 | schema_version: CACHE_SCHEMA_VERSION_V2, |
| 470 | fetched_at: cache.fetched_at, |
| 471 | source_fingerprint: cache.source_fingerprint.clone(), |
| 472 | source_label: cache.source_label.clone(), |
| 473 | }; |
| 474 | let mut header_line = |
| 475 | serde_json::to_vec(&header).map_err(|err| ModelsDevRefreshError::Io(err.to_string()))?; |
| 476 | header_line.push(b'\n'); |
| 477 | let mut payload = header_line; |
| 478 | payload.extend_from_slice(cache.body.as_bytes()); |
| 479 | atomic_write(path, &payload).map_err(|err| ModelsDevRefreshError::Io(err.to_string())) |
| 480 | } |
| 481 | |
| 482 | /// Compile helper exposed for unit tests: body → live offerings with normalized |
| 483 | /// provider ids. |
| 484 | #[cfg(test)] |
| 485 | pub(crate) fn offerings_from_json_for_test( |
| 486 | body: &str, |
| 487 | ) -> Result<Vec<codewhale_config::catalog::CatalogOffering>, String> { |
| 488 | let catalog = ModelsDevCatalog::parse_json(body).map_err(|e| e.to_string())?; |
| 489 | Ok(live_offerings_from_models_dev(&catalog, 1_700_000_000)) |
| 490 | } |
| 491 | |
| 492 | #[cfg(test)] |
| 493 | mod tests { |
| 494 | use super::*; |
| 495 | use crate::config::ApiProvider; |
| 496 | use crate::provider_lake::{ |
| 497 | all_catalog_models_for_provider, clear_live_snapshot, lock_live_snapshot, |
| 498 | }; |
| 499 | use crate::test_support::{EnvVarGuard, lock_test_env}; |
| 500 | use codewhale_config::catalog::CatalogSource; |
| 501 | |
| 502 | const FIXTURE: &str = r#"{ |
| 503 | "models": {}, |
| 504 | "providers": { |
| 505 | "togetherai": { |
| 506 | "id": "togetherai", |
| 507 | "models": { |
| 508 | "deepseek-ai/DeepSeek-V4-Pro": { |
| 509 | "id": "deepseek-ai/DeepSeek-V4-Pro", |
| 510 | "name": "DeepSeek V4 Pro", |
| 511 | "modalities": { "input": ["text"], "output": ["text"] }, |
| 512 | "limit": { "context": 128000, "output": 8192 } |
| 513 | } |
| 514 | } |
| 515 | }, |
| 516 | "moonshotai": { |
| 517 | "id": "moonshotai", |
| 518 | "models": { |
| 519 | "kimi-k2.5": { |
| 520 | "id": "kimi-k2.5", |
| 521 | "name": "Kimi K2.5", |
| 522 | "modalities": { "input": ["text"], "output": ["text"] }, |
| 523 | "limit": { "context": 256000, "output": 8192 } |
| 524 | } |
| 525 | } |
| 526 | }, |
| 527 | "unknown-gateway": { |
| 528 | "id": "unknown-gateway", |
| 529 | "models": { |
| 530 | "mystery-1": { |
| 531 | "id": "mystery-1", |
| 532 | "modalities": { "input": ["text"], "output": ["text"] } |
| 533 | } |
| 534 | } |
| 535 | } |
| 536 | } |
| 537 | }"#; |
| 538 | |
| 539 | /// An unguarded test must not resolve the developer's catalog cache. |
| 540 | /// |
| 541 | /// `resolve_state_dir` lives in `codewhale-config`, which is a plain |
| 542 | /// dependency here and cannot see this crate's isolated test root, so this |
| 543 | /// path had no equivalent of the settings confinement (#5359). A picker |
| 544 | /// test then rendered against whatever models the developer last fetched. |
| 545 | #[test] |
| 546 | fn unguarded_cache_path_stays_inside_the_isolated_test_root() { |
| 547 | let path = cache_path().expect("cache path"); |
| 548 | let isolated = crate::test_support::isolated_test_state_root(); |
| 549 | assert!( |
| 550 | path.starts_with(isolated), |
| 551 | "catalog cache escaped the isolated test root: {}", |
| 552 | path.display() |
| 553 | ); |
| 554 | assert_eq!(path.file_name().and_then(|n| n.to_str()), Some(CACHE_FILE)); |
| 555 | } |
| 556 | |
| 557 | /// A test that does seal the environment still resolves it, so the |
| 558 | /// confinement above cannot silently break the guarded callers. |
| 559 | #[test] |
| 560 | fn guarded_cache_path_follows_the_sealed_home() { |
| 561 | let _lock = lock_test_env(); |
| 562 | let home = tempfile::tempdir().expect("tempdir"); |
| 563 | let _guard = EnvVarGuard::set("CODEWHALE_HOME", home.path()); |
| 564 | |
| 565 | let path = cache_path().expect("cache path"); |
| 566 | |
| 567 | assert!( |
| 568 | path.starts_with(home.path()), |
| 569 | "sealed CODEWHALE_HOME was ignored: {}", |
| 570 | path.display() |
| 571 | ); |
| 572 | } |
| 573 | |
| 574 | #[test] |
| 575 | fn resolve_catalog_url_defaults_and_overrides() { |
| 576 | let _lock = lock_test_env(); |
| 577 | let _url = EnvVarGuard::remove(ENV_MODELS_DEV_URL); |
| 578 | assert_eq!(resolve_catalog_url(), MODELS_DEV_CATALOG_URL); |
| 579 | |
| 580 | let _url = EnvVarGuard::set(ENV_MODELS_DEV_URL, "https://example.test"); |
| 581 | assert_eq!(resolve_catalog_url(), "https://example.test/catalog.json"); |
| 582 | |
| 583 | let _url = EnvVarGuard::set(ENV_MODELS_DEV_URL, "https://example.test/api.json"); |
| 584 | assert_eq!(resolve_catalog_url(), "https://example.test/api.json"); |
| 585 | } |
| 586 | |
| 587 | #[test] |
| 588 | fn live_offerings_normalize_models_dev_provider_ids() { |
| 589 | let rows = offerings_from_json_for_test(FIXTURE).expect("fixture"); |
| 590 | let providers: Vec<_> = rows.iter().map(|r| r.provider.as_str()).collect(); |
| 591 | assert!(providers.contains(&"together")); |
| 592 | assert!(providers.contains(&"moonshot")); |
| 593 | assert!(providers.contains(&"unknown-gateway")); |
| 594 | assert!(!providers.contains(&"togetherai")); |
| 595 | assert!(!providers.contains(&"moonshotai")); |
| 596 | // Layer 10, not layer 20: a refresh of a public catalog is external |
| 597 | // enrichment about a model, not a provider's answer about an endpoint, |
| 598 | // so it stays correctable by the signed layer above it. |
| 599 | assert!( |
| 600 | rows.iter() |
| 601 | .all(|r| matches!(r.source, CatalogSource::ModelsDevLive { .. })) |
| 602 | ); |
| 603 | } |
| 604 | |
| 605 | #[test] |
| 606 | fn publish_from_path_updates_provider_lake() { |
| 607 | let _lock = lock_test_env(); |
| 608 | let _live = lock_live_snapshot(); |
| 609 | clear_live_snapshot(); |
| 610 | let dir = tempfile::tempdir().expect("tempdir"); |
| 611 | let path = dir.path().join("catalog.json"); |
| 612 | std::fs::write(&path, FIXTURE).expect("write fixture"); |
| 613 | |
| 614 | let _home = EnvVarGuard::set("CODEWHALE_HOME", dir.path().join("home")); |
| 615 | let _disable = EnvVarGuard::set(ENV_DISABLE_FETCH, "1"); |
| 616 | let _path = EnvVarGuard::set(ENV_MODELS_DEV_PATH, &path); |
| 617 | |
| 618 | let rt = tokio::runtime::Builder::new_current_thread() |
| 619 | .enable_all() |
| 620 | .build() |
| 621 | .expect("runtime"); |
| 622 | let count = rt.block_on(refresh(true)).expect("refresh from path"); |
| 623 | assert!(count >= 2); |
| 624 | |
| 625 | let together = all_catalog_models_for_provider(ApiProvider::Together); |
| 626 | assert!( |
| 627 | together.iter().any(|m| m == "deepseek-ai/DeepSeek-V4-Pro"), |
| 628 | "Together lake missing live Models.dev row: {together:?}" |
| 629 | ); |
| 630 | let moonshot = all_catalog_models_for_provider(ApiProvider::Moonshot); |
| 631 | assert!( |
| 632 | moonshot.iter().any(|m| m == "kimi-k2.5"), |
| 633 | "Moonshot lake missing live Models.dev row: {moonshot:?}" |
| 634 | ); |
| 635 | |
| 636 | let st = status(); |
| 637 | assert_eq!(st.freshness, ModelsDevFreshness::Live); |
| 638 | assert!(st.last_error.is_none()); |
| 639 | assert!(st.offering_count >= 2); |
| 640 | |
| 641 | // Cache file should exist and be secret-free. |
| 642 | let cache = cache_path().expect("cache path"); |
| 643 | assert!(cache.exists()); |
| 644 | let on_disk = std::fs::read_to_string(&cache).expect("read cache"); |
| 645 | let lowered = on_disk.to_lowercase(); |
| 646 | for needle in ["api_key", "authorization", "bearer", "password"] { |
| 647 | assert!( |
| 648 | !lowered.contains(&format!("\"{needle}\"")), |
| 649 | "cache must not persist `{needle}`" |
| 650 | ); |
| 651 | } |
| 652 | |
| 653 | clear_live_snapshot(); |
| 654 | } |
| 655 | |
| 656 | #[test] |
| 657 | fn bundled_staleness_flips_only_bundled_reports() { |
| 658 | let bundled = ModelsDevStatus::default(); |
| 659 | assert_eq!(bundled.freshness, ModelsDevFreshness::Bundled); |
| 660 | assert_eq!( |
| 661 | honor_bundled_staleness(bundled.clone(), true).freshness, |
| 662 | ModelsDevFreshness::Stale |
| 663 | ); |
| 664 | assert_eq!( |
| 665 | honor_bundled_staleness(bundled, false).freshness, |
| 666 | ModelsDevFreshness::Bundled |
| 667 | ); |
| 668 | let live = ModelsDevStatus { |
| 669 | freshness: ModelsDevFreshness::Live, |
| 670 | ..ModelsDevStatus::default() |
| 671 | }; |
| 672 | assert_eq!( |
| 673 | honor_bundled_staleness(live, true).freshness, |
| 674 | ModelsDevFreshness::Live |
| 675 | ); |
| 676 | } |
| 677 | |
| 678 | #[test] |
| 679 | fn invalid_json_keeps_bundled_and_marks_failed() { |
| 680 | let _lock = lock_test_env(); |
| 681 | let _live = lock_live_snapshot(); |
| 682 | clear_live_snapshot(); |
| 683 | let dir = tempfile::tempdir().expect("tempdir"); |
| 684 | let path = dir.path().join("bad.json"); |
| 685 | std::fs::write(&path, "{not-json").expect("write"); |
| 686 | |
| 687 | let _home = EnvVarGuard::set("CODEWHALE_HOME", dir.path().join("home")); |
| 688 | let _path = EnvVarGuard::set(ENV_MODELS_DEV_PATH, &path); |
| 689 | |
| 690 | let before = all_catalog_models_for_provider(ApiProvider::Together); |
| 691 | assert!(!before.is_empty(), "bundled Together rows required"); |
| 692 | |
| 693 | let rt = tokio::runtime::Builder::new_current_thread() |
| 694 | .enable_all() |
| 695 | .build() |
| 696 | .expect("runtime"); |
| 697 | let err = rt.block_on(refresh(true)).expect_err("bad json"); |
| 698 | assert!(matches!(err, ModelsDevRefreshError::InvalidResponse(_))); |
| 699 | |
| 700 | let after = all_catalog_models_for_provider(ApiProvider::Together); |
| 701 | assert_eq!(after, before, "bundled rows must survive parse failure"); |
| 702 | let st = status(); |
| 703 | assert_eq!(st.freshness, ModelsDevFreshness::Failed); |
| 704 | assert!(st.last_error.is_some()); |
| 705 | clear_live_snapshot(); |
| 706 | } |
| 707 | |
| 708 | #[test] |
| 709 | fn stale_disk_cache_still_publishes() { |
| 710 | let _lock = lock_test_env(); |
| 711 | let _live = lock_live_snapshot(); |
| 712 | clear_live_snapshot(); |
| 713 | let dir = tempfile::tempdir().expect("tempdir"); |
| 714 | let home = dir.path().join("home"); |
| 715 | let _home = EnvVarGuard::set("CODEWHALE_HOME", &home); |
| 716 | |
| 717 | let cache_dir = home.join("catalog"); |
| 718 | std::fs::create_dir_all(&cache_dir).expect("mkdir"); |
| 719 | let cache = cache_dir.join(CACHE_FILE); |
| 720 | let stale = PersistedModelsDevCache { |
| 721 | schema_version: CACHE_SCHEMA_VERSION, |
| 722 | fetched_at: 1, // far in the past → stale |
| 723 | source_fingerprint: "stale-fp".into(), |
| 724 | source_label: "https://models.dev/catalog.json".into(), |
| 725 | body: FIXTURE.into(), |
| 726 | }; |
| 727 | save_cache_file(&cache, &stale).expect("save"); |
| 728 | |
| 729 | maybe_load_persisted_cache(); |
| 730 | let st = status(); |
| 731 | assert_eq!(st.freshness, ModelsDevFreshness::Stale); |
| 732 | assert!(st.offering_count >= 2); |
| 733 | let together = all_catalog_models_for_provider(ApiProvider::Together); |
| 734 | assert!(together.iter().any(|m| m == "deepseek-ai/DeepSeek-V4-Pro")); |
| 735 | clear_live_snapshot(); |
| 736 | } |
| 737 | |
| 738 | #[test] |
| 739 | fn network_failure_keeps_prior_rows_and_marks_failed() { |
| 740 | let _lock = lock_test_env(); |
| 741 | let _live = lock_live_snapshot(); |
| 742 | clear_live_snapshot(); |
| 743 | let dir = tempfile::tempdir().expect("tempdir"); |
| 744 | let path = dir.path().join("catalog.json"); |
| 745 | std::fs::write(&path, FIXTURE).expect("write"); |
| 746 | |
| 747 | let _home = EnvVarGuard::set("CODEWHALE_HOME", dir.path().join("home")); |
| 748 | let _path = EnvVarGuard::set(ENV_MODELS_DEV_PATH, &path); |
| 749 | |
| 750 | let rt = tokio::runtime::Builder::new_current_thread() |
| 751 | .enable_all() |
| 752 | .build() |
| 753 | .expect("runtime"); |
| 754 | let count = rt.block_on(refresh(true)).expect("seed from path"); |
| 755 | assert!(count >= 2); |
| 756 | |
| 757 | // Point at a dead URL and force network (clear path override). |
| 758 | let _path = EnvVarGuard::remove(ENV_MODELS_DEV_PATH); |
| 759 | let _disable = EnvVarGuard::remove(ENV_DISABLE_FETCH); |
| 760 | let _url = EnvVarGuard::set(ENV_MODELS_DEV_URL, "http://127.0.0.1:1"); |
| 761 | |
| 762 | let err = rt.block_on(refresh(true)).expect_err("dead URL"); |
| 763 | assert!(matches!(err, ModelsDevRefreshError::Network(_))); |
| 764 | |
| 765 | let together = all_catalog_models_for_provider(ApiProvider::Together); |
| 766 | assert!( |
| 767 | together.iter().any(|m| m == "deepseek-ai/DeepSeek-V4-Pro"), |
| 768 | "prior live rows must survive network failure" |
| 769 | ); |
| 770 | let st = status(); |
| 771 | assert_eq!(st.freshness, ModelsDevFreshness::Failed); |
| 772 | assert!(st.last_error.is_some()); |
| 773 | assert!( |
| 774 | st.offering_count >= 2, |
| 775 | "status should retain prior live row count after failure" |
| 776 | ); |
| 777 | clear_live_snapshot(); |
| 778 | } |
| 779 | } |
| 780 |