| 1 | //! `model resolve` must report the route the runtime would actually take. |
| 2 | //! |
| 3 | //! Regression coverage for #4832, where a Z.ai config reported |
| 4 | //! `provider: deepseek` because the subcommand read only the CLI flags and |
| 5 | //! never consulted the resolved runtime. A diagnostic that confidently |
| 6 | //! reports the wrong provider is worse than one that reports nothing, so |
| 7 | //! every provider is asserted here rather than DeepSeek alone. |
| 8 | |
| 9 | use std::collections::BTreeMap; |
| 10 | use std::fs; |
| 11 | use std::path::PathBuf; |
| 12 | use std::process::Command; |
| 13 | |
| 14 | use tempfile::TempDir; |
| 15 | |
| 16 | /// Run `model resolve` against a sealed HOME containing `config`. |
| 17 | /// |
| 18 | /// `env_clear` plus a temporary HOME keeps this off the real |
| 19 | /// `~/.codewhale/config.toml`; the suite has written to real user state before |
| 20 | /// (#4831) and this test must never be the one that does it again. |
| 21 | fn resolve_with_config(config: &str, args: &[&str]) -> BTreeMap<String, String> { |
| 22 | let fixture = TempDir::new().expect("fixture root"); |
| 23 | let home = fixture.path().join("sealed-home"); |
| 24 | fs::create_dir_all(home.join(".codewhale")).expect("sealed config dir"); |
| 25 | fs::write(home.join(".codewhale").join("config.toml"), config).expect("seed config"); |
| 26 | |
| 27 | let mut command = Command::new(codewhale_binary()); |
| 28 | command.arg("model").arg("resolve").args(args); |
| 29 | let output = command |
| 30 | .env_clear() |
| 31 | .env("HOME", &home) |
| 32 | .env("USERPROFILE", &home) |
| 33 | .env("CODEWHALE_HOME", home.join(".codewhale")) |
| 34 | .env("CODEWHALE_SECRET_BACKEND", "file") |
| 35 | .output() |
| 36 | .expect("run model resolve"); |
| 37 | |
| 38 | assert!( |
| 39 | output.status.success(), |
| 40 | "model resolve {args:?} failed\nstdout:\n{}\nstderr:\n{}", |
| 41 | String::from_utf8_lossy(&output.stdout), |
| 42 | String::from_utf8_lossy(&output.stderr) |
| 43 | ); |
| 44 | |
| 45 | String::from_utf8_lossy(&output.stdout) |
| 46 | .lines() |
| 47 | .filter_map(|line| line.split_once(": ")) |
| 48 | .map(|(key, value)| (key.trim().to_string(), value.trim().to_string())) |
| 49 | .collect() |
| 50 | } |
| 51 | |
| 52 | /// Run a model query that must fail without reading ambient configuration or |
| 53 | /// credentials. Keeping the raw output lets the regression prove the CLI did |
| 54 | /// not print a fabricated provider route before exiting. |
| 55 | fn resolve_failure_with_config(config: &str, args: &[&str]) -> std::process::Output { |
| 56 | let fixture = TempDir::new().expect("fixture root"); |
| 57 | let home = fixture.path().join("sealed-home"); |
| 58 | fs::create_dir_all(home.join(".codewhale")).expect("sealed config dir"); |
| 59 | fs::write(home.join(".codewhale").join("config.toml"), config).expect("seed config"); |
| 60 | |
| 61 | Command::new(codewhale_binary()) |
| 62 | .arg("model") |
| 63 | .arg("resolve") |
| 64 | .args(args) |
| 65 | .env_clear() |
| 66 | .env("HOME", &home) |
| 67 | .env("USERPROFILE", &home) |
| 68 | .env("CODEWHALE_HOME", home.join(".codewhale")) |
| 69 | .env("CODEWHALE_SECRET_BACKEND", "file") |
| 70 | .output() |
| 71 | .expect("run failing model resolve") |
| 72 | } |
| 73 | |
| 74 | #[test] |
| 75 | fn resolve_reports_the_configured_provider_not_a_deepseek_fallback() { |
| 76 | let report = resolve_with_config( |
| 77 | "provider = \"zai\"\n\n[providers.zai]\napi_key = \"k\"\n", |
| 78 | &[], |
| 79 | ); |
| 80 | |
| 81 | assert_eq!( |
| 82 | report.get("provider").map(String::as_str), |
| 83 | Some("zai"), |
| 84 | "configured provider must survive to the diagnostic: {report:?}" |
| 85 | ); |
| 86 | assert_eq!( |
| 87 | report.get("provider_source").map(String::as_str), |
| 88 | Some("config"), |
| 89 | "provenance must name the config file: {report:?}" |
| 90 | ); |
| 91 | } |
| 92 | |
| 93 | #[test] |
| 94 | fn resolve_reports_a_provider_scoped_model_as_explicitly_configured() { |
| 95 | let report = resolve_with_config( |
| 96 | "provider = \"moonshot\"\n\n[providers.moonshot]\napi_key = \"k\"\nmodel = \"kimi-k3-turbo\"\n", |
| 97 | &[], |
| 98 | ); |
| 99 | |
| 100 | assert_eq!(report.get("provider").map(String::as_str), Some("moonshot")); |
| 101 | assert_eq!( |
| 102 | report.get("requested").map(String::as_str), |
| 103 | Some("kimi-k3-turbo"), |
| 104 | "a configured model is a request, not a fallback: {report:?}" |
| 105 | ); |
| 106 | assert_eq!( |
| 107 | report.get("used_fallback").map(String::as_str), |
| 108 | Some("false"), |
| 109 | "{report:?}" |
| 110 | ); |
| 111 | assert_eq!( |
| 112 | report.get("model_source").map(String::as_str), |
| 113 | Some("config [providers.*].model"), |
| 114 | "{report:?}" |
| 115 | ); |
| 116 | } |
| 117 | |
| 118 | #[test] |
| 119 | fn resolve_admits_when_nothing_was_configured() { |
| 120 | // The honest answer to "what did the user ask for" is "nothing". The |
| 121 | // built-in default may still be shown, but it must be labelled as ours. |
| 122 | let report = resolve_with_config("", &[]); |
| 123 | |
| 124 | assert_eq!( |
| 125 | report.get("requested").map(String::as_str), |
| 126 | Some(""), |
| 127 | "an unconfigured model must not be presented as a request: {report:?}" |
| 128 | ); |
| 129 | assert_eq!( |
| 130 | report.get("used_fallback").map(String::as_str), |
| 131 | Some("true"), |
| 132 | "{report:?}" |
| 133 | ); |
| 134 | assert_eq!( |
| 135 | report.get("model_source").map(String::as_str), |
| 136 | Some("provider default"), |
| 137 | "{report:?}" |
| 138 | ); |
| 139 | } |
| 140 | |
| 141 | #[test] |
| 142 | fn a_foreign_model_argument_cannot_switch_the_configured_provider() { |
| 143 | let output = resolve_failure_with_config( |
| 144 | "provider = \"zai\"\n\n[providers.zai]\napi_key = \"k\"\n", |
| 145 | &["deepseek-v4-flash"], |
| 146 | ); |
| 147 | |
| 148 | assert!(!output.status.success()); |
| 149 | let combined = format!( |
| 150 | "{}\n{}", |
| 151 | String::from_utf8_lossy(&output.stdout), |
| 152 | String::from_utf8_lossy(&output.stderr) |
| 153 | ) |
| 154 | .to_ascii_lowercase(); |
| 155 | assert!( |
| 156 | combined.contains("not available from provider 'zai'"), |
| 157 | "{combined}" |
| 158 | ); |
| 159 | assert!(!combined.contains("provider: deepseek"), "{combined}"); |
| 160 | } |
| 161 | |
| 162 | #[test] |
| 163 | fn an_explicit_matching_provider_can_resolve_its_model() { |
| 164 | let report = resolve_with_config( |
| 165 | "provider = \"zai\"\n\n[providers.zai]\napi_key = \"k\"\n", |
| 166 | &["deepseek-v4-flash", "--provider", "deepseek"], |
| 167 | ); |
| 168 | |
| 169 | assert_eq!(report.get("provider").map(String::as_str), Some("deepseek")); |
| 170 | assert_eq!( |
| 171 | report.get("requested").map(String::as_str), |
| 172 | Some("deepseek-v4-flash") |
| 173 | ); |
| 174 | assert_eq!( |
| 175 | report.get("used_fallback").map(String::as_str), |
| 176 | Some("false") |
| 177 | ); |
| 178 | assert_eq!( |
| 179 | report.get("provider_source").map(String::as_str), |
| 180 | Some("--provider") |
| 181 | ); |
| 182 | } |
| 183 | |
| 184 | #[test] |
| 185 | fn unknown_providerless_model_fails_without_printing_a_deepseek_route() { |
| 186 | let output = resolve_failure_with_config( |
| 187 | "provider = \"zai\"\n\n[providers.zai]\napi_key = \"k\"\n", |
| 188 | &["totally-unknown-model"], |
| 189 | ); |
| 190 | |
| 191 | assert!(!output.status.success(), "unknown model must fail closed"); |
| 192 | let stdout = String::from_utf8_lossy(&output.stdout).to_ascii_lowercase(); |
| 193 | let stderr = String::from_utf8_lossy(&output.stderr).to_ascii_lowercase(); |
| 194 | let combined = format!("{stdout}\n{stderr}"); |
| 195 | assert!( |
| 196 | combined.contains("not available from provider 'zai'"), |
| 197 | "{combined}" |
| 198 | ); |
| 199 | assert!(!combined.contains("provider: deepseek"), "{combined}"); |
| 200 | assert!(!combined.contains("deepseek-v4-pro"), "{combined}"); |
| 201 | } |
| 202 | |
| 203 | #[test] |
| 204 | fn an_explicit_provider_flag_is_reported_as_the_source() { |
| 205 | let report = resolve_with_config( |
| 206 | "provider = \"zai\"\n\n[providers.zai]\napi_key = \"k\"\n", |
| 207 | &["--provider", "moonshot"], |
| 208 | ); |
| 209 | |
| 210 | assert_eq!(report.get("provider").map(String::as_str), Some("moonshot")); |
| 211 | assert_eq!( |
| 212 | report.get("provider_source").map(String::as_str), |
| 213 | Some("--provider"), |
| 214 | "{report:?}" |
| 215 | ); |
| 216 | } |
| 217 | |
| 218 | #[test] |
| 219 | fn explicit_openai_without_a_model_reports_its_own_documented_default() { |
| 220 | let report = resolve_with_config( |
| 221 | "provider = \"zai\"\n\n[providers.zai]\napi_key = \"k\"\nmodel = \"GLM-5.2\"\n", |
| 222 | &["--provider", "openai"], |
| 223 | ); |
| 224 | |
| 225 | assert_eq!(report.get("provider").map(String::as_str), Some("openai")); |
| 226 | assert_eq!( |
| 227 | report.get("resolved").map(String::as_str), |
| 228 | Some("gpt-5.6"), |
| 229 | "an OpenAI query must use OpenAI's documented default, not the first catalog row or the configured Z.ai model: {report:?}" |
| 230 | ); |
| 231 | assert_eq!( |
| 232 | report.get("used_fallback").map(String::as_str), |
| 233 | Some("true"), |
| 234 | "{report:?}" |
| 235 | ); |
| 236 | assert_eq!( |
| 237 | report.get("provider_source").map(String::as_str), |
| 238 | Some("--provider"), |
| 239 | "{report:?}" |
| 240 | ); |
| 241 | assert_eq!( |
| 242 | report.get("model_source").map(String::as_str), |
| 243 | Some("provider default"), |
| 244 | "the overridden Z.ai model provenance must not leak into the OpenAI hypothetical: {report:?}" |
| 245 | ); |
| 246 | } |
| 247 | |
| 248 | /// Run `model resolve` with global flags placed before the subcommand, which |
| 249 | /// is where `--provider` / `--model` actually go. |
| 250 | fn resolve_with_global_flags( |
| 251 | config: &str, |
| 252 | global: &[&str], |
| 253 | args: &[&str], |
| 254 | ) -> BTreeMap<String, String> { |
| 255 | let fixture = TempDir::new().expect("fixture root"); |
| 256 | let home = fixture.path().join("sealed-home"); |
| 257 | fs::create_dir_all(home.join(".codewhale")).expect("sealed config dir"); |
| 258 | fs::write(home.join(".codewhale").join("config.toml"), config).expect("seed config"); |
| 259 | |
| 260 | let mut command = Command::new(codewhale_binary()); |
| 261 | command.args(global).arg("model").arg("resolve").args(args); |
| 262 | let output = command |
| 263 | .env_clear() |
| 264 | .env("HOME", &home) |
| 265 | .env("USERPROFILE", &home) |
| 266 | .env("CODEWHALE_HOME", home.join(".codewhale")) |
| 267 | .env("CODEWHALE_SECRET_BACKEND", "file") |
| 268 | .output() |
| 269 | .expect("run model resolve"); |
| 270 | |
| 271 | assert!( |
| 272 | output.status.success(), |
| 273 | "model resolve {global:?} {args:?} failed\nstdout:\n{}\nstderr:\n{}", |
| 274 | String::from_utf8_lossy(&output.stdout), |
| 275 | String::from_utf8_lossy(&output.stderr) |
| 276 | ); |
| 277 | |
| 278 | String::from_utf8_lossy(&output.stdout) |
| 279 | .lines() |
| 280 | .filter_map(|line| line.split_once(": ")) |
| 281 | .map(|(key, value)| (key.trim().to_string(), value.trim().to_string())) |
| 282 | .collect() |
| 283 | } |
| 284 | |
| 285 | /// v0.9.1 kimi-k3 dogfood report: `codewhale --provider moonshot --model kimi-k3 model resolve` |
| 286 | /// reported `kimi-k2.7-code`. The top-level flags are the route this process |
| 287 | /// is on, not a hypothetical, so the diagnostic has to answer with the runtime |
| 288 | /// resolution instead of re-deriving a registry default and ignoring `--model`. |
| 289 | #[test] |
| 290 | fn top_level_provider_and_model_flags_report_the_runtime_route() { |
| 291 | let report = resolve_with_global_flags( |
| 292 | "provider = \"zai\"\n\n[providers.zai]\napi_key = \"k\"\n", |
| 293 | &["--provider", "moonshot", "--model", "kimi-k3"], |
| 294 | &[], |
| 295 | ); |
| 296 | |
| 297 | assert_eq!(report.get("provider").map(String::as_str), Some("moonshot")); |
| 298 | assert_eq!( |
| 299 | report.get("resolved").map(String::as_str), |
| 300 | Some("kimi-k3"), |
| 301 | "the diagnostic must not contradict the model the run will use: {report:?}" |
| 302 | ); |
| 303 | assert_eq!( |
| 304 | report.get("requested").map(String::as_str), |
| 305 | Some("kimi-k3"), |
| 306 | "{report:?}" |
| 307 | ); |
| 308 | assert_eq!( |
| 309 | report.get("used_fallback").map(String::as_str), |
| 310 | Some("false"), |
| 311 | "{report:?}" |
| 312 | ); |
| 313 | assert_eq!( |
| 314 | report.get("model_source").map(String::as_str), |
| 315 | Some("--model"), |
| 316 | "{report:?}" |
| 317 | ); |
| 318 | } |
| 319 | |
| 320 | /// Moonshot ships `kimi-k3` on the direct platform API and `k3` on the Kimi |
| 321 | /// Code coding-plan API. Both must resolve, and neither may be answered by |
| 322 | /// another provider's identically named model (OpenCode Go also serves a |
| 323 | /// `kimi-k3`). |
| 324 | #[test] |
| 325 | fn moonshot_k3_products_resolve_without_crossing_providers() { |
| 326 | for model in ["kimi-k3", "k3"] { |
| 327 | let report = resolve_with_global_flags( |
| 328 | "provider = \"moonshot\"\n\n[providers.moonshot]\napi_key = \"k\"\n", |
| 329 | &[], |
| 330 | &[model, "--provider", "moonshot"], |
| 331 | ); |
| 332 | |
| 333 | assert_eq!( |
| 334 | report.get("provider").map(String::as_str), |
| 335 | Some("moonshot"), |
| 336 | "a Moonshot question must not be answered by another provider: {report:?}" |
| 337 | ); |
| 338 | assert_eq!( |
| 339 | report.get("resolved").map(String::as_str), |
| 340 | Some(model), |
| 341 | "{report:?}" |
| 342 | ); |
| 343 | assert_eq!( |
| 344 | report.get("used_fallback").map(String::as_str), |
| 345 | Some("false"), |
| 346 | "{report:?}" |
| 347 | ); |
| 348 | } |
| 349 | } |
| 350 | |
| 351 | /// An id the selected provider cannot serve must fail closed. Falling back |
| 352 | /// after a concrete request would silently run a different model. |
| 353 | #[test] |
| 354 | fn an_unservable_model_on_the_selected_provider_is_rejected() { |
| 355 | let output = resolve_failure_with_config( |
| 356 | "provider = \"moonshot\"\n\n[providers.moonshot]\napi_key = \"k\"\n", |
| 357 | &["glm-5.2", "--provider", "moonshot"], |
| 358 | ); |
| 359 | |
| 360 | assert!(!output.status.success()); |
| 361 | let combined = format!( |
| 362 | "{}\n{}", |
| 363 | String::from_utf8_lossy(&output.stdout), |
| 364 | String::from_utf8_lossy(&output.stderr) |
| 365 | ) |
| 366 | .to_ascii_lowercase(); |
| 367 | assert!( |
| 368 | combined.contains("model 'glm-5.2' is not available from provider 'moonshot'"), |
| 369 | "{combined}" |
| 370 | ); |
| 371 | } |
| 372 | |
| 373 | /// Adding a model to the catalog must make it servable on the provider that |
| 374 | /// carries it and nowhere else. `glm-5.3` was added as a peer of `glm-5.2`, so |
| 375 | /// it has to answer on Z.ai without a fallback while a Moonshot-scoped question |
| 376 | /// still refuses it — the same cross-provider boundary the `glm-5.2` case above |
| 377 | /// pins, asserted on the newest sibling so the boundary cannot rot as the |
| 378 | /// family grows. |
| 379 | #[test] |
| 380 | fn a_new_glm_sibling_is_servable_on_zai_but_not_on_moonshot() { |
| 381 | let served = resolve_with_global_flags( |
| 382 | "provider = \"zai\"\n\n[providers.zai]\napi_key = \"k\"\n", |
| 383 | &[], |
| 384 | &["glm-5.3", "--provider", "zai"], |
| 385 | ); |
| 386 | |
| 387 | assert_eq!(served.get("provider").map(String::as_str), Some("zai")); |
| 388 | assert_eq!( |
| 389 | served.get("resolved").map(String::as_str), |
| 390 | Some("GLM-5.3"), |
| 391 | "a catalogued model must resolve to itself, not to the provider default: {served:?}" |
| 392 | ); |
| 393 | assert_eq!( |
| 394 | served.get("used_fallback").map(String::as_str), |
| 395 | Some("false"), |
| 396 | "a model the provider serves must not be reported as a fallback: {served:?}" |
| 397 | ); |
| 398 | |
| 399 | let refused = resolve_failure_with_config( |
| 400 | "provider = \"moonshot\"\n\n[providers.moonshot]\napi_key = \"k\"\n", |
| 401 | &["glm-5.3", "--provider", "moonshot"], |
| 402 | ); |
| 403 | |
| 404 | assert!(!refused.status.success()); |
| 405 | let resolved = format!( |
| 406 | "{}\n{}", |
| 407 | String::from_utf8_lossy(&refused.stdout), |
| 408 | String::from_utf8_lossy(&refused.stderr) |
| 409 | ) |
| 410 | .to_ascii_lowercase(); |
| 411 | assert!( |
| 412 | resolved.contains("model 'glm-5.3' is not available from provider 'moonshot'"), |
| 413 | "a provider that cannot serve GLM must fail instead of choosing its default: {resolved}" |
| 414 | ); |
| 415 | } |
| 416 | |
| 417 | /// The OpenRouter sibling carries a different wire id (`z-ai/glm-5.3`) than the |
| 418 | /// direct Z.ai row (`GLM-5.3`), so the bare family alias has to be rewritten |
| 419 | /// per provider rather than passed through. This pins the OpenRouter half of |
| 420 | /// that rewrite, which the Z.ai case above cannot observe, and pins that adding |
| 421 | /// the sibling left the OpenRouter default alone. |
| 422 | #[test] |
| 423 | fn the_openrouter_glm_sibling_resolves_to_its_own_gateway_wire_id() { |
| 424 | let served = resolve_with_global_flags( |
| 425 | "provider = \"openrouter\"\n\n[providers.openrouter]\napi_key = \"k\"\n", |
| 426 | &[], |
| 427 | &["glm-5.3", "--provider", "openrouter"], |
| 428 | ); |
| 429 | |
| 430 | assert_eq!( |
| 431 | served.get("provider").map(String::as_str), |
| 432 | Some("openrouter") |
| 433 | ); |
| 434 | assert_eq!( |
| 435 | served.get("resolved").map(String::as_str), |
| 436 | Some("z-ai/glm-5.3"), |
| 437 | "the bare alias must be rewritten to the OpenRouter wire id, not passed through: {served:?}" |
| 438 | ); |
| 439 | assert_eq!( |
| 440 | served.get("used_fallback").map(String::as_str), |
| 441 | Some("false"), |
| 442 | "a gateway row the provider serves must not be reported as a fallback: {served:?}" |
| 443 | ); |
| 444 | |
| 445 | let default_route = resolve_with_config( |
| 446 | "provider = \"openrouter\"\n\n[providers.openrouter]\napi_key = \"k\"\n", |
| 447 | &[], |
| 448 | ); |
| 449 | let resolved = default_route |
| 450 | .get("resolved") |
| 451 | .map(String::as_str) |
| 452 | .unwrap_or_default(); |
| 453 | assert!( |
| 454 | !resolved.to_ascii_lowercase().contains("glm"), |
| 455 | "adding a GLM sibling must not make GLM the OpenRouter default: {default_route:?}" |
| 456 | ); |
| 457 | } |
| 458 | |
| 459 | /// A Z.ai config that names no model lands on the deliberate default, |
| 460 | /// `GLM-5.3`, with `provider default` provenance. This is the surface where a |
| 461 | /// default move would otherwise change silently under a user. |
| 462 | #[test] |
| 463 | fn zai_default_route_resolves_to_glm_5_3_with_provider_default_provenance() { |
| 464 | let report = resolve_with_config( |
| 465 | "provider = \"zai\"\n\n[providers.zai]\napi_key = \"k\"\n", |
| 466 | &[], |
| 467 | ); |
| 468 | |
| 469 | assert_eq!( |
| 470 | report.get("resolved").map(String::as_str), |
| 471 | Some("GLM-5.3"), |
| 472 | "the Z.ai default is GLM-5.3: {report:?}" |
| 473 | ); |
| 474 | assert_eq!( |
| 475 | report.get("model_source").map(String::as_str), |
| 476 | Some("provider default"), |
| 477 | "{report:?}" |
| 478 | ); |
| 479 | } |
| 480 | |
| 481 | /// An explicit `GLM-5.2` selection keeps its own id after the default moved |
| 482 | /// to `GLM-5.3`: only the default changed, never a user's saved route. |
| 483 | #[test] |
| 484 | fn explicit_glm_5_2_selection_survives_the_default_move() { |
| 485 | let report = resolve_with_config( |
| 486 | "provider = \"zai\"\n\n[providers.zai]\napi_key = \"k\"\nmodel = \"GLM-5.2\"\n", |
| 487 | &[], |
| 488 | ); |
| 489 | |
| 490 | assert_eq!( |
| 491 | report.get("resolved").map(String::as_str), |
| 492 | Some("GLM-5.2"), |
| 493 | "an explicit GLM-5.2 route must not be upgraded: {report:?}" |
| 494 | ); |
| 495 | assert_ne!( |
| 496 | report.get("model_source").map(String::as_str), |
| 497 | Some("provider default"), |
| 498 | "{report:?}" |
| 499 | ); |
| 500 | } |
| 501 | |
| 502 | fn codewhale_binary() -> PathBuf { |
| 503 | if let Some(path) = option_env!("CARGO_BIN_EXE_codewhale") { |
| 504 | return PathBuf::from(path); |
| 505 | } |
| 506 | if let Ok(path) = std::env::var("CARGO_BIN_EXE_codewhale") { |
| 507 | return PathBuf::from(path); |
| 508 | } |
| 509 | |
| 510 | let mut path = std::env::current_exe().expect("current test executable path"); |
| 511 | path.pop(); |
| 512 | if path.ends_with("deps") { |
| 513 | path.pop(); |
| 514 | } |
| 515 | path.push(format!("codewhale{}", std::env::consts::EXE_SUFFIX)); |
| 516 | path |
| 517 | } |
| 518 |