| 1 | //! Worker route **preflight**: everything about a route that must be true and |
| 2 | //! frozen *before* a Workflow starts, and certainly before any Router is |
| 3 | //! asked anything. |
| 4 | //! |
| 5 | //! An exact Fleet's promise is that the saved provider/model is the one that |
| 6 | //! runs. That promise is only worth something if it is *checked* at the point |
| 7 | //! the run is admitted, not discovered at the first API call: |
| 8 | //! |
| 9 | //! - **Provider identity** — the exact configured provider key and its kind. |
| 10 | //! - **Canonical wire model** — the model string that will actually be placed |
| 11 | //! on the request. Receipt and child spawn must use *this* value, not the |
| 12 | //! file's spelling of it, or the receipt describes a request nobody made. |
| 13 | //! - **Credential / readiness** — decided **locally**, from configuration. No |
| 14 | //! live probe: a preflight that hits the network would spend money and leak |
| 15 | //! the fact of the run before the operator's gates have even been evaluated. |
| 16 | //! Keyless local providers (`vllm`, `ollama`, `sglang`, …) are |
| 17 | //! [`CredentialReadiness::KeylessLocal`] and are perfectly valid. |
| 18 | //! - **Endpoint identity** — a non-secret label for *where* the request goes, |
| 19 | //! so two members pointed at different deployments of the same model id are |
| 20 | //! distinguishable on a receipt. Never a full URL with credentials in it. |
| 21 | //! - **Reasoning capability** — what the route can truthfully express, derived |
| 22 | //! once here so a later launch cannot invent one. |
| 23 | //! |
| 24 | //! Everything in this module is a plain value with no clock, no filesystem, and |
| 25 | //! no network. The host supplies the facts; this crate defines their shape and |
| 26 | //! the invariants over them. |
| 27 | |
| 28 | use serde::{Deserialize, Serialize}; |
| 29 | use thiserror::Error; |
| 30 | |
| 31 | use crate::fleet_exact::FrozenRoute; |
| 32 | use crate::fleet_reasoning::ReasoningCapability; |
| 33 | |
| 34 | /// Whether a route can be called at all, decided from local configuration. |
| 35 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] |
| 36 | #[serde(rename_all = "snake_case", tag = "kind")] |
| 37 | pub enum CredentialReadiness { |
| 38 | /// A credential for this provider is configured on this machine. |
| 39 | Configured, |
| 40 | /// The provider is a local, keyless endpoint. Valid, and not a downgrade. |
| 41 | KeylessLocal, |
| 42 | /// No credential is configured. The route cannot run. |
| 43 | Missing { detail: String }, |
| 44 | } |
| 45 | |
| 46 | impl CredentialReadiness { |
| 47 | #[must_use] |
| 48 | pub const fn is_ready(&self) -> bool { |
| 49 | matches!(self, Self::Configured | Self::KeylessLocal) |
| 50 | } |
| 51 | |
| 52 | #[must_use] |
| 53 | pub const fn as_str(&self) -> &'static str { |
| 54 | match self { |
| 55 | Self::Configured => "configured", |
| 56 | Self::KeylessLocal => "keyless_local", |
| 57 | Self::Missing { .. } => "missing", |
| 58 | } |
| 59 | } |
| 60 | } |
| 61 | |
| 62 | /// A non-secret identity for the endpoint a route talks to. |
| 63 | /// |
| 64 | /// Deliberately **not** a base URL: a configured base URL can carry a token in |
| 65 | /// its path or query, and receipts are durable. Host plus a coarse path label is |
| 66 | /// enough to tell two deployments apart, which is the only thing this is for. |
| 67 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] |
| 68 | pub struct EndpointIdentity { |
| 69 | /// Host (and port, when non-default), lowercased. Never credentials. |
| 70 | pub host: String, |
| 71 | /// Whether the endpoint resolves to loopback / a private address. |
| 72 | pub local: bool, |
| 73 | } |
| 74 | |
| 75 | impl EndpointIdentity { |
| 76 | /// Build an endpoint identity from a base URL, keeping only the host. |
| 77 | /// |
| 78 | /// Parsing is deliberately minimal and dependency-free: strip the scheme, |
| 79 | /// drop anything before an `@` (which is exactly where a credential would |
| 80 | /// live), then keep the authority up to the first `/`. |
| 81 | #[must_use] |
| 82 | pub fn from_base_url(base_url: &str) -> Self { |
| 83 | let without_scheme = base_url |
| 84 | .trim() |
| 85 | .split_once("://") |
| 86 | .map_or(base_url.trim(), |(_, rest)| rest); |
| 87 | let authority = without_scheme |
| 88 | .split(['/', '?', '#']) |
| 89 | .next() |
| 90 | .unwrap_or_default(); |
| 91 | // `user:password@host` — everything before the `@` is a credential. |
| 92 | let host = authority |
| 93 | .rsplit_once('@') |
| 94 | .map_or(authority, |(_, host)| host) |
| 95 | .to_ascii_lowercase(); |
| 96 | let bare = host.split(':').next().unwrap_or(&host); |
| 97 | let local = bare == "localhost" |
| 98 | || bare == "127.0.0.1" |
| 99 | || bare == "::1" |
| 100 | || bare.starts_with("192.168.") |
| 101 | || bare.starts_with("10.") |
| 102 | || bare.ends_with(".local"); |
| 103 | Self { host, local } |
| 104 | } |
| 105 | |
| 106 | /// The compact receipt form. |
| 107 | #[must_use] |
| 108 | pub fn label(&self) -> String { |
| 109 | if self.local { |
| 110 | format!("{} (local)", self.host) |
| 111 | } else { |
| 112 | self.host.clone() |
| 113 | } |
| 114 | } |
| 115 | } |
| 116 | |
| 117 | /// One worker's route, fully preflighted and frozen. |
| 118 | /// |
| 119 | /// Constructed once, at Workflow start. A launch reads it; nothing rewrites |
| 120 | /// it. The `wire_model` here is the single source of truth for both the |
| 121 | /// receipt and the child spawn. |
| 122 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] |
| 123 | pub struct PreflightedRoute { |
| 124 | /// The member this route belongs to. |
| 125 | pub member_id: String, |
| 126 | /// Exact configured provider key, as the operator named it. |
| 127 | pub provider_id: String, |
| 128 | /// Provider kind (`zai`, `openai`, `deepseek`, `vllm`, …). |
| 129 | pub provider_kind: String, |
| 130 | /// The model id exactly as saved in the Fleet file. |
| 131 | pub declared_model: String, |
| 132 | /// The canonical model string that will be placed on the wire. Receipt and |
| 133 | /// child spawn both use this. |
| 134 | pub wire_model: String, |
| 135 | /// Where the request goes. |
| 136 | pub endpoint: EndpointIdentity, |
| 137 | /// Locally decided readiness. Never a live probe. |
| 138 | pub credential: CredentialReadiness, |
| 139 | /// What the route can truthfully express about reasoning. |
| 140 | pub capability: ReasoningCapability, |
| 141 | } |
| 142 | |
| 143 | impl PreflightedRoute { |
| 144 | /// The frozen provider/model pair, in canonical wire form. |
| 145 | /// |
| 146 | /// This is what a receipt records and what a child spawns with — the two |
| 147 | /// cannot disagree because there is only one value. |
| 148 | #[must_use] |
| 149 | pub fn frozen(&self) -> FrozenRoute { |
| 150 | FrozenRoute { |
| 151 | provider: self.provider_id.clone(), |
| 152 | model: self.wire_model.clone(), |
| 153 | } |
| 154 | } |
| 155 | |
| 156 | /// Whether the declared model string differed from the canonical wire form. |
| 157 | /// Recorded rather than hidden: `glm-5` resolving to `glm-5-20260101` is a |
| 158 | /// fact the operator should be able to see on a receipt. |
| 159 | #[must_use] |
| 160 | pub fn model_canonicalized(&self) -> bool { |
| 161 | self.declared_model != self.wire_model |
| 162 | } |
| 163 | |
| 164 | /// Fail if this route is not runnable. Called at Workflow start. |
| 165 | pub fn require_ready(&self) -> Result<(), PreflightError> { |
| 166 | match &self.credential { |
| 167 | CredentialReadiness::Configured | CredentialReadiness::KeylessLocal => Ok(()), |
| 168 | CredentialReadiness::Missing { detail } => Err(PreflightError::CredentialMissing { |
| 169 | member: self.member_id.clone(), |
| 170 | provider: self.provider_id.clone(), |
| 171 | detail: detail.clone(), |
| 172 | }), |
| 173 | } |
| 174 | } |
| 175 | } |
| 176 | |
| 177 | /// A frozen preflight for every worker in a Workflow, plus the Router. |
| 178 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] |
| 179 | pub struct RoutePreflight { |
| 180 | workers: Vec<PreflightedRoute>, |
| 181 | router: Option<PreflightedRoute>, |
| 182 | } |
| 183 | |
| 184 | impl RoutePreflight { |
| 185 | #[must_use] |
| 186 | pub fn new(workers: Vec<PreflightedRoute>, router: Option<PreflightedRoute>) -> Self { |
| 187 | Self { workers, router } |
| 188 | } |
| 189 | |
| 190 | #[must_use] |
| 191 | pub fn workers(&self) -> &[PreflightedRoute] { |
| 192 | &self.workers |
| 193 | } |
| 194 | |
| 195 | #[must_use] |
| 196 | pub fn router(&self) -> Option<&PreflightedRoute> { |
| 197 | self.router.as_ref() |
| 198 | } |
| 199 | |
| 200 | /// The frozen route for one member id. |
| 201 | #[must_use] |
| 202 | pub fn worker(&self, member_id: &str) -> Option<&PreflightedRoute> { |
| 203 | let key = member_id.trim().to_ascii_lowercase(); |
| 204 | self.workers.iter().find(|route| route.member_id == key) |
| 205 | } |
| 206 | |
| 207 | /// Fail unless every worker route is runnable. |
| 208 | /// |
| 209 | /// Called before a Workflow is allowed to start, so a member with no |
| 210 | /// credential is a startup error rather than a first-task surprise. |
| 211 | pub fn require_all_ready(&self) -> Result<(), PreflightError> { |
| 212 | for route in &self.workers { |
| 213 | route.require_ready()?; |
| 214 | } |
| 215 | Ok(()) |
| 216 | } |
| 217 | |
| 218 | /// Whether any worker route talks to a different provider than the Router. |
| 219 | /// This is what a receipt discloses as cross-provider inference. |
| 220 | #[must_use] |
| 221 | pub fn crosses_providers(&self, member_id: &str) -> bool { |
| 222 | match (self.worker(member_id), self.router()) { |
| 223 | (Some(worker), Some(router)) => worker.provider_id != router.provider_id, |
| 224 | _ => false, |
| 225 | } |
| 226 | } |
| 227 | } |
| 228 | |
| 229 | #[derive(Debug, Clone, PartialEq, Eq, Error)] |
| 230 | pub enum PreflightError { |
| 231 | #[error( |
| 232 | "fleet member `{member}` is pinned to provider `{provider}`, which does not resolve to a \ |
| 233 | configured provider: {detail}" |
| 234 | )] |
| 235 | ProviderUnresolved { |
| 236 | member: String, |
| 237 | provider: String, |
| 238 | detail: String, |
| 239 | }, |
| 240 | #[error( |
| 241 | "fleet member `{member}` is pinned to model `{model}` on provider `{provider}`, which is \ |
| 242 | not a valid route: {detail}" |
| 243 | )] |
| 244 | ModelUnresolved { |
| 245 | member: String, |
| 246 | provider: String, |
| 247 | model: String, |
| 248 | detail: String, |
| 249 | }, |
| 250 | #[error( |
| 251 | "fleet member `{member}` cannot run: provider `{provider}` has no credential configured \ |
| 252 | on this machine ({detail}). This is decided locally — no provider was contacted. Keyless \ |
| 253 | local providers do not need one." |
| 254 | )] |
| 255 | CredentialMissing { |
| 256 | member: String, |
| 257 | provider: String, |
| 258 | detail: String, |
| 259 | }, |
| 260 | #[error( |
| 261 | "cannot determine what reasoning control provider `{provider}` actually expresses for \ |
| 262 | model `{model}`: {detail}. An exact fleet fails closed here rather than claiming a \ |
| 263 | capability it did not verify." |
| 264 | )] |
| 265 | CapabilityUnknown { |
| 266 | provider: String, |
| 267 | model: String, |
| 268 | detail: String, |
| 269 | }, |
| 270 | } |
| 271 | |
| 272 | #[cfg(test)] |
| 273 | mod tests { |
| 274 | use super::*; |
| 275 | |
| 276 | fn route(member: &str, provider: &str, wire: &str) -> PreflightedRoute { |
| 277 | PreflightedRoute { |
| 278 | member_id: member.to_string(), |
| 279 | provider_id: provider.to_string(), |
| 280 | provider_kind: provider.to_string(), |
| 281 | declared_model: wire.to_string(), |
| 282 | wire_model: wire.to_string(), |
| 283 | endpoint: EndpointIdentity::from_base_url("https://api.z.ai/api/paas/v4"), |
| 284 | credential: CredentialReadiness::Configured, |
| 285 | capability: ReasoningCapability::tiered(), |
| 286 | } |
| 287 | } |
| 288 | |
| 289 | #[test] |
| 290 | fn an_endpoint_identity_keeps_the_host_and_drops_credentials() { |
| 291 | let identity = |
| 292 | EndpointIdentity::from_base_url("https://user:sk-secret@api.z.ai/api/paas/v4"); |
| 293 | assert_eq!(identity.host, "api.z.ai"); |
| 294 | assert!(!identity.local); |
| 295 | assert!(!identity.label().contains("sk-secret")); |
| 296 | assert!(!identity.label().contains('/')); |
| 297 | } |
| 298 | |
| 299 | #[test] |
| 300 | fn loopback_and_private_endpoints_are_marked_local() { |
| 301 | for url in [ |
| 302 | "http://127.0.0.1:8000/v1", |
| 303 | "http://localhost:11434", |
| 304 | "http://192.168.1.20:8000/v1", |
| 305 | "http://box.local/v1", |
| 306 | ] { |
| 307 | let identity = EndpointIdentity::from_base_url(url); |
| 308 | assert!(identity.local, "{url} must be local"); |
| 309 | assert!(identity.label().ends_with("(local)")); |
| 310 | } |
| 311 | assert!(!EndpointIdentity::from_base_url("https://api.openai.com/v1").local); |
| 312 | } |
| 313 | |
| 314 | /// The receipt and the child spawn must not be able to disagree, so there |
| 315 | /// is exactly one canonical wire model and both read it. |
| 316 | #[test] |
| 317 | fn the_frozen_route_uses_the_canonical_wire_model() { |
| 318 | let mut preflighted = route("implementer", "zai", "glm-5"); |
| 319 | preflighted.wire_model = "glm-5-20260101".to_string(); |
| 320 | |
| 321 | assert_eq!(preflighted.frozen().model, "glm-5-20260101"); |
| 322 | assert_eq!(preflighted.frozen().provider, "zai"); |
| 323 | assert!(preflighted.model_canonicalized()); |
| 324 | assert_eq!(preflighted.declared_model, "glm-5"); |
| 325 | } |
| 326 | |
| 327 | /// Keyless local providers are first-class: readiness is about whether the |
| 328 | /// route can run, not about whether a key exists. |
| 329 | #[test] |
| 330 | fn keyless_local_providers_are_ready() { |
| 331 | let mut local = route("worker", "vllm", "qwen3"); |
| 332 | local.credential = CredentialReadiness::KeylessLocal; |
| 333 | local.endpoint = EndpointIdentity::from_base_url("http://127.0.0.1:8000/v1"); |
| 334 | |
| 335 | assert!(local.credential.is_ready()); |
| 336 | local.require_ready().expect("keyless local is valid"); |
| 337 | assert_eq!(local.credential.as_str(), "keyless_local"); |
| 338 | } |
| 339 | |
| 340 | #[test] |
| 341 | fn a_missing_credential_fails_the_workflow_locally() { |
| 342 | let mut route = route("implementer", "zai", "glm-5"); |
| 343 | route.credential = CredentialReadiness::Missing { |
| 344 | detail: "no ZAI_API_KEY".to_string(), |
| 345 | }; |
| 346 | |
| 347 | let preflight = RoutePreflight::new(vec![route], None); |
| 348 | let err = preflight |
| 349 | .require_all_ready() |
| 350 | .expect_err("a member with no credential must not start"); |
| 351 | assert!(matches!(err, PreflightError::CredentialMissing { .. })); |
| 352 | let message = err.to_string(); |
| 353 | assert!( |
| 354 | message.contains("decided locally"), |
| 355 | "the error must say no provider was contacted: {message}" |
| 356 | ); |
| 357 | } |
| 358 | |
| 359 | #[test] |
| 360 | fn cross_provider_inference_is_detectable_from_the_preflight() { |
| 361 | let preflight = RoutePreflight::new( |
| 362 | vec![route("implementer", "zai", "glm-5")], |
| 363 | Some(route("router", "openai", "gpt-5.6-luna")), |
| 364 | ); |
| 365 | assert!(preflight.crosses_providers("implementer")); |
| 366 | |
| 367 | let same = RoutePreflight::new( |
| 368 | vec![route("implementer", "zai", "glm-5")], |
| 369 | Some(route("router", "zai", "glm-5-turbo")), |
| 370 | ); |
| 371 | assert!(!same.crosses_providers("implementer")); |
| 372 | |
| 373 | // With no router, nothing crosses. |
| 374 | let none = RoutePreflight::new(vec![route("implementer", "zai", "glm-5")], None); |
| 375 | assert!(!none.crosses_providers("implementer")); |
| 376 | } |
| 377 | |
| 378 | #[test] |
| 379 | fn a_preflight_serializes_without_secrets_or_paths() { |
| 380 | let preflight = RoutePreflight::new( |
| 381 | vec![route("implementer", "zai", "glm-5")], |
| 382 | Some(route("router", "openai", "gpt-5.6-luna")), |
| 383 | ); |
| 384 | let json = serde_json::to_string(&preflight).expect("serialize"); |
| 385 | let lowered = json.to_ascii_lowercase(); |
| 386 | for forbidden in [ |
| 387 | "api_key", "secret", "bearer", "base_url", "/users/", "https://", |
| 388 | ] { |
| 389 | assert!(!lowered.contains(forbidden), "{forbidden} in {json}"); |
| 390 | } |
| 391 | let back: RoutePreflight = serde_json::from_str(&json).expect("round-trip"); |
| 392 | assert_eq!(back, preflight); |
| 393 | } |
| 394 | } |
| 395 |