| 1 | //! Shared Fleet control-plane surface (#1888, #4022). |
| 2 | //! |
| 3 | //! `codewhale fleet …` and the `/fleet …` slash command (and therefore its |
| 4 | //! hotbar action) run the *same* verbs against the *same* durable ledger and |
| 5 | //! render the *same* [`ControlReceipt`]. Nothing here formats twice: the CLI's |
| 6 | //! `print_status` / `print_inspection` delegate to the renderers below. |
| 7 | //! |
| 8 | //! Vocabulary: Fleet = who, Workflow = order, Lane = one running Workflow, |
| 9 | //! Runtime = where/how. Auto-Review is a permission posture and never appears |
| 10 | //! here as a role. |
| 11 | |
| 12 | use std::path::{Path, PathBuf}; |
| 13 | |
| 14 | use codewhale_lane::control::{ |
| 15 | Availability, ControlContext, ControlDomain, ControlFailure, ControlFailureKind, |
| 16 | ControlOperation, ControlReceipt, ControlSurface, DEFAULT_RUN_LIST_LIMIT, Known, RunListPage, |
| 17 | RunRouteDto, RunSummaryDto, RunUsageDto, UnknownReason, parse_target, redact_path, |
| 18 | sanitize_line, |
| 19 | }; |
| 20 | use codewhale_protocol::fleet::{ |
| 21 | FleetArtifactKind, FleetReceipt, FleetRun, FleetRunId, FleetRunStatus, FleetWorkerEventPayload, |
| 22 | FleetWorkerStatus, |
| 23 | }; |
| 24 | |
| 25 | use super::ledger::FleetLedgerState; |
| 26 | use super::manager::{FleetControlError, FleetManager, FleetStatusSnapshot, FleetWorkerInspection}; |
| 27 | |
| 28 | /// Maximum worker rows rendered in one durable status payload. |
| 29 | pub const MAX_STATUS_WORKER_ROWS: usize = 24; |
| 30 | /// Maximum artifact rows rendered in one inspection payload. |
| 31 | pub const MAX_INSPECTION_ARTIFACT_ROWS: usize = 24; |
| 32 | |
| 33 | /// The durable Fleet ledger for `workspace`, without creating it. |
| 34 | #[must_use] |
| 35 | pub fn fleet_ledger_path(workspace: &Path) -> PathBuf { |
| 36 | workspace.join(".codewhale").join("fleet.jsonl") |
| 37 | } |
| 38 | |
| 39 | /// Read-only availability probe for the Fleet domain. |
| 40 | /// |
| 41 | /// [`FleetManager::open`] creates the ledger as a side effect, so a status |
| 42 | /// surface must probe first; otherwise "this workspace has no Fleet ledger" |
| 43 | /// silently becomes "here is an empty Fleet ledger I just made". |
| 44 | #[must_use] |
| 45 | pub fn fleet_control_context(workspace: &Path) -> ControlContext { |
| 46 | ControlContext::probe(None, Some(&fleet_ledger_path(workspace))) |
| 47 | } |
| 48 | |
| 49 | // --------------------------------------------------------------------------- |
| 50 | // Labels and renderers (single source for CLI and TUI) |
| 51 | // --------------------------------------------------------------------------- |
| 52 | |
| 53 | #[must_use] |
| 54 | pub fn worker_status_label(status: &FleetWorkerStatus) -> &'static str { |
| 55 | match status { |
| 56 | FleetWorkerStatus::Unknown => "unknown", |
| 57 | FleetWorkerStatus::Online => "online", |
| 58 | FleetWorkerStatus::Busy => "busy", |
| 59 | FleetWorkerStatus::Offline => "offline", |
| 60 | FleetWorkerStatus::Unhealthy => "unhealthy", |
| 61 | FleetWorkerStatus::Draining => "draining", |
| 62 | FleetWorkerStatus::Retired => "retired", |
| 63 | } |
| 64 | } |
| 65 | |
| 66 | #[must_use] |
| 67 | pub fn run_status_label(status: &FleetRunStatus) -> &'static str { |
| 68 | match status { |
| 69 | FleetRunStatus::Pending => "pending", |
| 70 | FleetRunStatus::Queued => "queued", |
| 71 | FleetRunStatus::Running => "running", |
| 72 | FleetRunStatus::Paused => "paused", |
| 73 | FleetRunStatus::Completed => "completed", |
| 74 | FleetRunStatus::Failed => "failed", |
| 75 | FleetRunStatus::Cancelled => "cancelled", |
| 76 | } |
| 77 | } |
| 78 | |
| 79 | #[must_use] |
| 80 | pub fn artifact_kind_label(kind: &FleetArtifactKind) -> String { |
| 81 | match kind { |
| 82 | FleetArtifactKind::Log => "log".to_string(), |
| 83 | FleetArtifactKind::Patch => "patch".to_string(), |
| 84 | FleetArtifactKind::TestResult => "test_result".to_string(), |
| 85 | FleetArtifactKind::Report => "report".to_string(), |
| 86 | FleetArtifactKind::Checkpoint => "checkpoint".to_string(), |
| 87 | FleetArtifactKind::Receipt => "receipt".to_string(), |
| 88 | FleetArtifactKind::Other(value) => value.clone(), |
| 89 | } |
| 90 | } |
| 91 | |
| 92 | #[must_use] |
| 93 | pub fn event_label(payload: &FleetWorkerEventPayload) -> String { |
| 94 | match payload { |
| 95 | FleetWorkerEventPayload::Queued => "queued".to_string(), |
| 96 | FleetWorkerEventPayload::Leased { .. } => "leased".to_string(), |
| 97 | FleetWorkerEventPayload::Starting => "starting".to_string(), |
| 98 | FleetWorkerEventPayload::Running => "running".to_string(), |
| 99 | FleetWorkerEventPayload::ModelWait { model } => model |
| 100 | .as_ref() |
| 101 | .map(|model| format!("model_wait model={model}")) |
| 102 | .unwrap_or_else(|| "model_wait".to_string()), |
| 103 | FleetWorkerEventPayload::RunningTool { tool, call_id } => call_id |
| 104 | .as_ref() |
| 105 | .map(|call_id| format!("running_tool tool={tool} call_id={call_id}")) |
| 106 | .unwrap_or_else(|| format!("running_tool tool={tool}")), |
| 107 | FleetWorkerEventPayload::WorkflowEvent { |
| 108 | workflow_run_id, |
| 109 | event, |
| 110 | } => event |
| 111 | .get("type") |
| 112 | .and_then(serde_json::Value::as_str) |
| 113 | .map(|kind| format!("workflow_event run_id={workflow_run_id} type={kind}")) |
| 114 | .unwrap_or_else(|| format!("workflow_event run_id={workflow_run_id}")), |
| 115 | FleetWorkerEventPayload::Heartbeat { .. } => "heartbeat".to_string(), |
| 116 | FleetWorkerEventPayload::UsageReport { |
| 117 | input_tokens, |
| 118 | output_tokens, |
| 119 | } => format!("usage_report input={input_tokens} output={output_tokens}"), |
| 120 | FleetWorkerEventPayload::Artifact(artifact) => { |
| 121 | format!("artifact kind={}", artifact_kind_label(&artifact.kind)) |
| 122 | } |
| 123 | FleetWorkerEventPayload::Completed { exit_code, summary } => match (exit_code, summary) { |
| 124 | (Some(code), Some(summary)) => format!("completed exit_code={code} {summary}"), |
| 125 | (Some(code), None) => format!("completed exit_code={code}"), |
| 126 | (None, Some(summary)) => format!("completed {summary}"), |
| 127 | (None, None) => "completed".to_string(), |
| 128 | }, |
| 129 | FleetWorkerEventPayload::Failed { |
| 130 | reason, |
| 131 | recoverable, |
| 132 | } => format!("failed recoverable={recoverable} reason={reason}"), |
| 133 | FleetWorkerEventPayload::Cancelled { cancelled_by } => cancelled_by |
| 134 | .as_ref() |
| 135 | .map(|by| format!("cancelled by={by}")) |
| 136 | .unwrap_or_else(|| "cancelled".to_string()), |
| 137 | FleetWorkerEventPayload::Interrupted { signal } => signal |
| 138 | .as_ref() |
| 139 | .map(|signal| format!("interrupted signal={signal}")) |
| 140 | .unwrap_or_else(|| "interrupted".to_string()), |
| 141 | FleetWorkerEventPayload::Stale { last_heartbeat_at } => last_heartbeat_at |
| 142 | .as_ref() |
| 143 | .map(|ts| format!("stale last_heartbeat_at={ts}")) |
| 144 | .unwrap_or_else(|| "stale".to_string()), |
| 145 | FleetWorkerEventPayload::Restarted { restart_count } => { |
| 146 | format!("restarted count={restart_count}") |
| 147 | } |
| 148 | FleetWorkerEventPayload::Escalated { channel, alert_id } => alert_id |
| 149 | .as_ref() |
| 150 | .map(|alert_id| format!("escalated channel={channel} alert_id={alert_id}")) |
| 151 | .unwrap_or_else(|| format!("escalated channel={channel}")), |
| 152 | } |
| 153 | } |
| 154 | |
| 155 | /// Durable status snapshot as bounded Fleet receipt lines. |
| 156 | /// |
| 157 | /// The command and slash surfaces call the customer-facing concept a Fleet, but |
| 158 | /// these strings are nested in the shared [`ControlReceipt`] detail contract. |
| 159 | /// Keep the established `fleet:` prefix so existing receipt consumers and |
| 160 | /// scripts do not need to parse a presentation rename. |
| 161 | #[must_use] |
| 162 | pub fn status_lines(status: &FleetStatusSnapshot) -> Vec<String> { |
| 163 | let mut lines = vec![format!( |
| 164 | "fleet: runs={} queued={} running={} completed={} partial={} failed={} restarted={} \ |
| 165 | escalated={} transport_failed={} task_failed={} verifier_failed={} cancelled={} stale={}", |
| 166 | status.runs, |
| 167 | status.queued, |
| 168 | status.running, |
| 169 | status.completed, |
| 170 | status.partial, |
| 171 | status.failed, |
| 172 | status.restarted, |
| 173 | status.escalated, |
| 174 | status.transport_failed, |
| 175 | status.task_failed, |
| 176 | status.verifier_failed, |
| 177 | status.cancelled, |
| 178 | status.stale |
| 179 | )]; |
| 180 | if !status.workers.is_empty() { |
| 181 | lines.push("workers:".to_string()); |
| 182 | for (worker_id, worker_status) in status.workers.iter().take(MAX_STATUS_WORKER_ROWS) { |
| 183 | lines.push(format!( |
| 184 | " {worker_id} {}", |
| 185 | worker_status_label(worker_status) |
| 186 | )); |
| 187 | } |
| 188 | let omitted = status.workers.len().saturating_sub(MAX_STATUS_WORKER_ROWS); |
| 189 | if omitted > 0 { |
| 190 | lines.push(format!( |
| 191 | " [{omitted} more worker(s) omitted by the {MAX_STATUS_WORKER_ROWS}-row bound]" |
| 192 | )); |
| 193 | } |
| 194 | } |
| 195 | lines |
| 196 | } |
| 197 | |
| 198 | /// Compatibility renderer shared by `codewhale fleet status` and `/fleet status`. |
| 199 | /// |
| 200 | /// The invocation names are public Fleet wording; the returned detail stays in |
| 201 | /// the durable Fleet receipt spelling by way of [`status_lines`]. |
| 202 | #[must_use] |
| 203 | pub fn render_fleet_status_snapshot(status: &FleetStatusSnapshot) -> String { |
| 204 | status_lines(status).join("\n") |
| 205 | } |
| 206 | |
| 207 | /// Durable worker inspection as bounded lines. |
| 208 | #[must_use] |
| 209 | pub fn inspection_lines(inspection: &FleetWorkerInspection) -> Vec<String> { |
| 210 | let mut lines = vec![ |
| 211 | format!("worker: {}", inspection.worker_id), |
| 212 | format!("status: {}", worker_status_label(&inspection.status)), |
| 213 | ]; |
| 214 | if let Some(run_id) = &inspection.current_run_id { |
| 215 | lines.push(format!("run: {}", run_id.0)); |
| 216 | } |
| 217 | if let Some(task_id) = &inspection.current_task_id { |
| 218 | lines.push(format!("task: {task_id}")); |
| 219 | } |
| 220 | if let Some(objective) = &inspection.objective { |
| 221 | lines.push(format!("objective: {objective}")); |
| 222 | } |
| 223 | if let Some(role) = &inspection.role { |
| 224 | lines.push(format!("role: {role}")); |
| 225 | } |
| 226 | if let Some(host) = &inspection.host { |
| 227 | lines.push(format!("host: {host}")); |
| 228 | } |
| 229 | if let Some(heartbeat) = &inspection.latest_heartbeat_at { |
| 230 | lines.push(format!("heartbeat: {heartbeat}")); |
| 231 | } |
| 232 | if let Some(event) = &inspection.latest_event { |
| 233 | lines.push(format!( |
| 234 | "latest_event: seq={} {}", |
| 235 | event.seq, |
| 236 | event_label(&event.payload) |
| 237 | )); |
| 238 | } |
| 239 | if !inspection.artifacts.is_empty() { |
| 240 | lines.push("artifacts:".to_string()); |
| 241 | for artifact in inspection |
| 242 | .artifacts |
| 243 | .iter() |
| 244 | .take(MAX_INSPECTION_ARTIFACT_ROWS) |
| 245 | { |
| 246 | lines.push(format!( |
| 247 | " {} {}", |
| 248 | artifact_kind_label(&artifact.kind), |
| 249 | artifact.path.display() |
| 250 | )); |
| 251 | } |
| 252 | let omitted = inspection |
| 253 | .artifacts |
| 254 | .len() |
| 255 | .saturating_sub(MAX_INSPECTION_ARTIFACT_ROWS); |
| 256 | if omitted > 0 { |
| 257 | lines.push(format!(" [{omitted} more artifact(s) omitted]")); |
| 258 | } |
| 259 | } |
| 260 | if let Some(receipt) = &inspection.receipt_summary { |
| 261 | lines.push(format!("receipt: {receipt}")); |
| 262 | } |
| 263 | if let Some(error) = &inspection.last_error { |
| 264 | lines.push(format!("last_error: {error}")); |
| 265 | } |
| 266 | if let Some(alert) = &inspection.alert_state { |
| 267 | lines.push(format!("alert: {alert}")); |
| 268 | } |
| 269 | lines |
| 270 | } |
| 271 | |
| 272 | #[must_use] |
| 273 | pub fn render_inspection(inspection: &FleetWorkerInspection) -> String { |
| 274 | inspection_lines(inspection).join("\n") |
| 275 | } |
| 276 | |
| 277 | /// Artifact listing lines for `codewhale fleet artifacts`. |
| 278 | #[must_use] |
| 279 | pub fn artifact_lines(inspection: &FleetWorkerInspection) -> Vec<String> { |
| 280 | if inspection.artifacts.is_empty() { |
| 281 | return vec!["artifacts: none".to_string()]; |
| 282 | } |
| 283 | let mut lines = vec!["artifacts:".to_string()]; |
| 284 | for artifact in inspection |
| 285 | .artifacts |
| 286 | .iter() |
| 287 | .take(MAX_INSPECTION_ARTIFACT_ROWS) |
| 288 | { |
| 289 | let size = artifact |
| 290 | .size_bytes |
| 291 | .map(|size| format!(" size={size}")) |
| 292 | .unwrap_or_default(); |
| 293 | let mime = artifact |
| 294 | .mime_type |
| 295 | .as_ref() |
| 296 | .map(|mime| format!(" mime={mime}")) |
| 297 | .unwrap_or_default(); |
| 298 | lines.push(format!( |
| 299 | " {} {}{}{}", |
| 300 | artifact_kind_label(&artifact.kind), |
| 301 | artifact.path.display(), |
| 302 | size, |
| 303 | mime |
| 304 | )); |
| 305 | } |
| 306 | let omitted = inspection |
| 307 | .artifacts |
| 308 | .len() |
| 309 | .saturating_sub(MAX_INSPECTION_ARTIFACT_ROWS); |
| 310 | if omitted > 0 { |
| 311 | lines.push(format!(" [{omitted} more artifact(s) omitted]")); |
| 312 | } |
| 313 | lines |
| 314 | } |
| 315 | |
| 316 | #[must_use] |
| 317 | pub fn render_artifacts(inspection: &FleetWorkerInspection) -> String { |
| 318 | artifact_lines(inspection).join("\n") |
| 319 | } |
| 320 | |
| 321 | // --------------------------------------------------------------------------- |
| 322 | // Fleet run DTOs |
| 323 | // --------------------------------------------------------------------------- |
| 324 | |
| 325 | fn label(run: &FleetRun, key: &str) -> Option<String> { |
| 326 | run.labels.get(key).cloned() |
| 327 | } |
| 328 | |
| 329 | /// Project the most recent receipt's resolved route onto the shared DTO. |
| 330 | /// |
| 331 | /// Everything the receipt records exactly becomes `Known`; everything it does |
| 332 | /// not record stays typed-unknown. In particular the Fleet receipt persists |
| 333 | /// the *effective* reasoning tier only, so `requested_reasoning` is |
| 334 | /// `not_recorded` rather than being back-filled from the effective value. |
| 335 | fn route_dto(receipt: Option<&FleetReceipt>) -> RunRouteDto { |
| 336 | let Some(route) = receipt.and_then(|receipt| receipt.resolved_route.as_ref()) else { |
| 337 | return RunRouteDto::all_unknown(UnknownReason::NotRecorded); |
| 338 | }; |
| 339 | RunRouteDto { |
| 340 | provider_id: Known::Known(route.provider_id.clone()), |
| 341 | provider_exact_id: Known::from_option(route.provider_exact_id.clone()), |
| 342 | model: Known::Known( |
| 343 | route |
| 344 | .canonical_model |
| 345 | .clone() |
| 346 | .unwrap_or_else(|| route.wire_model_id.clone()), |
| 347 | ), |
| 348 | requested_reasoning: Known::unknown(), |
| 349 | effective_reasoning: Known::from_option(route.reasoning_effort.clone()), |
| 350 | route_source: Known::Known(route.source.clone()), |
| 351 | } |
| 352 | } |
| 353 | |
| 354 | /// Project one durable Fleet run into the shared run DTO. |
| 355 | #[must_use] |
| 356 | pub fn fleet_run_summary(run: &FleetRun, receipt: Option<&FleetReceipt>) -> RunSummaryDto { |
| 357 | RunSummaryDto { |
| 358 | domain: ControlDomain::Fleet, |
| 359 | run_id: run.id.0.clone(), |
| 360 | status: run_status_label(&run.status).to_string(), |
| 361 | // The Fleet ledger fences lifecycle per task, not per run. |
| 362 | lifecycle_seq: Known::not_applicable(), |
| 363 | runtime: Known::from_option(label(run, "runtime")), |
| 364 | workflow: Known::from_option(label(run, "workflow")), |
| 365 | fleet: Known::from_option( |
| 366 | label(run, "fleet").or_else(|| (!run.name.trim().is_empty()).then(|| run.name.clone())), |
| 367 | ), |
| 368 | issue: Known::from_option(label(run, "issue")), |
| 369 | goal: Known::from_option(label(run, "goal").or_else(|| label(run, "objective"))), |
| 370 | started_at: Known::Known(run.created_at.clone()), |
| 371 | stopped_at: Known::from_option(run.completed_at.clone()), |
| 372 | // Fleet runs are workspace-scoped; there is no per-run worktree, and |
| 373 | // the Lane-shaped Runtime handles do not apply to a Fleet run. |
| 374 | location: Known::not_applicable(), |
| 375 | branch: Known::not_applicable(), |
| 376 | runtime_session: Known::not_applicable(), |
| 377 | runtime_socket: Known::not_applicable(), |
| 378 | attach: Known::not_applicable(), |
| 379 | log: Known::not_applicable(), |
| 380 | route: route_dto(receipt), |
| 381 | // The ledger does not persist token counts or wall-clock duration. |
| 382 | usage: RunUsageDto::all_unknown(UnknownReason::NotRecorded), |
| 383 | } |
| 384 | } |
| 385 | |
| 386 | /// Bounded page of durable Fleet runs, newest first. |
| 387 | #[must_use] |
| 388 | pub fn fleet_run_page(state: &FleetLedgerState, limit: usize) -> RunListPage { |
| 389 | let mut summaries: Vec<RunSummaryDto> = state |
| 390 | .runs |
| 391 | .values() |
| 392 | .map(|run| { |
| 393 | let status = state |
| 394 | .run_status_overrides |
| 395 | .get(&run.id.0) |
| 396 | .cloned() |
| 397 | .unwrap_or_else(|| run.status.clone()); |
| 398 | let receipt = state |
| 399 | .receipts |
| 400 | .values() |
| 401 | .filter(|receipt| receipt.run_id.0 == run.id.0) |
| 402 | .max_by(|a, b| a.completed_at.cmp(&b.completed_at)); |
| 403 | let mut summary = fleet_run_summary(run, receipt); |
| 404 | summary.status = run_status_label(&status).to_string(); |
| 405 | summary |
| 406 | }) |
| 407 | .collect(); |
| 408 | // Newest first, matching `lane list`. Sort on the parsed UTC instant, not |
| 409 | // the rendered text: two timestamps written at different offsets order |
| 410 | // wrongly under a string compare. Unparseable timestamps sort last rather |
| 411 | // than being silently interleaved, and the exact id breaks every tie so |
| 412 | // the order is total and stable. |
| 413 | summaries.sort_by(|a, b| { |
| 414 | instant_of(&a.started_at) |
| 415 | .cmp(&instant_of(&b.started_at)) |
| 416 | .reverse() |
| 417 | .then_with(|| a.run_id.cmp(&b.run_id)) |
| 418 | }); |
| 419 | RunListPage::bounded(summaries, limit) |
| 420 | } |
| 421 | |
| 422 | /// Parse a recorded timestamp into a comparable UTC instant. |
| 423 | /// |
| 424 | /// `None` for unknown or unparseable values, which `Option`'s ordering places |
| 425 | /// before every real instant — and therefore last under the reversed |
| 426 | /// newest-first sort. |
| 427 | fn instant_of(value: &Known<String>) -> Option<chrono::DateTime<chrono::Utc>> { |
| 428 | let raw = value.as_known()?; |
| 429 | chrono::DateTime::parse_from_rfc3339(raw) |
| 430 | .ok() |
| 431 | .map(|parsed| parsed.with_timezone(&chrono::Utc)) |
| 432 | } |
| 433 | |
| 434 | // --------------------------------------------------------------------------- |
| 435 | // Executor — the one code path behind `codewhale fleet …` and `/fleet …` |
| 436 | // --------------------------------------------------------------------------- |
| 437 | |
| 438 | /// Run a Fleet control verb against the durable workspace ledger, using a |
| 439 | /// default manager. |
| 440 | /// |
| 441 | /// The slash command and hotbar use this. The CLI uses |
| 442 | /// [`execute_fleet_control_with`] so its configured manager (exec config, |
| 443 | /// stale-after window, session model, route config) still applies — same code |
| 444 | /// path, same receipt, caller-owned policy. |
| 445 | #[must_use] |
| 446 | pub fn execute_fleet_control( |
| 447 | surface: ControlSurface, |
| 448 | workspace: &Path, |
| 449 | operation: ControlOperation, |
| 450 | raw_target: Option<&str>, |
| 451 | ) -> ControlReceipt { |
| 452 | let descriptor = operation.descriptor(); |
| 453 | let availability = descriptor.availability(surface, fleet_control_context(workspace)); |
| 454 | if !availability.is_available() { |
| 455 | return ControlReceipt::unavailable(descriptor, surface, availability); |
| 456 | } |
| 457 | match FleetManager::open(workspace) { |
| 458 | Ok(manager) => execute_fleet_control_with( |
| 459 | surface, |
| 460 | workspace, |
| 461 | fleet_control_context(workspace), |
| 462 | &manager, |
| 463 | operation, |
| 464 | raw_target, |
| 465 | ), |
| 466 | Err(err) => ControlReceipt::failed( |
| 467 | descriptor, |
| 468 | surface, |
| 469 | None, |
| 470 | ControlFailure::backend(format!("{err:#}")), |
| 471 | ), |
| 472 | } |
| 473 | } |
| 474 | |
| 475 | /// Run a Fleet control verb against a caller-configured [`FleetManager`]. |
| 476 | /// |
| 477 | /// The CLI and the slash command both land here, so availability, target |
| 478 | /// selection, lifecycle outcome, retryability, and the sanitized failure are |
| 479 | /// decided once. `fleet.restart` is declared `SurfaceLimited` to the CLI |
| 480 | /// because it drives the manager loop to completion; this function reports |
| 481 | /// that as a typed unavailability on other surfaces rather than quietly doing |
| 482 | /// a different, smaller thing. |
| 483 | #[must_use] |
| 484 | pub fn execute_fleet_control_with( |
| 485 | surface: ControlSurface, |
| 486 | workspace: &Path, |
| 487 | ctx: ControlContext, |
| 488 | manager: &FleetManager, |
| 489 | operation: ControlOperation, |
| 490 | raw_target: Option<&str>, |
| 491 | ) -> ControlReceipt { |
| 492 | let descriptor = operation.descriptor(); |
| 493 | if descriptor.domain != ControlDomain::Fleet { |
| 494 | return ControlReceipt::rejected( |
| 495 | descriptor, |
| 496 | surface, |
| 497 | None, |
| 498 | ControlFailure::new( |
| 499 | ControlFailureKind::InvalidTarget, |
| 500 | format!("{} is not a Fleet verb", descriptor.id), |
| 501 | ), |
| 502 | ); |
| 503 | } |
| 504 | |
| 505 | let availability = descriptor.availability(surface, ctx); |
| 506 | if !availability.is_available() { |
| 507 | return ControlReceipt::unavailable(descriptor, surface, availability); |
| 508 | } |
| 509 | |
| 510 | let target = match parse_target(descriptor, raw_target) { |
| 511 | Ok(target) => target, |
| 512 | Err(failure) => return ControlReceipt::rejected(descriptor, surface, None, failure), |
| 513 | }; |
| 514 | |
| 515 | match operation { |
| 516 | ControlOperation::FleetList => match manager.rebuild_state() { |
| 517 | Ok(state) => ControlReceipt::inspected(descriptor, surface, None) |
| 518 | .with_runs(fleet_run_page(&state, DEFAULT_RUN_LIST_LIMIT)) |
| 519 | .with_detail([format!( |
| 520 | "ledger: {}", |
| 521 | redact_path(&fleet_ledger_path(workspace)) |
| 522 | )]), |
| 523 | Err(err) => ControlReceipt::failed( |
| 524 | descriptor, |
| 525 | surface, |
| 526 | None, |
| 527 | ControlFailure::backend(format!("{err:#}")), |
| 528 | ), |
| 529 | }, |
| 530 | ControlOperation::FleetStatus => match manager.status() { |
| 531 | Ok(status) => ControlReceipt::inspected(descriptor, surface, None).with_detail( |
| 532 | status_lines(&status).into_iter().chain([format!( |
| 533 | "ledger: {}", |
| 534 | redact_path(&fleet_ledger_path(workspace)) |
| 535 | )]), |
| 536 | ), |
| 537 | Err(err) => ControlReceipt::failed( |
| 538 | descriptor, |
| 539 | surface, |
| 540 | None, |
| 541 | ControlFailure::backend(format!("{err:#}")), |
| 542 | ), |
| 543 | }, |
| 544 | ControlOperation::FleetInterrupt => { |
| 545 | let Some(target) = target else { |
| 546 | return ControlReceipt::rejected( |
| 547 | descriptor, |
| 548 | surface, |
| 549 | None, |
| 550 | ControlFailure::invalid_target(format!( |
| 551 | "{} needs an exact worker id", |
| 552 | descriptor.id |
| 553 | )), |
| 554 | ); |
| 555 | }; |
| 556 | // Exact identity against the real ledger before any mutation: a |
| 557 | // worker id that was never seen in this workspace is `not_found`, |
| 558 | // which is a different fact from "known worker, nothing leased" |
| 559 | // (a conflict). Checking here keeps a typo from reaching the |
| 560 | // mutation path at all. |
| 561 | match manager.rebuild_state() { |
| 562 | Ok(state) => { |
| 563 | if !state.workers.contains_key(&target.id) { |
| 564 | return ControlReceipt::rejected( |
| 565 | descriptor, |
| 566 | surface, |
| 567 | Some(target.clone()), |
| 568 | ControlFailure::not_found(format!( |
| 569 | "no Fleet worker with id {} in this workspace's ledger", |
| 570 | target.id |
| 571 | )), |
| 572 | ); |
| 573 | } |
| 574 | } |
| 575 | Err(err) => { |
| 576 | return ControlReceipt::failed( |
| 577 | descriptor, |
| 578 | surface, |
| 579 | Some(target), |
| 580 | ControlFailure::backend(format!("{err:#}")), |
| 581 | ); |
| 582 | } |
| 583 | } |
| 584 | // Bind before matching so the borrow of `target.id` is over |
| 585 | // before the arms move `target` into the receipt. |
| 586 | let interrupted = manager.interrupt_worker(&target.id); |
| 587 | match interrupted { |
| 588 | Ok(inspection) => ControlReceipt::transitioned(descriptor, surface, Some(target)) |
| 589 | .with_detail(inspection_lines(&inspection)), |
| 590 | Err(err) => { |
| 591 | // The manager refuses when the exact worker has no active |
| 592 | // task. That is a state conflict, not a transient backend |
| 593 | // fault: retrying it will keep failing until work is |
| 594 | // leased again. Classified by type, not by message text. |
| 595 | let message = format!("{err:#}"); |
| 596 | let failure = match err.downcast_ref::<FleetControlError>() { |
| 597 | Some(FleetControlError::NoActiveTask { .. }) => { |
| 598 | ControlFailure::conflict(message) |
| 599 | } |
| 600 | Some(FleetControlError::UnknownRun { .. }) => { |
| 601 | ControlFailure::not_found(message) |
| 602 | } |
| 603 | None => ControlFailure::backend(message), |
| 604 | }; |
| 605 | ControlReceipt::failed(descriptor, surface, Some(target), failure) |
| 606 | } |
| 607 | } |
| 608 | } |
| 609 | ControlOperation::FleetResume => { |
| 610 | let Some(target) = target else { |
| 611 | return ControlReceipt::rejected( |
| 612 | descriptor, |
| 613 | surface, |
| 614 | None, |
| 615 | ControlFailure::invalid_target(format!( |
| 616 | "{} needs an exact run id", |
| 617 | descriptor.id |
| 618 | )), |
| 619 | ); |
| 620 | }; |
| 621 | // Exact identity first. `resume_run` reconciles by run id and, for |
| 622 | // an id that is not in the ledger, would still write a run-status |
| 623 | // record keyed by whatever string the caller typed — durable |
| 624 | // pollution from a typo, reported as a benign no-op. Refuse before |
| 625 | // any write happens (#4022). |
| 626 | match manager.rebuild_state() { |
| 627 | Ok(state) => { |
| 628 | if !state.runs.contains_key(&target.id) { |
| 629 | return ControlReceipt::rejected( |
| 630 | descriptor, |
| 631 | surface, |
| 632 | Some(target.clone()), |
| 633 | ControlFailure::not_found( |
| 634 | FleetControlError::UnknownRun { |
| 635 | run_id: target.id.clone(), |
| 636 | } |
| 637 | .to_string(), |
| 638 | ), |
| 639 | ); |
| 640 | } |
| 641 | } |
| 642 | Err(err) => { |
| 643 | return ControlReceipt::failed( |
| 644 | descriptor, |
| 645 | surface, |
| 646 | Some(target), |
| 647 | ControlFailure::backend(format!("{err:#}")), |
| 648 | ); |
| 649 | } |
| 650 | } |
| 651 | let resumed = manager.resume_run(&FleetRunId::from(target.id.clone())); |
| 652 | match resumed { |
| 653 | Ok(report) => { |
| 654 | let reconciled = report.reclaimed_stale |
| 655 | + report.restarted |
| 656 | + report.failed |
| 657 | + report.escalated; |
| 658 | let detail = [format!( |
| 659 | "fleet resume: {} reclaimed_stale={} restarted={} failed={} escalated={}", |
| 660 | report.run_id.0, |
| 661 | report.reclaimed_stale, |
| 662 | report.restarted, |
| 663 | report.failed, |
| 664 | report.escalated |
| 665 | )] |
| 666 | .into_iter() |
| 667 | .chain(status_lines(&report.status)); |
| 668 | if reconciled == 0 { |
| 669 | ControlReceipt::no_change(descriptor, surface, Some(target)) |
| 670 | .with_detail(detail) |
| 671 | } else { |
| 672 | ControlReceipt::transitioned(descriptor, surface, Some(target)) |
| 673 | .with_detail(detail) |
| 674 | } |
| 675 | } |
| 676 | Err(err) => ControlReceipt::failed( |
| 677 | descriptor, |
| 678 | surface, |
| 679 | Some(target), |
| 680 | ControlFailure::backend(format!("{err:#}")), |
| 681 | ), |
| 682 | } |
| 683 | } |
| 684 | // `fleet.restart` is CLI-only (it drives the manager loop). The |
| 685 | // availability gate above already rejected it elsewhere; this arm |
| 686 | // keeps the refusal explicit if the table ever changes. |
| 687 | _ => ControlReceipt::unavailable( |
| 688 | descriptor, |
| 689 | surface, |
| 690 | Availability::Unavailable { |
| 691 | reason: codewhale_lane::UnavailableReason::SurfaceNotSupported, |
| 692 | hint: sanitize_line(descriptor.cli_invocation), |
| 693 | }, |
| 694 | ), |
| 695 | } |
| 696 | } |
| 697 | |
| 698 | #[cfg(test)] |
| 699 | mod tests { |
| 700 | use super::*; |
| 701 | use crate::fleet::ledger::FleetLedger; |
| 702 | use codewhale_lane::{ControlAuthority, LifecycleOutcome, PersistenceScope, UnavailableReason}; |
| 703 | use codewhale_protocol::fleet::{FleetResolvedRoute, FleetTaskResult}; |
| 704 | use std::collections::BTreeMap; |
| 705 | |
| 706 | fn run(id: &str) -> FleetRun { |
| 707 | FleetRun { |
| 708 | id: FleetRunId::from(id.to_string()), |
| 709 | name: "stopship".to_string(), |
| 710 | status: FleetRunStatus::Running, |
| 711 | target: None, |
| 712 | workflow: None, |
| 713 | roles: Vec::new(), |
| 714 | max_workers: Some(2), |
| 715 | usage_ceiling: None, |
| 716 | task_specs: Vec::new(), |
| 717 | worker_specs: Vec::new(), |
| 718 | labels: BTreeMap::new(), |
| 719 | security_policy: None, |
| 720 | created_at: "2026-07-26T00:00:00Z".to_string(), |
| 721 | updated_at: None, |
| 722 | completed_at: None, |
| 723 | } |
| 724 | } |
| 725 | |
| 726 | fn receipt_with_route(run_id: &str) -> FleetReceipt { |
| 727 | FleetReceipt { |
| 728 | run_id: FleetRunId::from(run_id.to_string()), |
| 729 | task_id: "task-1".to_string(), |
| 730 | worker_id: "worker-1".to_string(), |
| 731 | attempt: Some(1), |
| 732 | terminal_seq: Some(9), |
| 733 | completed_at: "2026-07-26T00:01:00Z".to_string(), |
| 734 | result: FleetTaskResult::Pass, |
| 735 | failure_kind: None, |
| 736 | artifacts: Vec::new(), |
| 737 | score: None, |
| 738 | resolved_route: Some(FleetResolvedRoute { |
| 739 | provider_id: "deepseek".to_string(), |
| 740 | provider_exact_id: Some("custom".to_string()), |
| 741 | provider_kind: "deepseek".to_string(), |
| 742 | canonical_model: Some("deepseek-v3".to_string()), |
| 743 | wire_model_id: "deepseek-chat".to_string(), |
| 744 | protocol: "chat_completions".to_string(), |
| 745 | role: Some("implementer".to_string()), |
| 746 | loadout: None, |
| 747 | model_class: None, |
| 748 | model_route: None, |
| 749 | reasoning_effort: Some("high".to_string()), |
| 750 | role_source: None, |
| 751 | loadout_source: None, |
| 752 | model_class_source: None, |
| 753 | model_source: None, |
| 754 | source: "resolver".to_string(), |
| 755 | }), |
| 756 | saved_session_id: None, |
| 757 | effective_permissions: None, |
| 758 | } |
| 759 | } |
| 760 | |
| 761 | #[test] |
| 762 | fn fleet_run_dto_uses_exact_route_and_types_what_the_ledger_omits() { |
| 763 | let run = run("run-1"); |
| 764 | let receipt = receipt_with_route("run-1"); |
| 765 | let summary = fleet_run_summary(&run, Some(&receipt)); |
| 766 | |
| 767 | assert_eq!(summary.domain, ControlDomain::Fleet); |
| 768 | assert_eq!(summary.run_id, "run-1"); |
| 769 | assert_eq!(summary.status, "running"); |
| 770 | assert_eq!( |
| 771 | summary.route.provider_id, |
| 772 | Known::Known("deepseek".to_string()) |
| 773 | ); |
| 774 | assert_eq!( |
| 775 | summary.route.provider_exact_id, |
| 776 | Known::Known("custom".to_string()), |
| 777 | "the exact provider-table id must not collapse into the generic id" |
| 778 | ); |
| 779 | assert_eq!(summary.route.model, Known::Known("deepseek-v3".to_string())); |
| 780 | assert_eq!( |
| 781 | summary.route.effective_reasoning, |
| 782 | Known::Known("high".to_string()) |
| 783 | ); |
| 784 | assert_eq!( |
| 785 | summary.route.requested_reasoning.unknown_reason(), |
| 786 | Some(UnknownReason::NotRecorded), |
| 787 | "the ledger records the effective tier only; do not invent the request" |
| 788 | ); |
| 789 | assert_eq!(summary.route.reasoning_downgraded(), None); |
| 790 | assert_eq!( |
| 791 | summary.route.route_source, |
| 792 | Known::Known("resolver".to_string()) |
| 793 | ); |
| 794 | assert_eq!( |
| 795 | summary.usage.total_tokens.unknown_reason(), |
| 796 | Some(UnknownReason::NotRecorded) |
| 797 | ); |
| 798 | assert_eq!( |
| 799 | summary.lifecycle_seq.unknown_reason(), |
| 800 | Some(UnknownReason::NotApplicable) |
| 801 | ); |
| 802 | let detail = summary.render_detail(); |
| 803 | assert!(detail.starts_with("fleet: run-1\n"), "{detail}"); |
| 804 | assert!(detail.contains("\nfleet: stopship\n"), "{detail}"); |
| 805 | assert!(!detail.contains("\npod:"), "{detail}"); |
| 806 | let wire = serde_json::to_value(&summary).expect("serialize stable run DTO"); |
| 807 | assert!(wire.get("fleet").is_some(), "{wire}"); |
| 808 | } |
| 809 | |
| 810 | #[test] |
| 811 | fn a_run_without_a_receipt_reports_every_route_field_unknown() { |
| 812 | let summary = fleet_run_summary(&run("run-2"), None); |
| 813 | for reason in [ |
| 814 | summary.route.provider_id.unknown_reason(), |
| 815 | summary.route.model.unknown_reason(), |
| 816 | summary.route.effective_reasoning.unknown_reason(), |
| 817 | summary.route.route_source.unknown_reason(), |
| 818 | ] { |
| 819 | assert_eq!(reason, Some(UnknownReason::NotRecorded)); |
| 820 | } |
| 821 | } |
| 822 | |
| 823 | #[test] |
| 824 | fn fleet_run_pages_are_bounded() { |
| 825 | let mut state = FleetLedgerState::default(); |
| 826 | for index in 0..10 { |
| 827 | let id = format!("run-{index:03}"); |
| 828 | state.runs.insert(id.clone(), run(&id)); |
| 829 | } |
| 830 | let page = fleet_run_page(&state, 3); |
| 831 | assert_eq!(page.total, 10); |
| 832 | assert_eq!(page.runs.len(), 3); |
| 833 | assert_eq!(page.truncated, 7); |
| 834 | } |
| 835 | |
| 836 | #[test] |
| 837 | fn an_absent_ledger_is_reported_and_not_created() { |
| 838 | let dir = tempfile::tempdir().unwrap(); |
| 839 | for surface in ControlSurface::ALL { |
| 840 | let receipt = |
| 841 | execute_fleet_control(*surface, dir.path(), ControlOperation::FleetStatus, None); |
| 842 | assert_eq!( |
| 843 | receipt.availability.reason(), |
| 844 | Some(UnavailableReason::NoFleetLedger), |
| 845 | "{surface}" |
| 846 | ); |
| 847 | assert_eq!(receipt.outcome, LifecycleOutcome::Rejected); |
| 848 | } |
| 849 | assert!( |
| 850 | !fleet_ledger_path(dir.path()).exists(), |
| 851 | "a read verb must not create the durable ledger" |
| 852 | ); |
| 853 | } |
| 854 | |
| 855 | #[test] |
| 856 | fn durable_fleet_status_is_identical_on_every_surface() { |
| 857 | let dir = tempfile::tempdir().unwrap(); |
| 858 | // Creating the manager is what makes the ledger exist. |
| 859 | FleetManager::open(dir.path()).unwrap(); |
| 860 | assert_eq!( |
| 861 | ControlOperation::FleetStatus.descriptor().cli_invocation, |
| 862 | "codewhale fleet status", |
| 863 | "fleet remains the canonical public command" |
| 864 | ); |
| 865 | let mut rendered = std::collections::BTreeSet::new(); |
| 866 | for surface in ControlSurface::ALL { |
| 867 | let receipt = |
| 868 | execute_fleet_control(*surface, dir.path(), ControlOperation::FleetStatus, None); |
| 869 | assert_eq!(receipt.operation_id, "fleet.status"); |
| 870 | assert_eq!(receipt.authority, ControlAuthority::Read); |
| 871 | assert_eq!(receipt.persistence, PersistenceScope::FleetLedger); |
| 872 | assert_eq!(receipt.outcome, LifecycleOutcome::Inspected); |
| 873 | assert!( |
| 874 | receipt |
| 875 | .detail |
| 876 | .iter() |
| 877 | .any(|line| line.starts_with("fleet: runs=")), |
| 878 | "the durable ledger snapshot must keep its receipt prefix" |
| 879 | ); |
| 880 | let mut normalized = receipt.clone(); |
| 881 | normalized.surface = ControlSurface::Cli; |
| 882 | rendered.insert(normalized.render()); |
| 883 | } |
| 884 | assert_eq!(rendered.len(), 1, "surfaces rendered different results"); |
| 885 | } |
| 886 | |
| 887 | #[test] |
| 888 | fn fleet_resume_receipt_preserves_established_fleet_detail_prefixes() { |
| 889 | let dir = tempfile::tempdir().unwrap(); |
| 890 | let ledger = FleetLedger::open(dir.path()).unwrap(); |
| 891 | ledger.create_run(&run("run-1")).unwrap(); |
| 892 | let manager = FleetManager::open(dir.path()).unwrap(); |
| 893 | |
| 894 | let receipt = execute_fleet_control_with( |
| 895 | ControlSurface::Cli, |
| 896 | dir.path(), |
| 897 | fleet_control_context(dir.path()), |
| 898 | &manager, |
| 899 | ControlOperation::FleetResume, |
| 900 | Some("run-1"), |
| 901 | ); |
| 902 | |
| 903 | assert_eq!(receipt.operation_id, "fleet.resume"); |
| 904 | assert_eq!(receipt.outcome, LifecycleOutcome::NoChange); |
| 905 | assert_eq!( |
| 906 | receipt.detail.first().map(String::as_str), |
| 907 | Some("fleet resume: run-1 reclaimed_stale=0 restarted=0 failed=0 escalated=0") |
| 908 | ); |
| 909 | assert_eq!( |
| 910 | receipt.detail.get(1).map(String::as_str), |
| 911 | Some( |
| 912 | "fleet: runs=1 queued=0 running=0 completed=0 partial=0 failed=0 restarted=0 \ |
| 913 | escalated=0 transport_failed=0 task_failed=0 verifier_failed=0 cancelled=0 stale=0" |
| 914 | ) |
| 915 | ); |
| 916 | } |
| 917 | |
| 918 | #[test] |
| 919 | fn fleet_restart_is_cli_only_and_says_so_elsewhere() { |
| 920 | let dir = tempfile::tempdir().unwrap(); |
| 921 | FleetManager::open(dir.path()).unwrap(); |
| 922 | { |
| 923 | let surface = ControlSurface::Slash; |
| 924 | let receipt = execute_fleet_control( |
| 925 | surface, |
| 926 | dir.path(), |
| 927 | ControlOperation::FleetRestart, |
| 928 | Some("worker-1"), |
| 929 | ); |
| 930 | assert_eq!(receipt.outcome, LifecycleOutcome::Rejected); |
| 931 | assert_eq!( |
| 932 | receipt.availability.reason(), |
| 933 | Some(UnavailableReason::SurfaceNotSupported) |
| 934 | ); |
| 935 | assert!( |
| 936 | receipt |
| 937 | .availability |
| 938 | .hint() |
| 939 | .is_some_and(|hint| hint.contains("codewhale fleet restart")) |
| 940 | ); |
| 941 | } |
| 942 | } |
| 943 | |
| 944 | #[test] |
| 945 | fn interrupt_requires_an_exact_worker_id() { |
| 946 | let dir = tempfile::tempdir().unwrap(); |
| 947 | FleetManager::open(dir.path()).unwrap(); |
| 948 | for bad in [None, Some(""), Some("worker one"), Some("../escape")] { |
| 949 | let receipt = execute_fleet_control( |
| 950 | ControlSurface::Slash, |
| 951 | dir.path(), |
| 952 | ControlOperation::FleetInterrupt, |
| 953 | bad, |
| 954 | ); |
| 955 | assert_eq!( |
| 956 | receipt.failure.as_ref().map(|failure| failure.kind), |
| 957 | Some(ControlFailureKind::InvalidTarget), |
| 958 | "{bad:?}" |
| 959 | ); |
| 960 | } |
| 961 | } |
| 962 | |
| 963 | /// #4022: an id that is not in the ledger must be refused as `not_found` |
| 964 | /// *before* any durable write. Resuming a typo used to reconcile nothing, |
| 965 | /// write a run-status record under the typed id, and report `no_change`. |
| 966 | #[test] |
| 967 | fn resuming_an_unknown_run_is_not_found_and_writes_nothing() { |
| 968 | let dir = tempfile::tempdir().unwrap(); |
| 969 | FleetManager::open(dir.path()).unwrap(); |
| 970 | let ledger = fleet_ledger_path(dir.path()); |
| 971 | let before = std::fs::read(&ledger).unwrap(); |
| 972 | |
| 973 | for surface in ControlSurface::ALL { |
| 974 | let receipt = execute_fleet_control( |
| 975 | *surface, |
| 976 | dir.path(), |
| 977 | ControlOperation::FleetResume, |
| 978 | Some("run-does-not-exist"), |
| 979 | ); |
| 980 | assert_eq!(receipt.outcome, LifecycleOutcome::Rejected, "{surface}"); |
| 981 | assert_eq!( |
| 982 | receipt.failure.as_ref().map(|failure| failure.kind), |
| 983 | Some(ControlFailureKind::NotFound), |
| 984 | "{surface}" |
| 985 | ); |
| 986 | } |
| 987 | |
| 988 | assert_eq!( |
| 989 | std::fs::read(&ledger).unwrap(), |
| 990 | before, |
| 991 | "a refused resume must not append to the durable ledger" |
| 992 | ); |
| 993 | let state = FleetManager::open(dir.path()) |
| 994 | .unwrap() |
| 995 | .rebuild_state() |
| 996 | .unwrap(); |
| 997 | assert!( |
| 998 | !state |
| 999 | .run_status_overrides |
| 1000 | .contains_key("run-does-not-exist"), |
| 1001 | "a caller-supplied id must never become a durable ledger key" |
| 1002 | ); |
| 1003 | } |
| 1004 | |
| 1005 | /// #4022: newest-first ordering is computed on parsed instants, so a run |
| 1006 | /// recorded at a non-UTC offset still sorts by when it actually happened. |
| 1007 | #[test] |
| 1008 | fn runs_sort_by_utc_instant_not_by_rendered_text() { |
| 1009 | let mut state = FleetLedgerState::default(); |
| 1010 | // 2026-07-26T00:30:00+02:00 == 2026-07-25T22:30:00Z, i.e. *earlier* |
| 1011 | // than the UTC-stamped run even though its text sorts later. |
| 1012 | let mut earlier = run("run-offset"); |
| 1013 | earlier.created_at = "2026-07-26T00:30:00+02:00".to_string(); |
| 1014 | let mut later = run("run-utc"); |
| 1015 | later.created_at = "2026-07-25T23:00:00Z".to_string(); |
| 1016 | state.runs.insert("run-offset".to_string(), earlier); |
| 1017 | state.runs.insert("run-utc".to_string(), later); |
| 1018 | |
| 1019 | let page = fleet_run_page(&state, DEFAULT_RUN_LIST_LIMIT); |
| 1020 | assert_eq!( |
| 1021 | page.runs |
| 1022 | .iter() |
| 1023 | .map(|run| run.run_id.as_str()) |
| 1024 | .collect::<Vec<_>>(), |
| 1025 | vec!["run-utc", "run-offset"], |
| 1026 | "a string compare would have put run-offset first" |
| 1027 | ); |
| 1028 | } |
| 1029 | |
| 1030 | #[test] |
| 1031 | fn status_and_inspection_rendering_stay_bounded() { |
| 1032 | let mut snapshot = FleetStatusSnapshot::default(); |
| 1033 | for index in 0..(MAX_STATUS_WORKER_ROWS + 5) { |
| 1034 | snapshot |
| 1035 | .workers |
| 1036 | .insert(format!("worker-{index:03}"), FleetWorkerStatus::Online); |
| 1037 | } |
| 1038 | let lines = status_lines(&snapshot); |
| 1039 | // summary + "workers:" + capped rows + the omission notice |
| 1040 | assert_eq!(lines.len(), 1 + 1 + MAX_STATUS_WORKER_ROWS + 1); |
| 1041 | assert_eq!( |
| 1042 | lines.first().map(String::as_str), |
| 1043 | Some( |
| 1044 | "fleet: runs=0 queued=0 running=0 completed=0 partial=0 failed=0 restarted=0 \ |
| 1045 | escalated=0 transport_failed=0 task_failed=0 verifier_failed=0 cancelled=0 stale=0" |
| 1046 | ) |
| 1047 | ); |
| 1048 | assert!(lines.last().unwrap().contains("5 more worker(s) omitted")); |
| 1049 | } |
| 1050 | } |
| 1051 |