返回 CodeWhale
mod.rs
根目录 / crates / tui / src / tools / workflow / mod.rs
1 //! Model-facing Workflow runner over the live sub-agent runtime.
2 //!
3 //! The JS VM stays in `codewhale-workflow-js`; this module supplies the TUI
4 //! driver that turns each `task(...)` call into a real `SubAgentManager` spawn.
5
6 mod plan_schema;
7 #[cfg(test)]
8 mod shortlist_tests;
9
10 use std::collections::{HashMap, HashSet};
11 use std::path::{Path, PathBuf};
12 use std::sync::atomic::{AtomicU32, Ordering};
13 use std::sync::{Arc, Mutex, MutexGuard, OnceLock};
14 use std::time::{SystemTime, UNIX_EPOCH};
15
16 use async_trait::async_trait;
17 use codewhale_workflow::{
18 AgentType, BranchResult, BranchSpec, BudgetSpec, ControlNodeKind, ControlNodeResult,
19 FleetRoleMap, GateKind, GateOn, GateOutcome, GateSpec, GateState, GateStatusLine,
20 HandoffArtifact, LaneGateBoard, LeafResult, LeafSpec, ReduceSpec, SequenceSpec, TaskMode,
21 WorkflowExecution as IrWorkflowExecution, WorkflowFleetLimits, WorkflowMemoUsage, WorkflowNode,
22 WorkflowRunStatus as IrWorkflowRunStatus, WorkflowSpec, WorkflowUsage,
23 compile_javascript_workflow, compile_typescript_workflow, leaf_wants_worktree,
24 resolve_workflow_agent,
25 };
26 use codewhale_workflow_js::{
27 BudgetSnapshot, DriverError, ProgressEvent, SCHEMA_RAW_PREVIEW_CHARS, SpawnedTask,
28 TaskCompletion, TaskRequest, WORKFLOW_LIFETIME_CAP, WORKFLOW_MAX_CONCURRENT, WorkflowDriver,
29 WorkflowRunCancel, WorkflowVm,
30 };
31 use serde::{Deserialize, Serialize};
32 use serde_json::{Value, json};
33 use tokio::sync::{OwnedSemaphorePermit, Semaphore, mpsc, oneshot};
34 use tokio_util::sync::CancellationToken;
35 use uuid::Uuid;
36
37 use crate::core::events::Event;
38 use crate::fleet::role::{FleetRole, public_role_label};
39 use crate::tools::spec::{
40 ApprovalRequirement, ToolCapability, ToolContext, ToolError, ToolResult, ToolSpec,
41 optional_bool, optional_str, optional_u64,
42 };
43 use crate::tools::subagent::{
44 SharedSubAgentManager, SubAgentCompletion, SubAgentManager, SubAgentResult, SubAgentRuntime,
45 SubAgentStatus, WorkflowTaskSpawnIdentity, WorkflowTaskSpawnMetadata, spawn_workflow_task,
46 };
47 use crate::tools::verifier::run_workflow_completion_gates;
48 use crate::tools::workflow_plan_approval::{
49 WorkflowPlanApprovalReceipt, analyze_workflow_plan_approval_with_config, analyze_workflow_spec,
50 workflow_approval_requirement_for,
51 };
52 use crate::utils::spawn_supervised;
53 use crate::work_graph::{
54 CancelOutcome, EvidenceKind, EvidenceRef, OperationIntent, OperationObservation,
55 OperationOwnerSnapshot, OwnerState, SharedWorkRuntime,
56 };
57
58 /// Keep promoted artifacts compact without clipping ordinary evidence reports.
59 /// A 900-character cap cut six-line source receipts in half during live Fleet
60 /// acceptance, so downstream roles could not evaluate evidence the host had
61 /// already approved.
62 const WORKFLOW_HANDOFF_MAX_CHARS: usize = 4_000;
63
64 /// Model-facing run-record payloads carry only the newest events; the full
65 /// stream persists per-event in `.codewhale/workflow-runs.jsonl` (#2974).
66 const WORKFLOW_RESULT_EVENTS_TAIL: usize = 50;
67 /// Bounded tail for free-form progress lines in model-facing payloads.
68 const WORKFLOW_RESULT_PROGRESS_TAIL: usize = 20;
69 /// Bounded tail for rejected child dispatches in the model-facing payload.
70 /// The durable run journal retains the complete failure ledger.
71 const WORKFLOW_RESULT_DISPATCH_FAILURES_TAIL: usize = 12;
72 /// Per-field cap for one model-facing dispatch-failure receipt.
73 const WORKFLOW_RESULT_DISPATCH_FAILURE_FIELD_MAX_CHARS: usize = 320;
74 /// Char cap for the VM `result` / `verification` values in model-facing
75 /// payloads (matches the handoff compaction budget); oversized values
76 /// collapse to a preview plus a journal pointer.
77 const WORKFLOW_RESULT_VALUE_MAX_CHARS: usize = 4_000;
78 /// Char cap per leaf output preview inside the model-facing execution
79 /// receipt; full child output stays retrievable via the worker ledger and
80 /// the journal.
81 const WORKFLOW_RESULT_LEAF_OUTPUT_MAX_CHARS: usize = 500;
82 const WORKFLOW_COMPLETION_MAX_BYTES: usize = 8 * 1024;
83 /// Stated upper bound for a bounded model-facing run-record payload; the
84 /// payload tests assert every `start`/`run`/`status` result stays below it.
85 const WORKFLOW_RESULT_MAX_CHARS: usize = 24_000;
86 /// In-memory (and snapshot) event retention per run: only the newest events
87 /// are kept; older ones remain in the per-event journal lines (#2974).
88 const WORKFLOW_RUN_EVENTS_MAX_RETAINED: usize = 1_000;
89 /// In-memory progress retention per run. Progress is journaled line-by-line,
90 /// so the owner record only needs a bounded newest tail for status/history.
91 const WORKFLOW_RUN_PROGRESS_MAX_RETAINED: usize = 1_000;
92 /// In-memory structured dispatch-failure retention per run. The exact count
93 /// is stored separately and each rejection remains durable as a typed event.
94 const WORKFLOW_RUN_DISPATCH_FAILURES_MAX_RETAINED: usize = WORKFLOW_RESULT_DISPATCH_FAILURES_TAIL;
95 /// Progress lines the host detail projection keeps for the run manager's
96 /// detail pane — the newest few, matching what a human scans at a glance.
97 const HOST_RUN_PROGRESS_TAIL: usize = 3;
98
99 #[derive(Clone)]
100 pub struct WorkflowTool {
101 manager: SharedSubAgentManager,
102 runtime: SubAgentRuntime,
103 approval_decision: &'static str,
104 }
105
106 impl WorkflowTool {
107 #[must_use]
108 pub fn new(manager: SharedSubAgentManager, runtime: SubAgentRuntime) -> Self {
109 Self {
110 manager,
111 runtime,
112 approval_decision: "approved",
113 }
114 }
115
116 /// Mark execution as approved by the user's explicit `workflow run`
117 /// command rather than by an Engine tool-call approval gate.
118 #[must_use]
119 pub(crate) fn with_explicit_cli_approval(mut self) -> Self {
120 self.approval_decision = "approved_explicit_cli_command";
121 self
122 }
123 }
124
125 type SharedWorkflowRuns = Arc<Mutex<HashMap<String, WorkflowRunRecord>>>;
126 type SharedWorkflowControllers = Arc<Mutex<HashMap<String, Arc<WorkflowRunController>>>>;
127 type SharedWorkflowLifecycles = Arc<Mutex<HashMap<String, WorkflowWorkLifecycle>>>;
128
129 #[derive(Clone)]
130 struct WorkflowWorkLifecycle {
131 work: SharedWorkRuntime,
132 session_id: String,
133 external: String,
134 }
135
136 impl WorkflowWorkLifecycle {
137 fn register(
138 context: &ToolContext,
139 run_id: &str,
140 title: &str,
141 ) -> Result<Option<Self>, ToolError> {
142 let Some(work) = context.runtime.work.clone() else {
143 return Ok(None);
144 };
145 let lifecycle = Self {
146 work,
147 session_id: context.state_namespace.clone(),
148 external: format!("workflow:{run_id}"),
149 };
150 // Work-graph registration is observability bookkeeping: a transiently
151 // busy To-do/Plan state must not veto the run. Unbound workflows keep
152 // every later reconcile skipped (`if let Some(lifecycle)` at the call
153 // sites, and `attach_bound_workflow_lifecycles` binds by lookup).
154 if let Err(err) = lifecycle.work.register_operation(
155 &lifecycle.session_id,
156 OperationIntent::new(
157 lifecycle.external.clone(),
158 title,
159 true,
160 "workflow",
161 format!("workflow:{run_id}:start"),
162 ),
163 ) {
164 tracing::warn!(
165 run_id = %run_id,
166 error = %err,
167 "workflow work-graph registration skipped; running unbound"
168 );
169 return Ok(None);
170 }
171 Ok(Some(lifecycle))
172 }
173
174 fn for_bound(context: &ToolContext, run_id: &str) -> Option<Self> {
175 let work = context.runtime.work.clone()?;
176 let external = format!("workflow:{run_id}");
177 work.has_operation_binding(Some(&context.state_namespace), &external)
178 .then(|| Self {
179 work,
180 session_id: context.state_namespace.clone(),
181 external,
182 })
183 }
184
185 fn reconcile_record(&self, record: &WorkflowRunRecord) -> Result<bool, String> {
186 let output = record.result.as_ref().and_then(|result| {
187 serde_json::to_vec(result).ok().and_then(|bytes| {
188 EvidenceRef::new(
189 EvidenceKind::Receipt {
190 owner: "workflow".to_string(),
191 },
192 format!("workflow:{}:result", record.run_id),
193 Some(u64::try_from(bytes.len()).unwrap_or(u64::MAX)),
194 false,
195 )
196 .ok()
197 })
198 });
199 let state = owner_state_for_run_status(record.status);
200 let mut snapshot = OperationOwnerSnapshot::new(
201 self.external.clone(),
202 state,
203 record.lifecycle_seq,
204 i64::try_from(record.completed_at_ms.unwrap_or(record.started_at_ms))
205 .unwrap_or(i64::MAX),
206 );
207 if let Some(output) = output {
208 snapshot = snapshot.with_output(output);
209 }
210 self.work.reconcile_operation(&self.session_id, snapshot)
211 }
212
213 fn reconcile_cancel(&self, outcome: CancelOutcome) -> Result<bool, String> {
214 self.work.reconcile_observation(
215 &self.session_id,
216 &self.external,
217 OperationObservation::CancelUpdate {
218 outcome,
219 at: i64::try_from(now_ms()).unwrap_or(i64::MAX),
220 },
221 )
222 }
223
224 fn reconcile_spawn_failure(&self) {
225 let _ = self.work.reconcile_operation(
226 &self.session_id,
227 OperationOwnerSnapshot::new(
228 self.external.clone(),
229 OwnerState::Failed,
230 1,
231 i64::try_from(now_ms()).unwrap_or(i64::MAX),
232 ),
233 );
234 }
235
236 fn reconcile_missing(&self) {
237 let _ = self.work.reconcile_observation(
238 &self.session_id,
239 &self.external,
240 OperationObservation::OwnerMissing {
241 checked_at: i64::try_from(now_ms()).unwrap_or(i64::MAX),
242 },
243 );
244 }
245 }
246
247 /// Owner-snapshot projection of a workflow run status. Total and lossless
248 /// for terminal truth: Degraded stays Degraded — collapsing it into
249 /// Completed let dashboards and automation read a partial workflow as an
250 /// ordinary success (#5582). The run record and report keep the per-slot
251 /// detail either way.
252 fn owner_state_for_run_status(status: WorkflowRunStatus) -> OwnerState {
253 match status {
254 WorkflowRunStatus::Running => OwnerState::Running,
255 WorkflowRunStatus::Completed => OwnerState::Completed,
256 WorkflowRunStatus::Degraded => OwnerState::Degraded,
257 WorkflowRunStatus::Failed => OwnerState::Failed,
258 WorkflowRunStatus::Cancelled => OwnerState::Cancelled,
259 }
260 }
261
262 struct WorkflowRunController {
263 driver: Arc<SubAgentWorkflowDriver>,
264 vm_cancel: WorkflowRunCancel,
265 run_handle: Mutex<Option<tokio::task::JoinHandle<()>>>,
266 /// Only detached starts wake the parent; `run` already returns the result.
267 parent_completion_tx: Option<mpsc::Sender<SubAgentCompletion>>,
268 }
269
270 impl WorkflowRunController {
271 fn new(driver: Arc<SubAgentWorkflowDriver>, vm_cancel: WorkflowRunCancel) -> Self {
272 Self {
273 driver,
274 vm_cancel,
275 run_handle: Mutex::new(None),
276 parent_completion_tx: None,
277 }
278 }
279
280 fn with_parent_completion(mut self, detached: bool) -> Self {
281 if detached {
282 self.parent_completion_tx = self.driver.runtime.parent_completion_tx.clone();
283 }
284 self
285 }
286
287 fn set_run_handle(&self, handle: tokio::task::JoinHandle<()>) {
288 if let Ok(mut guard) = self.run_handle.lock() {
289 *guard = Some(handle);
290 }
291 }
292
293 fn cancel(&self) {
294 self.vm_cancel.cancel();
295 self.driver.finalize_running_tasks_cancelled();
296 self.driver.force_cancel_all();
297 if let Ok(mut guard) = self.run_handle.lock()
298 && let Some(handle) = guard.take()
299 {
300 handle.abort();
301 }
302 }
303 }
304
305 /// Queue the terminal receipt before releasing the live controller. The
306 /// headless settlement probe observes this same map, so it cannot see an idle
307 /// Workflow before the receipt is in the existing Engine completion inbox.
308 fn finish_workflow_controller(state: &WorkflowWorkspaceState, record: &WorkflowRunRecord) {
309 let mut controllers = state
310 .controllers
311 .lock()
312 .unwrap_or_else(|poison| poison.into_inner());
313 if let Some(controller) = controllers.get(&record.run_id)
314 && let Some(tx) = &controller.parent_completion_tx
315 {
316 // A racing explicit cancel is authoritative even if the VM captured
317 // its successful snapshot just before the cancellation was recorded.
318 let cancelled = state
319 .runs
320 .lock()
321 .unwrap_or_else(|poison| poison.into_inner())
322 .get(&record.run_id)
323 .filter(|current| current.status == WorkflowRunStatus::Cancelled)
324 .cloned();
325 let record = cancelled.as_ref().unwrap_or(record);
326 let receipt = json!({
327 "event": if record.status == WorkflowRunStatus::Completed {
328 "workflow.completed"
329 } else {
330 "workflow.failed"
331 },
332 "agent_id": truncate_chars(&record.run_id, 64),
333 "agent_type": "workflow",
334 "run_id": truncate_chars(&record.run_id, 64),
335 "status": record.status,
336 "child_count": record.child_ids.len(),
337 "detail": { "tool": "workflow", "action": "status", "run_id": truncate_chars(&record.run_id, 64) },
338 });
339 let mut summary = format!(
340 "Workflow {} ended {:?}.",
341 truncate_chars(&record.run_id, 64),
342 record.status
343 );
344 if let Some(goal) = &record.workflow_goal {
345 summary.push_str(&format!("\nGoal: {}", truncate_chars(goal, 160)));
346 }
347 if let Some(error) = &record.error {
348 summary.push_str(&format!("\nError: {}", truncate_chars(error, 512)));
349 }
350 if let Some(result) = &record.result {
351 summary.push_str(&format!(
352 "\nResult preview: {}",
353 truncate_chars(&result.to_string(), 512)
354 ));
355 }
356 summary.push_str("\nInspect the recorded result and evidence before summarizing the outcome and remaining work.");
357 let payload =
358 format!("{summary}\n<codewhale:subagent.done>{receipt}</codewhale:subagent.done>");
359 debug_assert!(payload.len() <= WORKFLOW_COMPLETION_MAX_BYTES);
360 let _ = tx.try_send(SubAgentCompletion {
361 owner_session_id: controller.driver.owner_session_id.clone(),
362 agent_id: record.run_id.clone(),
363 payload,
364 });
365 }
366 controllers.remove(&record.run_id);
367 }
368
369 /// Active controllers, including gaps between child phases. This reads only
370 /// existing live state and never hydrates a journal or creates a workspace.
371 pub(crate) fn live_workflow_count(workspace: &Path, owner_session_id: &str) -> usize {
372 let Some(state) = peek_shared_workflow_state(workspace) else {
373 return 0;
374 };
375 state
376 .controllers
377 .lock()
378 .unwrap_or_else(|poison| poison.into_inner())
379 .values()
380 .filter(|controller| controller.driver.owner_session_id == owner_session_id)
381 .count()
382 }
383
384 #[derive(Debug, Clone, Serialize)]
385 struct WorkflowRunSummary {
386 run_id: String,
387 status: WorkflowRunStatus,
388 lifecycle_seq: u64,
389 started_at_ms: u64,
390 completed_at_ms: Option<u64>,
391 source_path: Option<PathBuf>,
392 workflow_id: Option<String>,
393 workflow_goal: Option<String>,
394 token_budget: Option<u64>,
395 child_count: usize,
396 schema_error_count: usize,
397 /// Failed `responseSchema` decodes a bounded repair followed (#5583),
398 /// including succeeded ones.
399 #[serde(default)]
400 schema_repair_count: u64,
401 dispatch_failure_count: u64,
402 progress_count: u64,
403 last_progress: Option<String>,
404 event_count: usize,
405 last_event_type: Option<String>,
406 leaf_count: usize,
407 branch_count: usize,
408 control_count: usize,
409 execution_status: Option<IrWorkflowRunStatus>,
410 gate_count: usize,
411 blocked_gate_count: usize,
412 gate_status: Vec<GateStatusLine>,
413 error: Option<String>,
414 /// Run-wide usage totals reconciled from per-task telemetry (#2974).
415 usage: Option<WorkflowRunUsage>,
416 /// Events evicted from the retained tail; full stream in the journal.
417 events_dropped: u64,
418 }
419
420 #[derive(Debug, Clone, Serialize, Deserialize)]
421 struct WorkflowSchemaError {
422 task_id: String,
423 message: String,
424 /// Which decode stage failed (#5583): `json_parse` or
425 /// `schema_validation`. Journals written before the split existed
426 /// default to the validation bucket — the common failure shape.
427 #[serde(default = "legacy_schema_failure_kind")]
428 kind: String,
429 /// 1-based attempt that failed terminally (1 = no repair was tried).
430 #[serde(default = "legacy_schema_failure_attempt")]
431 attempt: u32,
432 /// Bounded preview of the raw reply; the full text lives in `artifact`
433 /// when one was written.
434 #[serde(default)]
435 raw_preview: String,
436 /// True when the carried raw text was capped at the carry limit.
437 #[serde(default)]
438 raw_truncated: bool,
439 /// Path of the durable raw-output artifact, when the reply exceeded the
440 /// preview bound.
441 #[serde(default, skip_serializing_if = "Option::is_none")]
442 artifact: Option<String>,
443 }
444
445 /// One failed `responseSchema` decode that a bounded repair followed
446 /// (#5583). Recorded even when the repair succeeds, so a repaired run shows
447 /// exactly what was repaired and why.
448 #[derive(Debug, Clone, Serialize, Deserialize)]
449 struct WorkflowSchemaRepairAttempt {
450 task_id: String,
451 #[serde(default = "legacy_schema_failure_kind")]
452 kind: String,
453 /// 1-based number of the attempt that failed.
454 attempt: u32,
455 message: String,
456 #[serde(default)]
457 raw_preview: String,
458 #[serde(default)]
459 raw_truncated: bool,
460 #[serde(default, skip_serializing_if = "Option::is_none")]
461 artifact: Option<String>,
462 }
463
464 fn legacy_schema_failure_kind() -> String {
465 "schema_validation".to_string()
466 }
467
468 fn legacy_schema_failure_attempt() -> u32 {
469 1
470 }
471
472 /// One `task()` dispatch the driver rejected before any child agent existed.
473 /// Inside `parallel()` the JS throw collapses into a `null` slot, so without
474 /// this ledger a run whose fan-out never dispatched anything still reads as
475 /// successful orchestration (#5035).
476 #[derive(Debug, Clone, Serialize, Deserialize)]
477 struct WorkflowDispatchFailure {
478 at_ms: u64,
479 #[serde(default, skip_serializing_if = "Option::is_none")]
480 label: Option<String>,
481 #[serde(default, skip_serializing_if = "Option::is_none")]
482 phase: Option<String>,
483 message: String,
484 }
485
486 #[derive(Debug, Clone, Serialize, Deserialize)]
487 struct WorkflowUiEvent {
488 at_ms: u64,
489 /// Conversation that owns this event. Legacy journal entries omit it and
490 /// therefore fail closed at every session-facing projection.
491 #[serde(default, skip_serializing_if = "Option::is_none")]
492 owner_session_id: Option<String>,
493 #[serde(flatten)]
494 kind: WorkflowUiEventKind,
495 }
496
497 impl WorkflowUiEvent {
498 fn new(owner_session_id: &str, kind: WorkflowUiEventKind) -> Self {
499 Self {
500 at_ms: now_ms(),
501 owner_session_id: Some(owner_session_id.to_string()),
502 kind,
503 }
504 }
505
506 fn at(at_ms: u64, owner_session_id: &str, kind: WorkflowUiEventKind) -> Self {
507 Self {
508 at_ms,
509 owner_session_id: Some(owner_session_id.to_string()),
510 kind,
511 }
512 }
513
514 fn event_type(&self) -> &'static str {
515 self.kind.event_type()
516 }
517 }
518
519 #[derive(Debug, Clone, Serialize, Deserialize)]
520 #[serde(tag = "type", rename_all = "snake_case")]
521 enum WorkflowUiEventKind {
522 RunStarted {
523 workflow_id: Option<String>,
524 workflow_goal: Option<String>,
525 source_path: Option<PathBuf>,
526 token_budget: Option<u64>,
527 },
528 RunCompleted {
529 status: WorkflowRunStatus,
530 error: Option<String>,
531 /// Run-wide usage totals reconciled from per-task telemetry (#2974).
532 #[serde(default, skip_serializing_if = "Option::is_none")]
533 usage: Option<WorkflowRunUsage>,
534 },
535 RunCancelled {
536 reason: String,
537 },
538 PhaseStarted {
539 title: String,
540 },
541 TaskStarted(Box<WorkflowTaskStartedEvent>),
542 TaskCompleted {
543 task_id: String,
544 status: IrWorkflowRunStatus,
545 /// Per-worker telemetry captured at terminal delivery (#2974).
546 #[serde(default, skip_serializing_if = "Option::is_none")]
547 usage: Option<WorkflowTaskUsage>,
548 },
549 GateUpdated {
550 gate_id: String,
551 role: String,
552 gate: String,
553 state: String,
554 blocked_role: Option<String>,
555 blocked_reason: Option<String>,
556 },
557 HandoffPromoted {
558 artifact_id: String,
559 gate_id: String,
560 kind: String,
561 from_role: String,
562 to_role: String,
563 producer_task_id: String,
564 },
565 HandoffConsumed {
566 artifact_id: String,
567 kind: String,
568 from_role: String,
569 to_role: String,
570 consumer_task_id: String,
571 },
572 TaskSchemaValidationFailed {
573 task_id: String,
574 message: String,
575 },
576 TaskDispatchFailed {
577 #[serde(default, skip_serializing_if = "Option::is_none")]
578 label: Option<String>,
579 #[serde(default, skip_serializing_if = "Option::is_none")]
580 phase: Option<String>,
581 message: String,
582 },
583 BudgetUpdated {
584 total: Option<u64>,
585 spent: u64,
586 remaining: Option<u64>,
587 },
588 Log {
589 message: String,
590 },
591 }
592
593 mod usage;
594
595 use usage::{WorkflowRunUsage, WorkflowTaskUsage, WorkflowTokenSource, sum_optional_usage};
596
597 #[derive(Debug, Clone, Serialize, Deserialize)]
598 struct WorkflowTaskStartedEvent {
599 task_id: String,
600 label: Option<String>,
601 /// Fleet role declared on the step, if any (#4177).
602 role: Option<String>,
603 profile: Option<String>,
604 model: Option<String>,
605 strength: Option<String>,
606 thinking: Option<String>,
607 /// Reasoning the task requested, verbatim (`inherit`/`auto`/effort) (#4039).
608 #[serde(default, skip_serializing_if = "Option::is_none")]
609 requested_reasoning: Option<String>,
610 /// Reasoning the child runtime was actually installed with (#4039). Absent
611 /// when the resolved route carries no reasoning control; consumers render
612 /// that as unknown rather than inventing an effort.
613 #[serde(default, skip_serializing_if = "Option::is_none")]
614 effective_reasoning: Option<String>,
615 /// Resolved fleet role after roster lookup (#4177).
616 resolved_role: Option<String>,
617 /// Resolved AgentProfile id after fleet resolution (#4177).
618 resolved_profile: Option<String>,
619 resolved_provider: String,
620 resolved_model: String,
621 route_source: String,
622 #[serde(default, skip_serializing_if = "Option::is_none")]
623 child_route: Option<crate::tools::subagent::ChildRouteReceipt>,
624 worktree: bool,
625 workspace: Option<PathBuf>,
626 git_branch: Option<String>,
627 parent_task_id: Option<String>,
628 depth: u32,
629 /// Workflow run that admitted this child (#4119).
630 workflow_run_id: Option<String>,
631 /// Phase title/id active (or declared on the task) at spawn (#4119).
632 workflow_phase_id: Option<String>,
633 /// Typed task label — UI must prefer this over prompt text (#4119).
634 workflow_task_label: Option<String>,
635 /// 0-based admission order among children of this run (#4119).
636 workflow_child_index: Option<u32>,
637 /// Durable exact-Fleet routing receipt: the fixed member identity, its
638 /// exact provider/model, the requested vs. selector vs. provider-effective
639 /// reasoning, where the decision came from, and the Router's exact identity
640 /// when a Router made it. `default` keeps events written before this field
641 /// existed — and every legacy/non-fleet task — readable unchanged.
642 #[serde(default, skip_serializing_if = "Option::is_none")]
643 fleet_receipt: Option<codewhale_workflow::FleetTaskReceipt>,
644 }
645
646 impl WorkflowUiEventKind {
647 fn event_type(&self) -> &'static str {
648 match self {
649 Self::RunStarted { .. } => "run_started",
650 Self::RunCompleted { .. } => "run_completed",
651 Self::RunCancelled { .. } => "run_cancelled",
652 Self::PhaseStarted { .. } => "phase_started",
653 Self::TaskStarted(_) => "task_started",
654 Self::TaskCompleted { .. } => "task_completed",
655 Self::GateUpdated { .. } => "gate_updated",
656 Self::HandoffPromoted { .. } => "handoff_promoted",
657 Self::HandoffConsumed { .. } => "handoff_consumed",
658 Self::TaskSchemaValidationFailed { .. } => "task_schema_validation_failed",
659 Self::TaskDispatchFailed { .. } => "task_dispatch_failed",
660 Self::BudgetUpdated { .. } => "budget_updated",
661 Self::Log { .. } => "log",
662 }
663 }
664 }
665
666 #[derive(Debug, Clone, Serialize, Deserialize)]
667 struct WorkflowRunRecord {
668 run_id: String,
669 /// Conversation that created the run. `None` means a legacy journal entry
670 /// with unknown ownership and is intentionally invisible/non-mutable from
671 /// every session-facing control.
672 #[serde(default, skip_serializing_if = "Option::is_none")]
673 owner_session_id: Option<String>,
674 status: WorkflowRunStatus,
675 #[serde(default)]
676 lifecycle_seq: u64,
677 started_at_ms: u64,
678 completed_at_ms: Option<u64>,
679 source_path: Option<PathBuf>,
680 workflow_id: Option<String>,
681 workflow_goal: Option<String>,
682 token_budget: Option<u64>,
683 child_ids: Vec<String>,
684 /// Exact progress-line count, including entries older than `progress`'s
685 /// bounded in-memory tail. Legacy snapshots are repaired on hydration.
686 #[serde(default)]
687 progress_count: u64,
688 progress: Vec<String>,
689 #[serde(default)]
690 events: Vec<WorkflowUiEvent>,
691 schema_errors: Vec<WorkflowSchemaError>,
692 /// Failed `responseSchema` decodes that a bounded repair followed
693 /// (#5583), including repairs that succeeded. Structured tail of a
694 /// monotonic `schema_repair_count`.
695 #[serde(default)]
696 schema_repairs: Vec<WorkflowSchemaRepairAttempt>,
697 #[serde(default)]
698 schema_repair_count: u64,
699 /// Task dispatches the driver rejected before any child ran (#5035).
700 /// This is the exact, saturating total; `dispatch_failures` is only the
701 /// newest structured tail.
702 #[serde(default)]
703 dispatch_failure_count: u64,
704 #[serde(default, skip_serializing_if = "Vec::is_empty")]
705 dispatch_failures: Vec<WorkflowDispatchFailure>,
706 result: Option<Value>,
707 execution: Option<IrWorkflowExecution>,
708 error: Option<String>,
709 #[serde(default)]
710 verify_on_complete: bool,
711 #[serde(default, skip_serializing_if = "Option::is_none")]
712 verification: Option<Value>,
713 /// Durable elevated-plan approval receipt for audit (#4126).
714 #[serde(default, skip_serializing_if = "Option::is_none")]
715 plan_approval: Option<WorkflowPlanApprovalReceipt>,
716 /// Compact lane gate state for status / panel surfaces (#4179).
717 #[serde(default)]
718 gate_status: Vec<GateStatusLine>,
719 /// Run-wide usage totals reconciled at completion (#2974).
720 #[serde(default, skip_serializing_if = "Option::is_none")]
721 usage: Option<WorkflowRunUsage>,
722 /// Total events recorded for this run (monotonic; survives the bounded
723 /// `events` tail retention) (#2974).
724 #[serde(default)]
725 events_total: u64,
726 /// Events evicted from the in-memory tail; available in the journal.
727 #[serde(default)]
728 events_dropped: u64,
729 }
730
731 impl WorkflowRunRecord {
732 fn new(
733 run_id: String,
734 owner_session_id: Option<String>,
735 source_path: Option<PathBuf>,
736 token_budget: Option<u64>,
737 spec: Option<&WorkflowSpec>,
738 ) -> Self {
739 let gate_status = spec
740 .map(|spec| initial_gate_status(&run_id, &spec.gates))
741 .unwrap_or_default();
742 Self {
743 run_id,
744 owner_session_id,
745 status: WorkflowRunStatus::Running,
746 lifecycle_seq: 1,
747 started_at_ms: now_ms(),
748 completed_at_ms: None,
749 source_path,
750 workflow_id: spec.and_then(|spec| spec.id.clone()),
751 workflow_goal: spec.map(|spec| spec.goal.clone()),
752 token_budget,
753 child_ids: Vec::new(),
754 progress_count: 0,
755 progress: Vec::new(),
756 events: Vec::new(),
757 schema_errors: Vec::new(),
758 schema_repairs: Vec::new(),
759 schema_repair_count: 0,
760 dispatch_failure_count: 0,
761 dispatch_failures: Vec::new(),
762 result: None,
763 execution: None,
764 error: None,
765 verify_on_complete: false,
766 verification: None,
767 plan_approval: None,
768 gate_status,
769 usage: None,
770 events_total: 0,
771 events_dropped: 0,
772 }
773 }
774
775 /// Record one event, bounding retention to the newest
776 /// `WORKFLOW_RUN_EVENTS_MAX_RETAINED` entries (#2974). Every event is
777 /// journaled per-line at record time, so evicted entries remain
778 /// available in `.codewhale/workflow-runs.jsonl`.
779 fn push_event(&mut self, event: WorkflowUiEvent) {
780 self.events_total = self.events_total.saturating_add(1);
781 self.events.push(event);
782 if self.events.len() > WORKFLOW_RUN_EVENTS_MAX_RETAINED {
783 let overflow = self.events.len() - WORKFLOW_RUN_EVENTS_MAX_RETAINED;
784 self.events.drain(..overflow);
785 self.events_dropped = self
786 .events_dropped
787 .saturating_add(u64::try_from(overflow).unwrap_or(u64::MAX));
788 }
789 }
790
791 /// Retain only the newest progress lines while preserving an exact,
792 /// saturating total for summaries and payload truncation receipts.
793 fn push_progress(&mut self, message: String) {
794 self.progress_count = self.progress_count.saturating_add(1);
795 self.progress.push(message);
796 if self.progress.len() > WORKFLOW_RUN_PROGRESS_MAX_RETAINED {
797 let overflow = self.progress.len() - WORKFLOW_RUN_PROGRESS_MAX_RETAINED;
798 self.progress.drain(..overflow);
799 }
800 }
801
802 /// Record one rejected task slot without allowing a malformed workflow's
803 /// rejection loop to grow the owner record without bound.
804 fn push_dispatch_failure(&mut self, failure: WorkflowDispatchFailure) {
805 self.dispatch_failure_count = self.dispatch_failure_count.saturating_add(1);
806 self.dispatch_failures.push(failure);
807 if self.dispatch_failures.len() > WORKFLOW_RUN_DISPATCH_FAILURES_MAX_RETAINED {
808 let overflow =
809 self.dispatch_failures.len() - WORKFLOW_RUN_DISPATCH_FAILURES_MAX_RETAINED;
810 self.dispatch_failures.drain(..overflow);
811 }
812 }
813
814 /// Repair legacy or malformed snapshot counters before exposing them.
815 /// A declared count may exceed the retained tail, but never trail it.
816 fn normalize_bounded_ledgers(&mut self) {
817 self.progress_count = self
818 .progress_count
819 .max(u64::try_from(self.progress.len()).unwrap_or(u64::MAX));
820 if self.progress.len() > WORKFLOW_RUN_PROGRESS_MAX_RETAINED {
821 let overflow = self.progress.len() - WORKFLOW_RUN_PROGRESS_MAX_RETAINED;
822 self.progress.drain(..overflow);
823 }
824
825 self.dispatch_failure_count = self
826 .dispatch_failure_count
827 .max(u64::try_from(self.dispatch_failures.len()).unwrap_or(u64::MAX));
828 if self.dispatch_failures.len() > WORKFLOW_RUN_DISPATCH_FAILURES_MAX_RETAINED {
829 let overflow =
830 self.dispatch_failures.len() - WORKFLOW_RUN_DISPATCH_FAILURES_MAX_RETAINED;
831 self.dispatch_failures.drain(..overflow);
832 }
833 }
834
835 fn summary(&self) -> WorkflowRunSummary {
836 WorkflowRunSummary {
837 run_id: self.run_id.clone(),
838 status: self.status,
839 lifecycle_seq: self.lifecycle_seq,
840 started_at_ms: self.started_at_ms,
841 completed_at_ms: self.completed_at_ms,
842 source_path: self.source_path.clone(),
843 workflow_id: self.workflow_id.clone(),
844 workflow_goal: self.workflow_goal.clone(),
845 token_budget: self.token_budget,
846 child_count: self.child_ids.len(),
847 schema_error_count: self.schema_errors.len(),
848 schema_repair_count: self.schema_repairs.len() as u64,
849 dispatch_failure_count: self.dispatch_failure_count,
850 progress_count: self.progress_count,
851 last_progress: self.progress.last().cloned(),
852 event_count: usize::try_from(self.events_total.max(self.events.len() as u64))
853 .unwrap_or(usize::MAX),
854 last_event_type: self
855 .events
856 .last()
857 .map(|event| event.event_type().to_string()),
858 leaf_count: self
859 .execution
860 .as_ref()
861 .map(|execution| execution.leaf_results.len())
862 .unwrap_or_default(),
863 branch_count: self
864 .execution
865 .as_ref()
866 .map(|execution| execution.branch_results.len())
867 .unwrap_or_default(),
868 control_count: self
869 .execution
870 .as_ref()
871 .map(|execution| execution.control_node_results.len())
872 .unwrap_or_default(),
873 execution_status: self.execution.as_ref().map(|execution| execution.status),
874 gate_count: self.gate_status.len(),
875 blocked_gate_count: self
876 .gate_status
877 .iter()
878 .filter(|line| line.blocked_reason.is_some())
879 .count(),
880 gate_status: self.gate_status.clone(),
881 error: self.error.clone(),
882 usage: self.usage.clone(),
883 events_dropped: self.events_dropped,
884 }
885 }
886 }
887
888 fn initial_gate_status(run_id: &str, gates: &[GateSpec]) -> Vec<GateStatusLine> {
889 if gates.is_empty() {
890 return Vec::new();
891 }
892 let mut board = LaneGateBoard::new(run_id);
893 board.install_gates(gates);
894 board.status_summary()
895 }
896
897 #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
898 #[serde(rename_all = "snake_case")]
899 enum WorkflowRunStatus {
900 Running,
901 Completed,
902 /// The script returned a value, but at least one requested task slot
903 /// failed or was rejected without the script declaring a partial-failure
904 /// contract. The output is preserved; the status refuses to call a run
905 /// with dropped slots a plain success (receipt honesty, morning-report
906 /// issue #2).
907 Degraded,
908 Failed,
909 Cancelled,
910 }
911
912 /// The per-slot outcome ledger a finished run is classified against.
913 ///
914 /// A workflow script can return a perfectly good-looking value while every
915 /// task it fanned out died — `parallel()`'s settled default resolves a failed
916 /// slot to `null`, so the script never sees a throw. Terminal status is
917 /// therefore decided from what actually ran, not from the script's return.
918 #[derive(Debug, Clone, Default, PartialEq, Eq)]
919 struct SlotLedger {
920 /// Task records the driver holds for this run (children who were admitted).
921 tasks: usize,
922 /// How many of those records ended in a failed terminal state — a
923 /// subagent failure, a budget-exhausted stop, or a replay divergence.
924 /// A budget-dead child produced no result either; counting only plain
925 /// `Failed` let an all-budget fan-out classify as a clean completion.
926 failed_tasks: usize,
927 /// Requested slots refused before a child existed — driver admission
928 /// refusals plus `ProgressEvent::TaskRejected`.
929 rejected: u64,
930 /// Fan-outs (`parallel()`/`pipeline()`) that resolved with every slot
931 /// failed and nothing surviving, script throws included. Those leave no
932 /// task record at all, so only the structured
933 /// `ProgressEvent::FanoutAllSlotsFailed` count makes them visible here.
934 dead_fanouts: u32,
935 /// Individual slots a settled fan-out dropped to null while other slots
936 /// survived (`ProgressEvent::FanoutSlotDropped`). A partial loss with
937 /// surviving work: Degraded, never a clean Completed.
938 dropped_slots: u32,
939 /// Whether any child agent was ever admitted for this run.
940 any_child_ran: bool,
941 /// First retained dispatch-failure message, for the all-rejected receipt.
942 retained_detail: Option<String>,
943 }
944
945 impl SlotLedger {
946 /// The terminal status a `Completed` script run should actually record,
947 /// with the receipt line, or `None` when every slot came back clean.
948 fn classify(&self) -> Option<(WorkflowRunStatus, String)> {
949 let dead_fanouts = if self.dead_fanouts > 0 {
950 format!(
951 " and {} fan-out(s) lost every slot (no work survived them)",
952 self.dead_fanouts
953 )
954 } else {
955 String::new()
956 };
957 if !self.any_child_ran && self.rejected > 0 {
958 // #5035: every dispatch was rejected before a child ran.
959 let retained = self
960 .retained_detail
961 .as_ref()
962 .map(|message| format!("; retained detail: {message}"))
963 .unwrap_or_default();
964 return Some((
965 WorkflowRunStatus::Failed,
966 format!(
967 "no child agents ran: all {} task dispatch(es) were rejected{retained}{dead_fanouts}",
968 self.rejected
969 ),
970 ));
971 }
972 // R9: a run whose every task failed produced nothing. Reporting that
973 // as a partial success let a workflow that lost all its work read as
974 // "completed with caveats"; it is a failure with a preserved output.
975 // `failed_tasks` counts every failure-ish terminal state, so a
976 // fan-out that died entirely of budget exhaustion lands here too.
977 if self.tasks > 0 && self.failed_tasks == self.tasks {
978 let rejected = if self.rejected > 0 {
979 format!(" and {} dispatch(es) were rejected", self.rejected)
980 } else {
981 String::new()
982 };
983 return Some((
984 WorkflowRunStatus::Failed,
985 format!(
986 "no task produced a result: all {} task(s) failed{rejected}{dead_fanouts}; \
987 the recorded result reflects no completed work",
988 self.tasks
989 ),
990 ));
991 }
992 // R9: a fan-out can die without a single task record — thunks that
993 // throw before (or instead of) calling `task()` resolve to null slots
994 // and leave only the dead-fan-out count behind. When no child
995 // anywhere survived, that run also produced nothing.
996 if self.dead_fanouts > 0 && self.tasks == 0 {
997 let rejected = if self.rejected > 0 {
998 format!(" and {} dispatch(es) were rejected", self.rejected)
999 } else {
1000 String::new()
1001 };
1002 return Some((
1003 WorkflowRunStatus::Failed,
1004 format!(
1005 "no work survived: {} fan-out(s) lost every slot{rejected}; \
1006 the recorded result reflects no completed work",
1007 self.dead_fanouts
1008 ),
1009 ));
1010 }
1011 if self.failed_tasks > 0
1012 || self.rejected > 0
1013 || self.dead_fanouts > 0
1014 || self.dropped_slots > 0
1015 {
1016 let mut parts = Vec::new();
1017 if self.failed_tasks > 0 {
1018 parts.push(format!(
1019 "{} of {} task(s) failed",
1020 self.failed_tasks, self.tasks
1021 ));
1022 }
1023 if self.rejected > 0 {
1024 parts.push(format!("{} dispatch(es) were rejected", self.rejected));
1025 }
1026 if self.dead_fanouts > 0 {
1027 parts.push(format!("{} fan-out(s) lost every slot", self.dead_fanouts));
1028 }
1029 if self.dropped_slots > 0 {
1030 parts.push(format!(
1031 "{} fan-out slot(s) dropped while others survived",
1032 self.dropped_slots
1033 ));
1034 }
1035 return Some((
1036 WorkflowRunStatus::Degraded,
1037 format!(
1038 "completed with dropped slots: {}; the recorded result may be partial",
1039 parts.join(" and ")
1040 ),
1041 ));
1042 }
1043 None
1044 }
1045 }
1046
1047 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
1048 enum WorkflowAction {
1049 Start,
1050 Run,
1051 Status,
1052 Cancel,
1053 }
1054
1055 fn parse_workflow_action(input: &Value) -> Result<WorkflowAction, ToolError> {
1056 let Some(action) = optional_str(input, "action")? else {
1057 return Ok(WorkflowAction::Start);
1058 };
1059 match action.trim().to_ascii_lowercase().as_str() {
1060 "" | "start" | "spawn" => Ok(WorkflowAction::Start),
1061 "run" | "wait" => Ok(WorkflowAction::Run),
1062 "status" | "list" | "inspect" => Ok(WorkflowAction::Status),
1063 "cancel" | "stop" | "abort" => Ok(WorkflowAction::Cancel),
1064 other => Err(ToolError::invalid_input(format!(
1065 "Invalid workflow action '{other}'. Use start, run, status, or cancel."
1066 ))),
1067 }
1068 }
1069
1070 #[async_trait]
1071 impl ToolSpec for WorkflowTool {
1072 fn name(&self) -> &'static str {
1073 "workflow"
1074 }
1075
1076 fn description(&self) -> &'static str {
1077 concat!(
1078 "Run named steps through the existing sub-agents with a structured plan, ordered phases, shared budgets and result handoffs. Fleet configures those same sub-agents and roles. ",
1079 "Inspect agent(action=\"roster\") before assigning steps; choose saved models or role/profile assignments and respect unavailable routes. ",
1080 "Prefer plan for multi-step work. Saved or advanced workflows can use script/source_path; provide exactly one input form. ",
1081 "Use action=start for detached orchestration and action=status with run_id to inspect progress. Use action=run when the model needs the final result before continuing. ",
1082 "Start a workflow on your own only for broad or staged work (the session [workflow].automatic table, default on). An explicit /workflow invocation is authorization. Do not start a workflow for one-file edits or simple questions."
1083 )
1084 }
1085
1086 fn input_schema(&self) -> Value {
1087 json!({
1088 "type": "object",
1089 "properties": {
1090 "action": {
1091 "type": "string",
1092 "enum": ["start", "run", "status", "cancel"],
1093 "description": "start (default) launches a Workflow in the background. run waits for completion. status lists runs or inspects run_id. cancel stops a run and its child agents."
1094 },
1095 "run_id": {
1096 "type": "string",
1097 "description": "Workflow run id for action=status or action=cancel."
1098 },
1099 "script": {
1100 "type": "string",
1101 "description": "Workflow JS source. The runtime provides args, task(...), parallel(thunks), pipeline(thunks), log(...), phase(...), and budget. Fan-out syntax: await parallel([() => task({...}), () => task({...})]). parallel() requires one array of zero-argument thunks, not variadic task promises."
1102 },
1103 "source_path": {
1104 "type": "string",
1105 "description": "Path to a .workflow.js script inside the workspace. Use instead of script for checked-in workflows."
1106 },
1107 "fleet": {
1108 "type": "string",
1109 "description": "Named Fleet from $CODEWHALE_HOME/fleets/ or workspace fleets/; qualified origin/name accepted. Exact Fleets freeze member identity, route, and reasoning. Runtime derives authority from role and live parent; per-task route/authority overrides are rejected."
1110 },
1111 "plan": plan_schema::structured_plan_schema(),
1112 "args": {
1113 "anyOf": [
1114 { "type": "null" },
1115 { "type": "boolean" },
1116 { "type": "integer" },
1117 { "type": "number" },
1118 { "type": "string" },
1119 { "type": "array" },
1120 {
1121 "type": "object",
1122 "additionalProperties": {}
1123 }
1124 ],
1125 "description": "JSON value exposed to the script as args. Defaults to null."
1126 },
1127 "token_budget": {
1128 "type": "integer",
1129 "minimum": 1,
1130 "description": "Optional shared Workflow admission cap; omit for no cap. Usage is reconciled when children report completion; already-running parallel children can take aggregate spent past the cap, while later and descendant spawns are rejected once exhausted."
1131 },
1132 "wait": {
1133 "type": "boolean",
1134 "description": "For action=start, wait for completion instead of returning immediately."
1135 },
1136 "verify": {
1137 "type": "boolean",
1138 "default": false,
1139 "description": "After a successful workflow completion, run quick workspace verifier gates (auto/quick profile)."
1140 }
1141 },
1142 "required": [],
1143 "additionalProperties": false
1144 })
1145 }
1146
1147 fn capabilities(&self) -> Vec<ToolCapability> {
1148 vec![
1149 ToolCapability::ExecutesCode,
1150 ToolCapability::RequiresApproval,
1151 ]
1152 }
1153
1154 fn approval_requirement(&self) -> ApprovalRequirement {
1155 // Default posture: elevated starts require approval. Concrete inputs
1156 // refine this via `approval_requirement_for` (#4126).
1157 ApprovalRequirement::Required
1158 }
1159
1160 fn approval_requirement_for(&self, input: &Value) -> ApprovalRequirement {
1161 // The session's `[workflow]` table decides read-only auto-start and
1162 // write approval; product defaults apply only when the runtime never
1163 // threaded a config. YOLO/bypass still short-circuit upstream.
1164 let config = workflow_config_for(&self.runtime);
1165 workflow_approval_requirement_for(input, &config)
1166 }
1167
1168 fn starts_detached_for(&self, input: &Value) -> bool {
1169 // A scheduling hint, not an authority decision, and this trait method
1170 // cannot report an error. A malformed `wait` reads as "not detached"
1171 // so the call stays in the foreground; `execute` then refuses it with
1172 // the named-parameter error rather than running anything.
1173 matches!(parse_workflow_action(input), Ok(WorkflowAction::Start))
1174 && !optional_bool(input, "wait", false).unwrap_or(true)
1175 }
1176
1177 fn supports_parallel_for(&self, input: &Value) -> bool {
1178 matches!(parse_workflow_action(input), Ok(WorkflowAction::Status))
1179 }
1180
1181 fn is_read_only_for(&self, input: &Value) -> bool {
1182 matches!(parse_workflow_action(input), Ok(WorkflowAction::Status))
1183 }
1184
1185 async fn execute(&self, input: Value, context: &ToolContext) -> Result<ToolResult, ToolError> {
1186 let state = shared_workflow_state(&context.workspace);
1187 attach_bound_workflow_lifecycles(context, &state)?;
1188 // Keyed off the parsed `WorkflowAction` discriminant, never off
1189 // `input["action"]`. The JSON Schema published to the model is a
1190 // declaration, not a guard: the real parse also accepts `spawn`,
1191 // `wait`, `list`, `inspect`, `stop`, and `abort`, and its reject arm
1192 // embeds the model's string verbatim.
1193 let action = parse_workflow_action(&input)?;
1194 codewhale_telemetry::session_counters().bump(codewhale_telemetry::Counter::WorkflowRun);
1195 match action {
1196 WorkflowAction::Start => {
1197 let wait = optional_bool(&input, "wait", false)?;
1198 start_workflow(
1199 input,
1200 context,
1201 self.manager.clone(),
1202 self.runtime.clone(),
1203 state,
1204 wait,
1205 self.approval_decision,
1206 )
1207 .await
1208 }
1209 WorkflowAction::Run => {
1210 start_workflow(
1211 input,
1212 context,
1213 self.manager.clone(),
1214 self.runtime.clone(),
1215 state,
1216 true,
1217 self.approval_decision,
1218 )
1219 .await
1220 }
1221 WorkflowAction::Status => status_workflow(input, state, &context.state_namespace),
1222 WorkflowAction::Cancel => cancel_workflow(input, state, &context.state_namespace).await,
1223 }
1224 }
1225 }
1226
1227 fn attach_bound_workflow_lifecycles(
1228 context: &ToolContext,
1229 state: &Arc<WorkflowWorkspaceState>,
1230 ) -> Result<(), ToolError> {
1231 let records = lock_mutex(&state.runs)?
1232 .values()
1233 .filter(|record| {
1234 record.owner_session_id.as_deref() == Some(context.state_namespace.as_str())
1235 })
1236 .cloned()
1237 .collect::<Vec<_>>();
1238 for record in records {
1239 if let Some(lifecycle) = WorkflowWorkLifecycle::for_bound(context, &record.run_id) {
1240 state.attach_lifecycle(&record.run_id, lifecycle);
1241 state.reconcile_snapshot(&record);
1242 }
1243 }
1244 Ok(())
1245 }
1246
1247 fn fail_workflow_start(state: &Arc<WorkflowWorkspaceState>, run_id: &str, message: String) {
1248 let snapshot = state.runs.lock().ok().and_then(|mut runs| {
1249 let record = runs.get_mut(run_id)?;
1250 record.status = WorkflowRunStatus::Failed;
1251 record.lifecycle_seq = record.lifecycle_seq.saturating_add(1);
1252 record.completed_at_ms = Some(now_ms());
1253 record.error = Some(message);
1254 Some(record.clone())
1255 });
1256 let Some(snapshot) = snapshot else {
1257 state.mark_owner_missing(run_id);
1258 return;
1259 };
1260 if state.try_record_snapshot(&snapshot).is_ok() {
1261 state.reconcile_snapshot(&snapshot);
1262 } else {
1263 state.mark_owner_missing(run_id);
1264 }
1265 }
1266
1267 fn fail_workflow_after_controller_registration(
1268 state: &Arc<WorkflowWorkspaceState>,
1269 run_id: &str,
1270 controller: &Arc<WorkflowRunController>,
1271 message: String,
1272 ) {
1273 controller.cancel();
1274 if let Ok(mut controllers) = state.controllers.lock() {
1275 controllers.remove(run_id);
1276 }
1277 fail_workflow_start(state, run_id, message);
1278 }
1279
1280 #[allow(clippy::too_many_arguments)]
1281 async fn start_workflow(
1282 input: Value,
1283 context: &ToolContext,
1284 manager: SharedSubAgentManager,
1285 runtime: SubAgentRuntime,
1286 state: Arc<WorkflowWorkspaceState>,
1287 wait: bool,
1288 approval_decision: &str,
1289 ) -> Result<ToolResult, ToolError> {
1290 if runtime.cancel_token.is_cancelled() {
1291 return Err(ToolError::execution_failed(
1292 "Workflow parent session is cancelled",
1293 ));
1294 }
1295 let mut source = workflow_source(&input, context)?;
1296 let workflow_cfg = workflow_config_for(&runtime);
1297 for (name, value) in [
1298 ("max_children", workflow_cfg.max_children),
1299 ("max_depth", workflow_cfg.max_depth),
1300 ("max_concurrent", workflow_cfg.max_concurrent),
1301 ] {
1302 if value == 0 {
1303 return Err(ToolError::invalid_input(format!(
1304 "workflow.{name} must be greater than zero"
1305 )));
1306 }
1307 }
1308 if let Some(spec) = source.spec.as_mut() {
1309 validate_runtime_gates(&mut spec.gates)?;
1310 spec.validate_for_fleet_with_limits(WorkflowFleetLimits {
1311 max_total_agents: (workflow_cfg.max_children as usize)
1312 .min(codewhale_workflow::DEFAULT_FLEET_WORKFLOW_MAX_AGENTS),
1313 max_depth: (workflow_cfg.max_depth as usize)
1314 .min(codewhale_workflow::DEFAULT_FLEET_WORKFLOW_MAX_DEPTH),
1315 })
1316 .map_err(|err| ToolError::invalid_input(format!(
1317 "Workflow plan exceeds workflow.max_children or workflow.max_depth, or is invalid: {err}"
1318 )))?;
1319 }
1320 let args = input.get("args").cloned().unwrap_or(Value::Null);
1321 let token_budget = optional_u64(&input, "token_budget", 0)?;
1322 let token_budget = [
1323 (token_budget > 0).then_some(token_budget),
1324 source.spec.as_ref().and_then(|spec| spec.budget.max_tokens),
1325 ]
1326 .into_iter()
1327 .flatten()
1328 .min()
1329 .or((workflow_cfg.default_token_budget > 0).then_some(workflow_cfg.default_token_budget));
1330 let verify_on_complete = optional_bool(&input, "verify", false)?;
1331 let fleet = workflow_fleet_binding(&input, context, runtime.api_config.as_deref())?;
1332 let run_id = format!("workflow_{}", &Uuid::new_v4().to_string()[..8]);
1333 let gate_specs = source
1334 .spec
1335 .as_ref()
1336 .map(|spec| spec.gates.clone())
1337 .unwrap_or_default();
1338
1339 // Capture the approved plan envelope for audit/receipt (#4126). Reaching
1340 // execute means the approval gate already passed (or YOLO/auto-start).
1341 let summary = source
1342 .spec
1343 .as_ref()
1344 .map(|spec| analyze_workflow_spec(spec, token_budget, &workflow_cfg))
1345 .unwrap_or_else(|| analyze_workflow_plan_approval_with_config(&input, &workflow_cfg));
1346 let approval_decision = if summary.is_read_only_envelope() {
1347 "auto_read_only"
1348 } else {
1349 approval_decision
1350 };
1351 let plan_approval = summary.to_receipt(approval_decision, now_ms());
1352 let workflow_title = source
1353 .spec
1354 .as_ref()
1355 .map(|spec| spec.goal.as_str())
1356 .or_else(|| {
1357 source
1358 .path
1359 .as_ref()
1360 .and_then(|path| path.file_name()?.to_str())
1361 })
1362 .unwrap_or("Workflow run");
1363 let lifecycle = WorkflowWorkLifecycle::register(context, &run_id, workflow_title)?;
1364
1365 {
1366 let mut runs_guard = match lock_mutex(&state.runs) {
1367 Ok(guard) => guard,
1368 Err(err) => {
1369 if let Some(lifecycle) = lifecycle.as_ref() {
1370 lifecycle.reconcile_spawn_failure();
1371 }
1372 return Err(err);
1373 }
1374 };
1375 let mut record = WorkflowRunRecord::new(
1376 run_id.clone(),
1377 Some(context.state_namespace.clone()),
1378 source.path.clone(),
1379 token_budget,
1380 source.spec.as_ref(),
1381 );
1382 record.verify_on_complete = verify_on_complete;
1383 record.plan_approval = Some(plan_approval.clone());
1384 let started = WorkflowUiEvent::at(
1385 record.started_at_ms,
1386 &context.state_namespace,
1387 WorkflowUiEventKind::RunStarted {
1388 workflow_id: record.workflow_id.clone(),
1389 workflow_goal: record.workflow_goal.clone(),
1390 source_path: record.source_path.clone(),
1391 token_budget: record.token_budget,
1392 },
1393 );
1394 record.push_event(started.clone());
1395 runs_guard.insert(run_id.clone(), record.clone());
1396 if let Err(err) = state.try_record_snapshot(&record) {
1397 runs_guard.remove(&run_id);
1398 if let Some(lifecycle) = lifecycle.as_ref() {
1399 lifecycle.reconcile_spawn_failure();
1400 }
1401 return Err(ToolError::execution_failed(format!(
1402 "workflow journal snapshot failed before launch: {err}"
1403 )));
1404 }
1405 // #4122: emit RunStarted immediately so the panel + history card open
1406 // before the first task/phase (including wait:false fire-and-forget).
1407 if let Some(tx) = runtime.event_tx.as_ref()
1408 && let Ok(mut value) = serde_json::to_value(&started)
1409 {
1410 if let Some(obj) = value.as_object_mut() {
1411 obj.insert("run_id".to_string(), json!(run_id));
1412 }
1413 let _ = tx.try_send(Event::WorkflowUi {
1414 owner_session_id: context.state_namespace.clone(),
1415 run_id: run_id.clone(),
1416 event: value,
1417 });
1418 }
1419 }
1420 if let Some(lifecycle) = lifecycle {
1421 state.attach_lifecycle(&run_id, lifecycle);
1422 }
1423
1424 // Children share the spawn runtime; the driver keeps the exact Fleet
1425 // binding frozen for this run, including native model selectors.
1426 let driver = SubAgentWorkflowDriver::new(
1427 run_id.clone(),
1428 context.state_namespace.clone(),
1429 manager,
1430 runtime,
1431 state.clone(),
1432 token_budget,
1433 fleet,
1434 gate_specs,
1435 context.workspace.clone(),
1436 );
1437 let vm_cancel = WorkflowRunCancel::new();
1438 let controller = Arc::new(
1439 WorkflowRunController::new(driver.clone(), vm_cancel.clone()).with_parent_completion(!wait),
1440 );
1441 if let Err(err) = lock_mutex(&state.controllers).map(|mut controllers_guard| {
1442 controllers_guard.insert(run_id.clone(), controller.clone());
1443 }) {
1444 fail_workflow_start(&state, &run_id, err.to_string());
1445 return Err(err);
1446 }
1447 let running_snapshot = {
1448 let mut runs_guard = match lock_mutex(&state.runs) {
1449 Ok(guard) => guard,
1450 Err(err) => {
1451 fail_workflow_after_controller_registration(
1452 &state,
1453 &run_id,
1454 &controller,
1455 err.to_string(),
1456 );
1457 return Err(err);
1458 }
1459 };
1460 let Some(record) = runs_guard.get_mut(&run_id) else {
1461 drop(runs_guard);
1462 fail_workflow_after_controller_registration(
1463 &state,
1464 &run_id,
1465 &controller,
1466 "workflow owner record disappeared before launch".to_string(),
1467 );
1468 return Err(ToolError::execution_failed(
1469 "workflow owner record disappeared before launch",
1470 ));
1471 };
1472 record.lifecycle_seq = record.lifecycle_seq.saturating_add(1);
1473 record.clone()
1474 };
1475 if let Err(err) = state.try_record_snapshot(&running_snapshot) {
1476 fail_workflow_after_controller_registration(
1477 &state,
1478 &run_id,
1479 &controller,
1480 format!("workflow journal failed while activating owner: {err}"),
1481 );
1482 return Err(ToolError::execution_failed(format!(
1483 "workflow journal failed while activating owner: {err}"
1484 )));
1485 }
1486 state.reconcile_snapshot(&running_snapshot);
1487
1488 let run = run_workflow_vm(
1489 run_id.clone(),
1490 source.source,
1491 source.spec,
1492 args,
1493 driver,
1494 state.clone(),
1495 context.clone(),
1496 vm_cancel,
1497 );
1498 if wait {
1499 run.await;
1500 } else {
1501 let handle = spawn_supervised("workflow-run", std::panic::Location::caller(), run);
1502 controller.set_run_handle(handle);
1503 }
1504
1505 workflow_result_for(&run_id, state, &context.state_namespace)
1506 }
1507
1508 fn status_workflow(
1509 input: Value,
1510 state: Arc<WorkflowWorkspaceState>,
1511 owner_session_id: &str,
1512 ) -> Result<ToolResult, ToolError> {
1513 if let Some(run_id) = optional_str(&input, "run_id")? {
1514 return workflow_result_for(run_id, state, owner_session_id);
1515 }
1516 let mut summaries = {
1517 let runs_guard = lock_mutex(&state.runs)?;
1518 runs_guard
1519 .values()
1520 .filter(|record| record.owner_session_id.as_deref() == Some(owner_session_id))
1521 .map(WorkflowRunRecord::summary)
1522 .collect::<Vec<_>>()
1523 };
1524 summaries.sort_by_key(|record| record.started_at_ms);
1525 ToolResult::json(&json!({
1526 "action": "status",
1527 "count": summaries.len(),
1528 "runs": summaries,
1529 }))
1530 .map_err(|err| ToolError::execution_failed(err.to_string()))
1531 }
1532
1533 async fn cancel_workflow(
1534 input: Value,
1535 state: Arc<WorkflowWorkspaceState>,
1536 owner_session_id: &str,
1537 ) -> Result<ToolResult, ToolError> {
1538 let run_id =
1539 optional_str(&input, "run_id")?.ok_or_else(|| ToolError::missing_field("run_id"))?;
1540 cancel_workflow_run(run_id, state, owner_session_id)
1541 }
1542
1543 /// Synchronous cancellation core shared by the model-facing tool action and
1544 /// the host `/workflow cancel` command. When a live controller exists this
1545 /// signals the VM, aborts the run task, journals the terminal snapshot, and
1546 /// streams the cancelled event. When the journal has a running line but no
1547 /// controller (typical after a restart), the journal is still marked
1548 /// cancelled with an honest nothing-live receipt. Nothing here waits on the
1549 /// network.
1550 fn cancel_workflow_run(
1551 run_id: &str,
1552 state: Arc<WorkflowWorkspaceState>,
1553 owner_session_id: &str,
1554 ) -> Result<ToolResult, ToolError> {
1555 // Resolve ownership before touching the controller map or disclosing
1556 // status. Foreign and legacy-ownerless ids are indistinguishable from an
1557 // unknown run.
1558 {
1559 let runs_guard = lock_mutex(&state.runs)?;
1560 let record = runs_guard
1561 .get(run_id)
1562 .filter(|record| record.owner_session_id.as_deref() == Some(owner_session_id));
1563 record.ok_or_else(|| {
1564 ToolError::invalid_input(format!("Unknown workflow run_id '{run_id}'"))
1565 })?;
1566 };
1567 // Terminal publication and a cancel claim share this lock. If cancellation
1568 // claims a live controller first, no success receipt can pass that claim.
1569 let controllers_guard = lock_mutex(&state.controllers)?;
1570 let controller = controllers_guard.get(run_id).cloned();
1571 let current_status = lock_mutex(&state.runs)?
1572 .get(run_id)
1573 .map(|record| record.status)
1574 .ok_or_else(|| ToolError::invalid_input(format!("Unknown workflow run_id '{run_id}'")))?;
1575 if current_status != WorkflowRunStatus::Running && controller.is_none() {
1576 drop(controllers_guard);
1577 state.reconcile_cancel(run_id, CancelOutcome::AlreadyFinished);
1578 if let Ok(runs_guard) = state.runs.lock()
1579 && let Some(record) = runs_guard.get(run_id)
1580 {
1581 state.reconcile_snapshot(record);
1582 }
1583 return workflow_result_for(run_id, state, owner_session_id);
1584 }
1585 let live = controller.is_some();
1586 let reason = if live {
1587 "cancelled by workflow tool"
1588 } else {
1589 "cancelled; no live process to stop"
1590 }
1591 .to_string();
1592 let cancelled_event = WorkflowUiEvent::new(
1593 owner_session_id,
1594 WorkflowUiEventKind::RunCancelled {
1595 reason: reason.clone(),
1596 },
1597 );
1598 let snapshot = {
1599 let mut runs_guard = lock_mutex(&state.runs)?;
1600 let record = runs_guard.get_mut(run_id).ok_or_else(|| {
1601 ToolError::invalid_input(format!("Unknown workflow run_id '{run_id}'"))
1602 })?;
1603 record.status = WorkflowRunStatus::Cancelled;
1604 record.lifecycle_seq = record.lifecycle_seq.saturating_add(1);
1605 record.completed_at_ms = Some(now_ms());
1606 record.error = Some(reason);
1607 record.push_event(cancelled_event.clone());
1608 record.clone()
1609 };
1610 drop(controllers_guard);
1611 state.reconcile_cancel(
1612 run_id,
1613 if live {
1614 CancelOutcome::Requested
1615 } else {
1616 CancelOutcome::Acknowledged
1617 },
1618 );
1619 if let Some(controller) = controller.as_ref() {
1620 controller.cancel();
1621 }
1622 if let Err(err) = state.try_record_snapshot(&snapshot) {
1623 state.mark_owner_missing(run_id);
1624 let mut failed = snapshot;
1625 failed.error = Some(format!("workflow cancellation journal failed: {err}"));
1626 finish_workflow_controller(&state, &failed);
1627 return Err(ToolError::execution_failed(format!(
1628 "workflow cancellation journal failed: {err}"
1629 )));
1630 }
1631 state.reconcile_snapshot(&snapshot);
1632 // The VM may publish its terminal `run_completed` event while cancellation
1633 // is racing it. Always stream the authoritative cancellation afterward so
1634 // the live panel finalizes running rows and cannot remain visually failed.
1635 if let Some(controller) = controller {
1636 controller.driver.emit_ui_event(&cancelled_event);
1637 }
1638 finish_workflow_controller(&state, &snapshot);
1639 workflow_result_for(run_id, state, owner_session_id)
1640 }
1641
1642 fn workflow_fleet_name(input: &Value) -> Result<Option<String>, ToolError> {
1643 let named = match optional_str(input, "fleet")? {
1644 Some(name) => Some(name),
1645 None => match input.get("args") {
1646 Some(args) => optional_str(args, "fleet")?,
1647 None => None,
1648 },
1649 };
1650 Ok(named
1651 .map(str::trim)
1652 .filter(|name| !name.is_empty())
1653 .map(str::to_string))
1654 }
1655
1656 /// How a Workflow run is bound to a named Fleet.
1657 ///
1658 /// The two saved forms share one store and one `fleet: "<name>"` option. Legacy
1659 /// role maps keep their exact previous behavior; an exact fleet is frozen into
1660 /// an immutable Workflow snapshot at start and drives every task launch from
1661 /// that snapshot.
1662 #[derive(Debug, Clone, Default)]
1663 enum WorkflowFleetBinding {
1664 #[default]
1665 None,
1666 Legacy {
1667 name: String,
1668 roles: FleetRoleMap,
1669 },
1670 Exact(Arc<crate::fleet::exact::ExactFleetWorkflow>),
1671 }
1672
1673 impl WorkflowFleetBinding {
1674 fn name(&self) -> Option<String> {
1675 match self {
1676 Self::None => None,
1677 Self::Legacy { name, .. } => Some(name.clone()),
1678 Self::Exact(operation) => Some(operation.snapshot().fleet().qualified()),
1679 }
1680 }
1681
1682 fn legacy_roles(&self) -> Option<&FleetRoleMap> {
1683 match self {
1684 Self::Legacy { roles, .. } => Some(roles),
1685 Self::None | Self::Exact(_) => None,
1686 }
1687 }
1688
1689 fn exact(&self) -> Option<&Arc<crate::fleet::exact::ExactFleetWorkflow>> {
1690 match self {
1691 Self::Exact(operation) => Some(operation),
1692 Self::None | Self::Legacy { .. } => None,
1693 }
1694 }
1695 }
1696
1697 fn workflow_fleet_binding(
1698 input: &Value,
1699 context: &ToolContext,
1700 api_config: Option<&crate::config::Config>,
1701 ) -> Result<WorkflowFleetBinding, ToolError> {
1702 let Some(name) = workflow_fleet_name(input)? else {
1703 return Ok(WorkflowFleetBinding::None);
1704 };
1705 let roots = crate::fleet::exact::fleet_search_roots(&context.workspace);
1706 let (document, id) = crate::fleet::exact::load_fleet_document(&name, &context.workspace)
1707 .map_err(|err| {
1708 ToolError::invalid_input(format!(
1709 "Failed to load workflow Fleet '{name}' from {}: {err}",
1710 roots
1711 .iter()
1712 .map(|root| format!("{}/{}", root.origin, root.root.display()))
1713 .collect::<Vec<_>>()
1714 .join(", ")
1715 ))
1716 })?;
1717
1718 if let Some(legacy) = document.legacy() {
1719 let roles = FleetRoleMap::from_pairs(
1720 legacy
1721 .roles
1722 .iter()
1723 .map(|(role, profile)| (role.as_str(), profile.as_str())),
1724 )
1725 .map_err(|err| ToolError::invalid_input(err.to_string()))?;
1726 return Ok(WorkflowFleetBinding::Legacy { name, roles });
1727 }
1728
1729 // Exact: freeze the definition now. Everything the run launches afterwards
1730 // comes from this value, so editing the file mid-run cannot move a route.
1731 // The same labelled roots resolve the Fleet *and* the Reasoning Router
1732 // profile it references, so a Router is qualified (`workspace/luna-low`)
1733 // exactly the way a Fleet is and cannot be resolved by shadowing.
1734 let operation = crate::fleet::exact::ExactFleetWorkflow::capture(
1735 &document,
1736 id,
1737 chrono::Utc::now().to_rfc3339(),
1738 api_config,
1739 &roots,
1740 )
1741 .map_err(ToolError::invalid_input)?;
1742 Ok(WorkflowFleetBinding::Exact(Arc::new(operation)))
1743 }
1744
1745 fn apply_named_fleet_to_task_request(
1746 fleet_roles: Option<&FleetRoleMap>,
1747 request: &mut TaskRequest,
1748 ) -> Result<(), DriverError> {
1749 let Some(fleet_roles) = fleet_roles else {
1750 return Ok(());
1751 };
1752 let resolved = resolve_workflow_agent(
1753 request.role.as_deref(),
1754 request.profile.as_deref(),
1755 fleet_roles,
1756 true,
1757 )
1758 .map_err(|err| DriverError::Rejected(err.to_string()))?;
1759 request.role = resolved.resolved_role.as_deref().map(public_role_label);
1760 request.profile = Some(resolved.resolved_profile);
1761 Ok(())
1762 }
1763
1764 /// **Phase one** of an exact-Fleet task: resolve the member and stamp its
1765 /// Runtime-derived authority onto the request, contacting nobody.
1766 ///
1767 /// This runs *before* gate evaluation and before a concurrency slot is taken,
1768 /// which is what makes it safe: a task that is about to be rejected or queued
1769 /// must not have spent a router call or disclosed a summary to another
1770 /// provider. Everything that can cost money lives in
1771 /// [`route_admitted_exact_task`].
1772 fn bind_exact_fleet_task_request(
1773 operation: &crate::fleet::exact::ExactFleetWorkflow,
1774 session: codewhale_workflow::PermissionCeiling,
1775 request: &mut TaskRequest,
1776 ) -> Result<crate::fleet::exact::ExactMemberBinding, DriverError> {
1777 let fleet = operation.snapshot().fleet().qualified();
1778
1779 // The exact Fleet owns member identity, route, and reasoning. Runtime owns
1780 // authority: after selection it derives the closed role posture and
1781 // intersects it with the live parent. Tasks may narrow that authority, as
1782 // native read-only plans do, but cannot replace identity or widen posture.
1783 for (field, present) in [
1784 ("model", request.model.is_some()),
1785 ("model_strength", request.model_strength.is_some()),
1786 ("thinking", request.thinking.is_some()),
1787 ("subagent_type", request.subagent_type.is_some()),
1788 ] {
1789 if present {
1790 return Err(DriverError::Rejected(format!(
1791 "Fleet `{fleet}` is an exact Fleet: task option `{field}` is not allowed. Every \
1792 member's identity, provider, model, and reasoning are fixed by the saved Fleet, \
1793 while Runtime derives authority from its role and the live parent. Switch \
1794 Fleets/edit the Fleet for identity or route changes; do not override either \
1795 contract per task."
1796 )));
1797 }
1798 }
1799
1800 let mut binding = operation
1801 .bind_member(request.profile.as_deref(), request.role.as_deref(), session)
1802 .map_err(|err| DriverError::Rejected(format!("Fleet `{fleet}`: {err}")))?;
1803 if request.write_authority.as_deref() == Some("worktree_write") && !request.worktree {
1804 return Err(DriverError::Rejected(format!(
1805 "Fleet `{fleet}`: worktree_write requires worktree isolation"
1806 )));
1807 }
1808 binding
1809 .narrow_for_task(
1810 request.write_authority.as_deref(),
1811 request.allowed_tools.as_deref(),
1812 &request.disallowed_tools,
1813 request.max_depth,
1814 )
1815 .map_err(|err| DriverError::Rejected(format!("Fleet `{fleet}`: {err}")))?;
1816
1817 // Id and role are stamped **separately and semantically**. The member id
1818 // addresses the run-scoped roster profile projected from the snapshot,
1819 // which carries the exact provider pin and canonical wire model. The role
1820 // stays the Fleet's semantic role, because that is what gates, handoffs,
1821 // and records key on — overwriting it with the profile id (as an earlier
1822 // pass did) silently broke every gate whose member id differs from its
1823 // role.
1824 request.profile = Some(binding.member_id.clone());
1825 request.role = Some(binding.member_role.clone());
1826
1827 // `subagent_type` is cleared rather than defaulted so the selected roster
1828 // profile's Runtime role is what picks the child posture.
1829 request.subagent_type = None;
1830 // The Runtime/parent intersection becomes an actual tool policy the child
1831 // enforces, not a Fleet identity label.
1832 request.allowed_tools = binding.authority.allowed_tools.clone();
1833 request.disallowed_tools = binding.authority.disallowed_tools.clone();
1834 request.write_authority = Some(binding.authority.write_authority.to_string());
1835 request.max_depth = Some(
1836 request
1837 .max_depth
1838 .map_or(binding.authority.max_depth, |asked| {
1839 asked.min(binding.authority.max_depth)
1840 }),
1841 );
1842
1843 // Everything the spawn boundary will reject *predictably* is rejected here,
1844 // while the task has still cost nothing. The write-scope contract is the
1845 // one that bites: a write-capable member launched with no declared scope
1846 // fails at `validate_spawn_write_contract`, which runs long after the
1847 // Router has been paid for a decision about a task that could never run.
1848 validate_exact_write_scope(&fleet, &binding, request)?;
1849 Ok(binding)
1850 }
1851
1852 /// Which role a `task_started` event displays.
1853 ///
1854 /// An exact-Fleet receipt wins over the spawn metadata, because the metadata's
1855 /// role is the roster profile's **Runtime posture** — the closed role policy
1856 /// selected after identity resolution — and rendering that where the member's role belongs
1857 /// renames the operator's `auditor` to `scout` in the panel, the history card,
1858 /// and the journal. The posture is not lost: it rides the same receipt in its
1859 /// own field. Non-Fleet tasks keep the previous metadata-then-request order
1860 /// exactly.
1861 fn displayed_resolved_role(
1862 fleet_receipt: Option<&codewhale_workflow::FleetTaskReceipt>,
1863 metadata_role: Option<&str>,
1864 request_role: Option<&str>,
1865 ) -> Option<String> {
1866 fleet_receipt
1867 .map(|receipt| receipt.member_role.clone())
1868 .or_else(|| metadata_role.map(str::to_string))
1869 .or_else(|| request_role.map(str::to_string))
1870 }
1871
1872 /// The visible line for a routing decision whose spawn then failed.
1873 ///
1874 /// Kept separate from the recorder so the wording is testable without a live
1875 /// driver, and so the receipt's own content-free `line()` stays the single
1876 /// source of what a receipt may say.
1877 fn orphaned_fleet_receipt_line(
1878 receipt: &codewhale_workflow::FleetTaskReceipt,
1879 error: &str,
1880 ) -> String {
1881 format!(
1882 "Fleet route {} spawn_failed=true reason={}",
1883 receipt.line(),
1884 error.replace('\n', " ")
1885 )
1886 }
1887
1888 /// The write-scope half of the spawn contract, checked before anything costs.
1889 ///
1890 /// Deliberately a mirror of the spawn-boundary rule rather than a replacement
1891 /// for it: the boundary stays authoritative (it is reachable by other callers),
1892 /// and this exists so an exact-Fleet task fails on the same terms *before* the
1893 /// Router call rather than after it.
1894 fn validate_exact_write_scope(
1895 fleet: &str,
1896 binding: &crate::fleet::exact::ExactMemberBinding,
1897 request: &TaskRequest,
1898 ) -> Result<(), DriverError> {
1899 let declares_scope = !request.write_roots.is_empty()
1900 || !request.exact_files.is_empty()
1901 || !request.coordination_contracts.is_empty();
1902
1903 if binding.authority.write_authority == "read_only" {
1904 if declares_scope {
1905 return Err(DriverError::Rejected(format!(
1906 "fleet `{fleet}`: member `{}` is read-only under the effective Runtime posture, so this \
1907 task may not declare write_roots, exact_files, or coordination_contracts.",
1908 binding.member_id
1909 )));
1910 }
1911 return Ok(());
1912 }
1913
1914 if !declares_scope {
1915 return Err(DriverError::Rejected(format!(
1916 "fleet `{fleet}`: member `{}` is write-capable, so this task must declare \
1917 write_roots, exact_files, or coordination_contracts before it can start. An \
1918 unbounded write claim is refused at the spawn boundary, and this task would spend a \
1919 reasoning-router call on its way to that refusal.",
1920 binding.member_id
1921 )));
1922 }
1923 Ok(())
1924 }
1925
1926 /// **Phase two**: route an already admitted task.
1927 ///
1928 /// Only reachable once the task has passed its gates and holds a concurrency
1929 /// slot, so this is the one place a reasoning router call — and any
1930 /// cross-provider disclosure — can happen.
1931 async fn route_admitted_exact_task(
1932 operation: &crate::fleet::exact::ExactFleetWorkflow,
1933 binding: &crate::fleet::exact::ExactMemberBinding,
1934 request: &mut TaskRequest,
1935 ) -> Result<codewhale_workflow::FleetTaskReceipt, DriverError> {
1936 let fleet = operation.snapshot().fleet().qualified();
1937 let launch = operation
1938 .route_admitted_task(binding, &request.description)
1939 .await
1940 .map_err(|err| DriverError::Rejected(format!("Fleet `{fleet}`: {err}")))?;
1941
1942 request.thinking = Some(launch.thinking.clone());
1943 // **The launch authority is what the child runs under.** Binding stamped a
1944 // provisional copy so the write-scope contract could be checked for free;
1945 // this re-stamps from the value `route_admitted_task` recomputed and
1946 // verified, so the request that reaches the spawn boundary carries the
1947 // launched envelope and not an older one. Without this the launch's
1948 // `authority` was computed, put on a struct, and never read — a ceiling
1949 // that existed only as a field.
1950 apply_launch_authority(&fleet, &launch, request)?;
1951 Ok(launch.receipt)
1952 }
1953
1954 /// Stamp the launched authority onto the request and refuse any drift.
1955 ///
1956 /// Two things happen here and both are load-bearing. The envelope fields are
1957 /// overwritten from `launch.authority`, so the spawn input is built from the
1958 /// launched value rather than the admitted one. And `max_depth` is intersected
1959 /// rather than replaced, because a task may legitimately ask for *less* nesting
1960 /// than its ceiling allows — but never more.
1961 fn apply_launch_authority(
1962 fleet: &str,
1963 launch: &crate::fleet::exact::ExactMemberLaunch,
1964 request: &mut TaskRequest,
1965 ) -> Result<(), DriverError> {
1966 let authority = &launch.authority;
1967
1968 // Identity first: an envelope stamped onto the wrong member's request is a
1969 // widening as surely as a wider envelope would be.
1970 if request.profile.as_deref() != Some(launch.member_id.as_str())
1971 || request.role.as_deref() != Some(launch.member_role.as_str())
1972 {
1973 return Err(DriverError::Rejected(format!(
1974 "fleet `{fleet}`: task identity drifted between admission and launch (request \
1975 profile={:?} role={:?}, launch member `{}` role `{}`); the launch is refused rather \
1976 than run under an envelope resolved for a different member.",
1977 request.profile, request.role, launch.member_id, launch.member_role,
1978 )));
1979 }
1980
1981 request.allowed_tools = authority.allowed_tools.clone();
1982 request.disallowed_tools = authority.disallowed_tools.clone();
1983 request.write_authority = Some(authority.write_authority.to_string());
1984 request.subagent_type = None;
1985 request.max_depth = Some(
1986 request
1987 .max_depth
1988 .map_or(authority.max_depth, |asked| asked.min(authority.max_depth)),
1989 );
1990
1991 // The receipt records the fingerprint; the request now carries the envelope
1992 // it names. Recomputing the fingerprint from what was just stamped is the
1993 // check that the two describe each other — a mismatch here means a field
1994 // was added to the envelope and not to the stamping, which is exactly the
1995 // silent-gap failure this whole seam exists to prevent.
1996 let expected = authority.fingerprint();
1997 if launch.receipt.authority_fingerprint.as_deref() != Some(expected.as_str()) {
1998 return Err(DriverError::Rejected(format!(
1999 "fleet `{fleet}`: member `{}` produced a receipt whose authority fingerprint does not \
2000 match the envelope being installed (receipt={:?} envelope={expected}). Failing \
2001 closed.",
2002 launch.member_id, launch.receipt.authority_fingerprint,
2003 )));
2004 }
2005 Ok(())
2006 }
2007
2008 // Pre-existing spawn signature that grew `vm_cancel` for the cancel-interrupt
2009 // wiring; the args mirror one workflow run's context and are consumed once.
2010 #[allow(clippy::too_many_arguments)]
2011 async fn run_workflow_vm(
2012 run_id: String,
2013 source: String,
2014 spec: Option<WorkflowSpec>,
2015 args: Value,
2016 driver: Arc<SubAgentWorkflowDriver>,
2017 state: Arc<WorkflowWorkspaceState>,
2018 context: ToolContext,
2019 vm_cancel: WorkflowRunCancel,
2020 ) {
2021 let vm = WorkflowVm::new();
2022 let run = vm.run_script_with_cancel(&source, args, driver.clone(), vm_cancel.clone());
2023 tokio::pin!(run);
2024 // The VM closes its local driver during normal teardown too. Observe the
2025 // original caller token here, so that teardown cannot masquerade as a stop.
2026 let result = tokio::select! {
2027 biased;
2028 () = driver.parent_cancel_token.cancelled() => {
2029 vm_cancel.cancel();
2030 driver.force_cancel_all();
2031 // Dropping the run future signals its existing guard too. Do not
2032 // wait behind another VM's admission permit after an explicit stop.
2033 Err(codewhale_workflow_js::WorkflowJsError::Cancelled)
2034 }
2035 result = &mut run => result,
2036 };
2037 let mut status = WorkflowRunStatus::Completed;
2038 let mut output = None;
2039 let mut error = None;
2040 match result {
2041 Ok(value) => {
2042 if let Some(gate_error) = driver.terminal_gate_failure() {
2043 status = WorkflowRunStatus::Failed;
2044 error = Some(gate_error);
2045 } else {
2046 output = Some(value);
2047 }
2048 }
2049 Err(err) => {
2050 status = WorkflowRunStatus::Failed;
2051 error = Some(err.to_string());
2052 }
2053 }
2054 if driver.parent_cancel_token.is_cancelled() {
2055 driver.finalize_running_tasks_cancelled();
2056 status = WorkflowRunStatus::Cancelled;
2057 output = None;
2058 error = Some("Workflow cancelled by its parent session".to_string());
2059 }
2060 let snapshot = {
2061 let mut runs_guard = match state.runs.lock() {
2062 Ok(guard) => guard,
2063 Err(_) => {
2064 state.mark_owner_missing(&run_id);
2065 return;
2066 }
2067 };
2068 let Some(record) = runs_guard.get_mut(&run_id) else {
2069 state.mark_owner_missing(&run_id);
2070 return;
2071 };
2072 if record.status != WorkflowRunStatus::Cancelled {
2073 // Receipt honesty: a script that returns a value has not
2074 // necessarily orchestrated anything. Classify against the slot
2075 // ledger — every requested task either became a child with a
2076 // terminal record or landed in `dispatch_failures` (driver
2077 // rejections and, via `ProgressEvent::TaskRejected`, VM-level
2078 // rejections that previously vanished into null slots), and a
2079 // fan-out whose every slot failed — script throws included —
2080 // arrives via the dead-fan-out counter.
2081 if status == WorkflowRunStatus::Completed {
2082 let task_records = driver.task_records_snapshot();
2083 let ledger = SlotLedger {
2084 tasks: task_records.len(),
2085 failed_tasks: task_records
2086 .iter()
2087 .filter(|task| task.failed_for_ledger())
2088 .count(),
2089 rejected: record.dispatch_failure_count,
2090 dead_fanouts: driver.dead_fanout_count(),
2091 dropped_slots: driver.dropped_slot_count(),
2092 any_child_ran: !record.child_ids.is_empty(),
2093 retained_detail: record
2094 .dispatch_failures
2095 .first()
2096 .map(|failure| failure.message.clone()),
2097 };
2098 if let Some((slot_status, slot_error)) = ledger.classify() {
2099 status = slot_status;
2100 error = Some(slot_error);
2101 }
2102 }
2103 record.status = status;
2104 record.result = output;
2105 record.error = error.clone();
2106 if status == WorkflowRunStatus::Cancelled {
2107 let cancelled = WorkflowUiEvent::new(
2108 &driver.owner_session_id,
2109 WorkflowUiEventKind::RunCancelled {
2110 reason: error
2111 .clone()
2112 .unwrap_or_else(|| "Workflow cancelled".to_string()),
2113 },
2114 );
2115 record.push_event(cancelled.clone());
2116 driver.emit_ui_event(&cancelled);
2117 }
2118 record.execution = spec.as_ref().map(|spec| {
2119 execution_from_declarative_spec(spec, driver.task_records_snapshot(), status)
2120 });
2121 record.completed_at_ms = Some(now_ms());
2122 }
2123 record.clone()
2124 };
2125 let verify_on_complete = state
2126 .runs
2127 .lock()
2128 .ok()
2129 .and_then(|guard| guard.get(&run_id).map(|record| record.verify_on_complete))
2130 .unwrap_or(false);
2131 if snapshot.status == WorkflowRunStatus::Completed && verify_on_complete {
2132 let verification = tokio::select! {
2133 biased;
2134 () = driver.parent_cancel_token.cancelled() => {
2135 if let Ok(mut runs_guard) = state.runs.lock()
2136 && let Some(record) = runs_guard.get_mut(&run_id)
2137 {
2138 record.status = WorkflowRunStatus::Cancelled;
2139 record.error = Some("Workflow cancelled during completion verification".to_string());
2140 }
2141 None
2142 }
2143 verification = run_workflow_completion_gates(&context) => Some(verification),
2144 };
2145 match verification {
2146 Some(Ok(verification)) => {
2147 if let Ok(mut runs_guard) = state.runs.lock()
2148 && let Some(record) = runs_guard.get_mut(&run_id)
2149 && record.status != WorkflowRunStatus::Cancelled
2150 {
2151 record.verification = Some(verification);
2152 }
2153 }
2154 Some(Err(err)) => {
2155 if let Ok(mut runs_guard) = state.runs.lock()
2156 && let Some(record) = runs_guard.get_mut(&run_id)
2157 && record.status != WorkflowRunStatus::Cancelled
2158 {
2159 record.status = WorkflowRunStatus::Failed;
2160 record.error = Some(format!("verification gates failed: {err}"));
2161 }
2162 }
2163 None => {}
2164 }
2165 }
2166 let final_budget = driver.current_budget_snapshot();
2167 // Reconcile run-wide usage totals from per-task telemetry (#2974).
2168 let run_usage = run_usage_totals(&driver.task_records_snapshot());
2169 let snapshot = state
2170 .runs
2171 .lock()
2172 .ok()
2173 .and_then(|mut guard| {
2174 let record = guard.get_mut(&run_id)?;
2175 if record.status != WorkflowRunStatus::Cancelled {
2176 record.lifecycle_seq = record.lifecycle_seq.saturating_add(1);
2177 if run_usage.is_some() {
2178 record.usage = run_usage.clone();
2179 }
2180 let budget_event =
2181 WorkflowUiEvent::new(&driver.owner_session_id, budget_event_kind(final_budget));
2182 let completed = WorkflowUiEvent::new(
2183 &driver.owner_session_id,
2184 WorkflowUiEventKind::RunCompleted {
2185 status: record.status,
2186 error: record.error.clone(),
2187 usage: run_usage.clone(),
2188 },
2189 );
2190 record.push_event(budget_event.clone());
2191 record.push_event(completed.clone());
2192 // Live stream terminal events even when recorded outside the
2193 // driver helper (completion path).
2194 driver.emit_ui_event(&budget_event);
2195 driver.emit_ui_event(&completed);
2196 }
2197 Some(record.clone())
2198 })
2199 .unwrap_or(snapshot);
2200 let mut terminal = snapshot;
2201 if let Err(err) = state.try_record_snapshot(&terminal) {
2202 // Status reads and the parent receipt must agree when the final
2203 // journal append fails. Keep completed child output as evidence and
2204 // preserve any cancellation recorded while the append was attempted.
2205 terminal = {
2206 let mut runs = state
2207 .runs
2208 .lock()
2209 .unwrap_or_else(|poison| poison.into_inner());
2210 let record = runs.entry(run_id.clone()).or_insert(terminal);
2211 if record.status != WorkflowRunStatus::Cancelled {
2212 record.status = WorkflowRunStatus::Failed;
2213 }
2214 let persistence_error =
2215 format!("workflow terminal journal snapshot could not be persisted: {err}");
2216 let error = match record.error.take() {
2217 Some(error) => format!("{persistence_error}; prior outcome: {error}"),
2218 None => persistence_error,
2219 };
2220 record.error = Some(error.clone());
2221 record.lifecycle_seq = record.lifecycle_seq.saturating_add(1);
2222 if let Some(execution) = record.execution.as_mut() {
2223 if record.status == WorkflowRunStatus::Cancelled {
2224 execution.mark_cancelled();
2225 } else {
2226 execution.mark_failed();
2227 }
2228 }
2229 let event = WorkflowUiEvent::new(
2230 &driver.owner_session_id,
2231 if record.status == WorkflowRunStatus::Cancelled {
2232 WorkflowUiEventKind::RunCancelled { reason: error }
2233 } else {
2234 WorkflowUiEventKind::RunCompleted {
2235 status: record.status,
2236 error: record.error.clone(),
2237 usage: record.usage.clone(),
2238 }
2239 },
2240 );
2241 record.push_event(event.clone());
2242 driver.emit_ui_event(&event);
2243 record.clone()
2244 };
2245 state.mark_owner_missing(&run_id);
2246 } else {
2247 state.reconcile_snapshot(&terminal);
2248 }
2249 write_run_report_artifact(&context.workspace, &terminal);
2250 finish_workflow_controller(&state, &terminal);
2251 }
2252
2253 mod report;
2254
2255 use report::{bounded_raw_preview, write_run_report_artifact, write_schema_raw_artifact};
2256
2257 fn session_workflow_config_store()
2258 -> &'static Mutex<HashMap<PathBuf, codewhale_config::WorkflowConfigToml>> {
2259 static STORE: OnceLock<Mutex<HashMap<PathBuf, codewhale_config::WorkflowConfigToml>>> =
2260 OnceLock::new();
2261 STORE.get_or_init(|| Mutex::new(HashMap::new()))
2262 }
2263
2264 fn workflow_session_key(workspace: &Path) -> PathBuf {
2265 workspace
2266 .canonicalize()
2267 .unwrap_or_else(|_| workspace.to_path_buf())
2268 }
2269
2270 /// Install the session `[workflow]` table after a config.toml reload (or a
2271 /// test mutation). `/workflow settings` and the workflow tool both read this
2272 /// so a refresh cannot leave the two surfaces disagreeing.
2273 pub(crate) fn set_session_workflow_config(
2274 workspace: &Path,
2275 config: codewhale_config::WorkflowConfigToml,
2276 ) {
2277 session_workflow_config_store()
2278 .lock()
2279 .unwrap_or_else(|poison| poison.into_inner())
2280 .insert(workflow_session_key(workspace), config);
2281 }
2282
2283 /// The refreshed session `[workflow]` table, if a reload (or test) installed
2284 /// one for this workspace.
2285 pub(crate) fn session_workflow_config(
2286 workspace: &Path,
2287 ) -> Option<codewhale_config::WorkflowConfigToml> {
2288 session_workflow_config_store()
2289 .lock()
2290 .unwrap_or_else(|poison| poison.into_inner())
2291 .get(&workflow_session_key(workspace))
2292 .cloned()
2293 }
2294
2295 /// The effective `[workflow]` table: the refreshed session table when one has
2296 /// been installed, otherwise the runtime snapshot, otherwise product defaults.
2297 fn workflow_config_for(runtime: &SubAgentRuntime) -> codewhale_config::WorkflowConfigToml {
2298 session_workflow_config(&runtime.context.workspace).unwrap_or_else(|| {
2299 runtime
2300 .api_config
2301 .as_deref()
2302 .map(crate::config::Config::workflow_config)
2303 .unwrap_or_default()
2304 })
2305 }
2306
2307 fn workflow_result_for(
2308 run_id: &str,
2309 state: Arc<WorkflowWorkspaceState>,
2310 owner_session_id: &str,
2311 ) -> Result<ToolResult, ToolError> {
2312 let record = {
2313 let runs_guard = lock_mutex(&state.runs)?;
2314 runs_guard
2315 .get(run_id)
2316 .filter(|record| record.owner_session_id.as_deref() == Some(owner_session_id))
2317 .cloned()
2318 .ok_or_else(|| {
2319 ToolError::invalid_input(format!("Unknown workflow run_id '{run_id}'"))
2320 })?
2321 };
2322 let journal_path = state.journal_path().to_path_buf();
2323 let (payload, bounds) = bounded_run_record_value(&record, &journal_path);
2324 let mut result =
2325 ToolResult::json(&payload).map_err(|err| ToolError::execution_failed(err.to_string()))?;
2326 let summary = record.summary();
2327 result.metadata = Some(json!({
2328 "run_id": summary.run_id,
2329 "status": summary.status,
2330 "terminal": summary.status != WorkflowRunStatus::Running,
2331 "child_count": summary.child_count,
2332 "schema_error_count": summary.schema_error_count,
2333 "schema_repair_count": summary.schema_repair_count,
2334 "dispatch_failure_count": summary.dispatch_failure_count,
2335 "event_count": summary.event_count,
2336 "events_returned": bounds.events_returned,
2337 "events_omitted": bounds.events_omitted,
2338 "dispatch_failures_returned": bounds.dispatch_failures_returned,
2339 "dispatch_failures_omitted": bounds.dispatch_failures_omitted,
2340 "events_dropped": summary.events_dropped,
2341 "last_event_type": summary.last_event_type,
2342 "leaf_count": summary.leaf_count,
2343 "branch_count": summary.branch_count,
2344 "control_count": summary.control_count,
2345 "execution_status": summary.execution_status,
2346 "gate_count": summary.gate_count,
2347 "blocked_gate_count": summary.blocked_gate_count,
2348 "gate_status": summary.gate_status,
2349 // #2974: bounded payload; full detail stays in the durable journal.
2350 "truncated": bounds.truncated(),
2351 "payload_budget_chars": WORKFLOW_RESULT_MAX_CHARS,
2352 "journal_path": journal_path.display().to_string(),
2353 // #4126: durable plan-approval receipt for audit/receipt consumers.
2354 "plan_approval": record.plan_approval,
2355 }));
2356 Ok(result)
2357 }
2358
2359 /// What `bounded_run_record_value` clipped out of the model-facing payload.
2360 #[derive(Debug, Default)]
2361 struct RunPayloadBounds {
2362 events_returned: usize,
2363 events_omitted: usize,
2364 progress_returned: usize,
2365 progress_omitted: u64,
2366 dispatch_failures_returned: usize,
2367 dispatch_failures_omitted: u64,
2368 dispatch_failure_fields_truncated: usize,
2369 result_truncated: bool,
2370 leaf_outputs_truncated: usize,
2371 }
2372
2373 impl RunPayloadBounds {
2374 fn truncated(&self) -> bool {
2375 self.events_omitted > 0
2376 || self.progress_omitted > 0
2377 || self.dispatch_failures_omitted > 0
2378 || self.dispatch_failure_fields_truncated > 0
2379 || self.result_truncated
2380 || self.leaf_outputs_truncated > 0
2381 }
2382 }
2383
2384 /// Build the model-facing view of a run record (#2974). The JSON shape is
2385 /// identical to the full record (panel hydration and history cards keep
2386 /// working unchanged), but the unbounded parts are clipped:
2387 ///
2388 /// - `events`: newest `WORKFLOW_RESULT_EVENTS_TAIL` entries.
2389 /// - `progress`: newest `WORKFLOW_RESULT_PROGRESS_TAIL` lines.
2390 /// - `dispatch_failures`: newest `WORKFLOW_RESULT_DISPATCH_FAILURES_TAIL`
2391 /// entries with bounded string fields.
2392 /// - `result` / `verification`: collapsed to a preview + journal pointer
2393 /// when the serialized value exceeds `WORKFLOW_RESULT_VALUE_MAX_CHARS`.
2394 /// - `execution.leaf_results[*].output`: per-leaf preview capped at
2395 /// `WORKFLOW_RESULT_LEAF_OUTPUT_MAX_CHARS`.
2396 ///
2397 /// Full detail remains available in `.codewhale/workflow-runs.jsonl`; every
2398 /// clip adds an explicit note/pointer so the model can fetch more on demand.
2399 fn bounded_run_record_value(
2400 record: &WorkflowRunRecord,
2401 journal_path: &Path,
2402 ) -> (Value, RunPayloadBounds) {
2403 let mut bounds = RunPayloadBounds::default();
2404 let journal = journal_path.display().to_string();
2405 let mut value = serde_json::to_value(record).unwrap_or_else(|_| json!({}));
2406 let Some(obj) = value.as_object_mut() else {
2407 return (value, bounds);
2408 };
2409
2410 // The structured failure tail below is intentionally clipped, but panel
2411 // summaries still need the exact saturating total to remain truthful
2412 // after replay.
2413 obj.insert(
2414 "dispatch_failure_count".to_string(),
2415 json!(record.dispatch_failure_count),
2416 );
2417 obj.insert("progress_count".to_string(), json!(record.progress_count));
2418
2419 if let Some(events) = obj.get_mut("events").and_then(Value::as_array_mut) {
2420 if events.len() > WORKFLOW_RESULT_EVENTS_TAIL {
2421 let omitted = events.len() - WORKFLOW_RESULT_EVENTS_TAIL;
2422 events.drain(..omitted);
2423 bounds.events_omitted = omitted;
2424 }
2425 bounds.events_returned = events.len();
2426 }
2427 if bounds.events_omitted > 0 {
2428 obj.insert(
2429 "events_note".to_string(),
2430 json!(format!(
2431 "showing the newest {} of {} events; full stream: {journal}",
2432 bounds.events_returned,
2433 bounds.events_returned + bounds.events_omitted,
2434 )),
2435 );
2436 }
2437
2438 if let Some(progress) = obj.get_mut("progress").and_then(Value::as_array_mut) {
2439 if progress.len() > WORKFLOW_RESULT_PROGRESS_TAIL {
2440 let omitted = progress.len() - WORKFLOW_RESULT_PROGRESS_TAIL;
2441 progress.drain(..omitted);
2442 }
2443 bounds.progress_returned = progress.len();
2444 let returned = u64::try_from(bounds.progress_returned).unwrap_or(u64::MAX);
2445 bounds.progress_omitted = record.progress_count.saturating_sub(returned);
2446 }
2447 if bounds.progress_omitted > 0 {
2448 obj.insert(
2449 "progress_note".to_string(),
2450 json!(format!(
2451 "showing the newest {} of {} progress lines; full log: {journal}",
2452 bounds.progress_returned, record.progress_count,
2453 )),
2454 );
2455 }
2456
2457 if let Some(failures) = obj
2458 .get_mut("dispatch_failures")
2459 .and_then(Value::as_array_mut)
2460 {
2461 if failures.len() > WORKFLOW_RESULT_DISPATCH_FAILURES_TAIL {
2462 let omitted = failures.len() - WORKFLOW_RESULT_DISPATCH_FAILURES_TAIL;
2463 failures.drain(..omitted);
2464 }
2465 bounds.dispatch_failures_returned = failures.len();
2466 for failure in failures {
2467 let Some(fields) = failure.as_object_mut() else {
2468 continue;
2469 };
2470 for key in ["label", "phase", "message"] {
2471 let Some(slot) = fields.get_mut(key) else {
2472 continue;
2473 };
2474 let Some(raw) = slot.as_str() else {
2475 continue;
2476 };
2477 if raw.chars().count() > WORKFLOW_RESULT_DISPATCH_FAILURE_FIELD_MAX_CHARS {
2478 *slot = Value::String(truncate_chars(
2479 raw,
2480 WORKFLOW_RESULT_DISPATCH_FAILURE_FIELD_MAX_CHARS,
2481 ));
2482 bounds.dispatch_failure_fields_truncated += 1;
2483 }
2484 }
2485 }
2486 }
2487 bounds.dispatch_failures_omitted = record
2488 .dispatch_failure_count
2489 .saturating_sub(u64::try_from(bounds.dispatch_failures_returned).unwrap_or(u64::MAX));
2490 if bounds.dispatch_failures_omitted > 0 || bounds.dispatch_failure_fields_truncated > 0 {
2491 obj.insert(
2492 "dispatch_failures_note".to_string(),
2493 json!(format!(
2494 "showing {} of {} dispatch failures with bounded fields; full record: {journal}",
2495 bounds.dispatch_failures_returned, record.dispatch_failure_count,
2496 )),
2497 );
2498 }
2499
2500 for key in ["result", "verification"] {
2501 let Some(raw) = obj.get(key).filter(|value| !value.is_null()) else {
2502 continue;
2503 };
2504 let text = raw.to_string();
2505 if text.chars().count() > WORKFLOW_RESULT_VALUE_MAX_CHARS {
2506 obj.insert(
2507 key.to_string(),
2508 json!({
2509 "truncated": true,
2510 "omitted_chars": text.chars().count() - WORKFLOW_RESULT_VALUE_MAX_CHARS,
2511 "preview": truncate_chars(&text, WORKFLOW_RESULT_VALUE_MAX_CHARS),
2512 "full_detail": journal,
2513 }),
2514 );
2515 bounds.result_truncated = true;
2516 }
2517 }
2518
2519 if let Some(leaves) = obj
2520 .get_mut("execution")
2521 .and_then(|execution| execution.get_mut("leaf_results"))
2522 .and_then(Value::as_array_mut)
2523 {
2524 for leaf in leaves {
2525 let too_long = leaf
2526 .get("output")
2527 .and_then(Value::as_str)
2528 .is_some_and(|output| {
2529 output.chars().count() > WORKFLOW_RESULT_LEAF_OUTPUT_MAX_CHARS
2530 });
2531 if too_long
2532 && let Some(slot) = leaf.get_mut("output")
2533 && let Some(output) = slot.as_str()
2534 {
2535 let clipped = format!(
2536 "{} [leaf output truncated to {WORKFLOW_RESULT_LEAF_OUTPUT_MAX_CHARS} chars; full text: {journal}]",
2537 truncate_chars(output, WORKFLOW_RESULT_LEAF_OUTPUT_MAX_CHARS),
2538 );
2539 *slot = Value::String(clipped);
2540 bounds.leaf_outputs_truncated += 1;
2541 }
2542 }
2543 }
2544
2545 (value, bounds)
2546 }
2547
2548 /// Char-boundary-safe truncation with an ellipsis (precedent:
2549 /// `cargo_failure_summary::truncate_chars`).
2550 fn truncate_chars(text: &str, max_chars: usize) -> String {
2551 if let Some((idx, _)) = text.char_indices().nth(max_chars) {
2552 if max_chars < 3 {
2553 return text[..idx].to_string();
2554 }
2555 let truncate_at = text
2556 .char_indices()
2557 .nth(max_chars - 3)
2558 .map(|(idx, _)| idx)
2559 .unwrap_or(0);
2560 format!("{}...", &text[..truncate_at])
2561 } else {
2562 text.to_string()
2563 }
2564 }
2565
2566 #[derive(Debug)]
2567 struct WorkflowSource {
2568 source: String,
2569 path: Option<PathBuf>,
2570 spec: Option<WorkflowSpec>,
2571 }
2572
2573 fn workflow_source(input: &Value, context: &ToolContext) -> Result<WorkflowSource, ToolError> {
2574 let script = match optional_str(input, "script")? {
2575 Some(script) => Some(script),
2576 None => optional_str(input, "source")?,
2577 }
2578 .map(str::to_string);
2579 let source_path = match optional_str(input, "source_path")? {
2580 Some(path) => Some(path),
2581 None => optional_str(input, "path")?,
2582 };
2583 let plan = input.get("plan").filter(|value| !value.is_null());
2584
2585 let provided = [
2586 script.as_ref().is_some_and(|s| !s.trim().is_empty()),
2587 source_path.is_some(),
2588 plan.is_some(),
2589 ]
2590 .into_iter()
2591 .filter(|present| *present)
2592 .count();
2593 if provided > 1 {
2594 return Err(ToolError::invalid_input(
2595 "Use exactly one of script, source_path, or plan",
2596 ));
2597 }
2598
2599 match (script, source_path, plan) {
2600 (Some(source), None, None) if !source.trim().is_empty() => {
2601 workflow_source_from_raw(source, None)
2602 }
2603 (None, Some(path), None) => read_workflow_source_path(path, context),
2604 (None, None, Some(plan_value)) => workflow_source_from_plan(plan_value),
2605 _ => Err(ToolError::missing_field("script")),
2606 }
2607 }
2608
2609 /// Planner-to-workflow structured launch path (#4124).
2610 ///
2611 /// Accepts product-shaped plans (`goal` + `phases`/`children`) or IR-shaped
2612 /// plans (`goal` + `nodes`), validates them, and lowers to imperative JS that
2613 /// uses `parallel()` (partial success) rather than raw `Promise.all()`.
2614 fn workflow_source_from_plan(plan_value: &Value) -> Result<WorkflowSource, ToolError> {
2615 let spec = structured_plan_to_workflow_spec(plan_value)?;
2616 let lowered = lower_declarative_workflow_to_imperative_js(&spec)?;
2617 Ok(WorkflowSource {
2618 source: lowered,
2619 path: None,
2620 spec: Some(spec),
2621 })
2622 }
2623
2624 #[derive(Debug, Deserialize)]
2625 struct StructuredWorkflowPlan {
2626 goal: String,
2627 #[serde(default)]
2628 risk: Option<String>,
2629 #[serde(default)]
2630 max_children: Option<usize>,
2631 #[serde(default)]
2632 token_budget: Option<u64>,
2633 #[serde(default)]
2634 phases: Vec<StructuredPlanPhase>,
2635 #[serde(default)]
2636 children: Vec<StructuredPlanChild>,
2637 /// Escape hatch: full Workflow IR nodes (kind/spec or JS authoring shapes).
2638 #[serde(default)]
2639 nodes: Option<Value>,
2640 /// Optional Workflow-owned gate specs (#4179).
2641 #[serde(default)]
2642 gates: Vec<GateSpec>,
2643 }
2644
2645 #[derive(Debug, Deserialize)]
2646 struct StructuredPlanPhase {
2647 #[serde(default)]
2648 id: Option<String>,
2649 #[serde(default)]
2650 title: Option<String>,
2651 #[serde(default)]
2652 parallel: Option<bool>,
2653 #[serde(default)]
2654 children: Vec<StructuredPlanChild>,
2655 }
2656
2657 #[derive(Debug, Deserialize)]
2658 struct StructuredPlanChild {
2659 #[serde(default)]
2660 id: Option<String>,
2661 #[serde(default)]
2662 label: Option<String>,
2663 #[serde(alias = "description")]
2664 prompt: String,
2665 #[serde(default, alias = "type", alias = "agent_type")]
2666 agent_type: Option<String>,
2667 /// Fleet role name (#4177). Preferred step identity; resolved via roster.
2668 #[serde(default)]
2669 role: Option<String>,
2670 #[serde(default)]
2671 profile: Option<String>,
2672 /// Saved Fleet shortlist selector, resolved by the existing task binder.
2673 #[serde(default)]
2674 model: Option<String>,
2675 #[serde(default)]
2676 mode: Option<String>,
2677 #[serde(default)]
2678 file_scope: Vec<String>,
2679 /// Optional child working directory, repository-relative like
2680 /// `task({cwd})`. Disambiguates multi-repo workspaces (#6232).
2681 #[serde(default)]
2682 cwd: Option<String>,
2683 }
2684
2685 fn structured_plan_to_workflow_spec(plan_value: &Value) -> Result<WorkflowSpec, ToolError> {
2686 if !plan_value.is_object() {
2687 return Err(ToolError::invalid_input(
2688 "Workflow plan must be a JSON object with goal and phases/children (or nodes)",
2689 ));
2690 }
2691
2692 let plan: StructuredWorkflowPlan =
2693 serde_json::from_value(plan_value.clone()).map_err(|err| {
2694 ToolError::invalid_input(format!("Invalid structured Workflow plan: {err}"))
2695 })?;
2696
2697 let goal = plan.goal.trim();
2698 if goal.is_empty() {
2699 return Err(ToolError::invalid_input(
2700 "Workflow plan goal must be a non-empty string",
2701 ));
2702 }
2703
2704 // IR / declarative nodes escape hatch: re-parse as workflow({...}) object.
2705 if let Some(nodes) = plan.nodes.as_ref() {
2706 if !nodes.is_array() {
2707 return Err(ToolError::invalid_input(
2708 "Workflow plan.nodes must be an array of workflow nodes",
2709 ));
2710 }
2711 let mut object = plan_value.clone();
2712 if let Some(obj) = object.as_object_mut() {
2713 obj.insert("goal".to_string(), Value::String(goal.to_string()));
2714 if let Some(token_budget) = plan.token_budget {
2715 let mut budget = obj.get("budget").cloned().unwrap_or_else(|| json!({}));
2716 if let Some(budget_obj) = budget.as_object_mut() {
2717 budget_obj.insert("max_tokens".to_string(), json!(token_budget));
2718 }
2719 obj.insert("budget".to_string(), budget);
2720 }
2721 }
2722 let wrapped = format!("workflow({});", object);
2723 return compile_javascript_workflow("<structured plan>", &wrapped).map_err(|err| {
2724 ToolError::invalid_input(format!("Invalid structured Workflow plan nodes: {err}"))
2725 });
2726 }
2727
2728 let default_mode = plan_risk_to_mode(plan.risk.as_deref())?;
2729 let mut nodes = Vec::new();
2730
2731 if !plan.phases.is_empty() {
2732 let mut prior_results = Vec::new();
2733 for (phase_index, phase) in plan.phases.iter().enumerate() {
2734 let phase_id = phase
2735 .id
2736 .as_deref()
2737 .or(phase.title.as_deref())
2738 .map(str::trim)
2739 .filter(|id| !id.is_empty())
2740 .map(str::to_string)
2741 .unwrap_or_else(|| format!("phase-{}", phase_index + 1));
2742 let mut children = plan_children_to_leaves(
2743 &phase.children,
2744 default_mode,
2745 plan.token_budget,
2746 &phase_id,
2747 )?;
2748 if children.is_empty() {
2749 return Err(ToolError::invalid_input(format!(
2750 "Workflow plan phase '{phase_id}' must declare at least one child"
2751 )));
2752 }
2753 let parallel = phase.parallel.unwrap_or(children.len() > 1);
2754 // Phases are a data handoff as well as an ordering boundary.
2755 // Reuse the IR's dependency field so ordinary structured plans
2756 // get the same result forwarding as authored Workflow nodes.
2757 for child in &mut children {
2758 child.depends_on_results = prior_results.clone();
2759 if !parallel {
2760 prior_results.push(child.id.clone());
2761 }
2762 }
2763 prior_results = children.iter().map(|child| child.id.clone()).collect();
2764 if parallel && children.len() > 1 {
2765 nodes.push(WorkflowNode::BranchSet(BranchSpec {
2766 id: phase_id,
2767 description: phase.title.clone(),
2768 parallel: true,
2769 budget: BudgetSpec {
2770 max_tokens: plan.token_budget,
2771 ..BudgetSpec::default()
2772 },
2773 permissions: Default::default(),
2774 model_policy: Default::default(),
2775 children: children.into_iter().map(WorkflowNode::Leaf).collect(),
2776 }));
2777 } else {
2778 // A one-child phase still owns a visible phase and task
2779 // grouping; flattening it loses the usual inspect/build/test
2780 // progression from the shared activity view.
2781 nodes.push(WorkflowNode::Sequence(SequenceSpec {
2782 id: phase_id,
2783 children: children.into_iter().map(WorkflowNode::Leaf).collect(),
2784 }));
2785 }
2786 }
2787 } else if !plan.children.is_empty() {
2788 let children =
2789 plan_children_to_leaves(&plan.children, default_mode, plan.token_budget, "plan")?;
2790 if children.len() == 1 {
2791 nodes.push(WorkflowNode::Leaf(
2792 children.into_iter().next().expect("one child"),
2793 ));
2794 } else {
2795 nodes.push(WorkflowNode::BranchSet(BranchSpec {
2796 id: "plan".to_string(),
2797 description: Some(goal.to_string()),
2798 parallel: true,
2799 budget: BudgetSpec {
2800 max_tokens: plan.token_budget,
2801 ..BudgetSpec::default()
2802 },
2803 permissions: Default::default(),
2804 model_policy: Default::default(),
2805 children: children.into_iter().map(WorkflowNode::Leaf).collect(),
2806 }));
2807 }
2808 } else {
2809 return Err(ToolError::invalid_input(
2810 "Workflow plan must include phases, children, or nodes",
2811 ));
2812 }
2813
2814 let mut total_children = 0usize;
2815 count_plan_leaves(&nodes, &mut total_children);
2816 if let Some(max_children) = plan.max_children
2817 && total_children > max_children
2818 {
2819 return Err(ToolError::invalid_input(format!(
2820 "Workflow plan declares {total_children} children which exceeds max_children={max_children}"
2821 )));
2822 }
2823
2824 let spec = WorkflowSpec {
2825 id: None,
2826 goal: goal.to_string(),
2827 description: plan.risk.clone(),
2828 budget: BudgetSpec {
2829 max_tokens: plan.token_budget,
2830 ..BudgetSpec::default()
2831 },
2832 permissions: Default::default(),
2833 model_policy: Default::default(),
2834 promotion_policy: Default::default(),
2835 gates: plan.gates,
2836 nodes,
2837 };
2838 spec.validate_for_fleet().map_err(|error| {
2839 ToolError::invalid_input(format!("Invalid structured Workflow plan: {error}"))
2840 })?;
2841 Ok(spec)
2842 }
2843
2844 fn plan_risk_to_mode(risk: Option<&str>) -> Result<TaskMode, ToolError> {
2845 match risk.map(str::trim).filter(|s| !s.is_empty()) {
2846 None | Some("read_only") | Some("readonly") | Some("low") | Some("safe") => {
2847 Ok(TaskMode::ReadOnly)
2848 }
2849 Some("writes") | Some("write") | Some("read_write") | Some("readwrite")
2850 | Some("medium") => Ok(TaskMode::ReadWrite),
2851 Some("elevated") | Some("high") | Some("shell") | Some("network") => {
2852 // Elevated risk still launches as read_write; approval gates (#4126)
2853 // consume the risk string via plan description.
2854 Ok(TaskMode::ReadWrite)
2855 }
2856 Some(other) => Err(ToolError::invalid_input(format!(
2857 "Invalid plan risk '{other}'. Use read_only, writes, or elevated."
2858 ))),
2859 }
2860 }
2861
2862 fn plan_children_to_leaves(
2863 children: &[StructuredPlanChild],
2864 default_mode: TaskMode,
2865 token_budget: Option<u64>,
2866 phase_id: &str,
2867 ) -> Result<Vec<LeafSpec>, ToolError> {
2868 if children.is_empty() {
2869 return Ok(Vec::new());
2870 }
2871 let mut leaves = Vec::with_capacity(children.len());
2872 for (index, child) in children.iter().enumerate() {
2873 let prompt = child.prompt.trim();
2874 if prompt.is_empty() {
2875 return Err(ToolError::invalid_input(format!(
2876 "Workflow plan child {} in phase '{phase_id}' must have a non-empty prompt",
2877 index + 1
2878 )));
2879 }
2880 let id = child
2881 .id
2882 .as_deref()
2883 .or(child.label.as_deref())
2884 .map(str::trim)
2885 .filter(|id| !id.is_empty())
2886 .map(str::to_string)
2887 .unwrap_or_else(|| format!("{phase_id}-child-{}", index + 1));
2888 let agent_type = parse_plan_agent_type(child.agent_type.as_deref())?;
2889 let mode = match child.mode.as_deref().map(str::trim) {
2890 None | Some("") => default_mode,
2891 Some("read_only") | Some("readonly") => TaskMode::ReadOnly,
2892 Some("read_write") | Some("readwrite") | Some("writes") | Some("write") => {
2893 TaskMode::ReadWrite
2894 }
2895 Some(other) => {
2896 return Err(ToolError::invalid_input(format!(
2897 "Invalid plan child mode '{other}' on '{id}'. Use read_only or read_write."
2898 )));
2899 }
2900 };
2901 let role = child
2902 .role
2903 .as_deref()
2904 .map(str::trim)
2905 .filter(|r| !r.is_empty())
2906 .map(|r| r.to_ascii_lowercase());
2907 let profile = child
2908 .profile
2909 .as_deref()
2910 .map(str::trim)
2911 .filter(|p| !p.is_empty())
2912 .map(|p| p.to_ascii_lowercase());
2913 let model = child.model.as_deref().map(str::trim);
2914 if model == Some("") {
2915 return Err(ToolError::invalid_input(format!(
2916 "Workflow plan child '{id}' model must not be empty"
2917 )));
2918 }
2919 let cwd = child.cwd.as_deref().map(str::trim);
2920 if cwd == Some("") {
2921 return Err(ToolError::invalid_input(format!(
2922 "Workflow plan child '{id}' cwd must not be empty"
2923 )));
2924 }
2925 leaves.push(LeafSpec {
2926 id,
2927 prompt: prompt.to_string(),
2928 agent_type,
2929 role,
2930 profile,
2931 mode,
2932 isolation: Default::default(),
2933 file_scope: child.file_scope.clone(),
2934 cwd: cwd.map(str::to_string),
2935 depends_on_results: Vec::new(),
2936 budget: BudgetSpec {
2937 max_tokens: token_budget,
2938 ..BudgetSpec::default()
2939 },
2940 permissions: Default::default(),
2941 model_policy: codewhale_workflow::ModelPolicy {
2942 model: model.map(str::to_string),
2943 ..Default::default()
2944 },
2945 });
2946 }
2947 Ok(leaves)
2948 }
2949
2950 /// Plan-child `type` vocabulary. Accepts the Agent tool's canonical types and
2951 /// legacy aliases so the same option value works for direct Agent dispatch and
2952 /// for Workflow plan children (#5035), normalized onto the workflow IR schema.
2953 /// Rejections use the Agent tool's error contract ("Invalid sub-agent type
2954 /// `'<value>'. Use: ...`") with field-specific guidance.
2955 fn parse_plan_agent_type(raw: Option<&str>) -> Result<AgentType, ToolError> {
2956 let Some(kind) = raw.map(str::trim).filter(|s| !s.is_empty()) else {
2957 return Ok(AgentType::General);
2958 };
2959 // `delegate` is a Workflow-only legacy alias. Every shared spelling uses
2960 // the same parser as direct sub-agent launches and Fleet configuration.
2961 let role = if kind.eq_ignore_ascii_case("delegate") {
2962 Some(FleetRole::Worker)
2963 } else {
2964 FleetRole::from_str(kind)
2965 };
2966 match role {
2967 Some(FleetRole::Worker) => Ok(AgentType::General),
2968 Some(FleetRole::Scout) => Ok(AgentType::Explore),
2969 Some(FleetRole::Planner) => Ok(AgentType::Plan),
2970 Some(FleetRole::Reviewer | FleetRole::Consultant) => Ok(AgentType::Review),
2971 Some(FleetRole::Builder) => Ok(AgentType::Implementer),
2972 Some(FleetRole::Verifier) => Ok(AgentType::Verifier),
2973 Some(FleetRole::Custom) => Err(ToolError::invalid_input(
2974 "Invalid sub-agent type 'custom' for a Workflow plan child: custom requires an \
2975 explicit allowed_tools list, which plan children cannot declare. Use role/profile \
2976 or another type.",
2977 )),
2978 None => Err(ToolError::invalid_input(format!(
2979 "Invalid sub-agent type '{kind}'. Use: worker, scout, planner, reviewer, builder, \
2980 verifier (legacy aliases remain accepted: general, explore/explorer, plan/awaiter, \
2981 review, implementer, consultant/oracle/advisor)."
2982 ))),
2983 }
2984 }
2985
2986 /// Gate identity is runtime input, including for structured plans that do not
2987 /// pass through the JavaScript author's normalization path.
2988 fn validate_runtime_gates(gates: &mut [GateSpec]) -> Result<(), ToolError> {
2989 let mut ids = HashSet::new();
2990 for gate in gates {
2991 gate.id = gate.id.trim().to_string();
2992 if gate.id.is_empty() || !ids.insert(gate.id.clone()) {
2993 return Err(ToolError::invalid_input(
2994 "Workflow gates require non-empty, unique ids",
2995 ));
2996 }
2997 gate.role = gate.role.trim().to_lowercase();
2998 if gate.role.is_empty() {
2999 return Err(ToolError::invalid_input(
3000 "Workflow gate role must not be empty",
3001 ));
3002 }
3003 if let Some(role) = gate.blocks_role.as_mut() {
3004 *role = role.trim().to_lowercase();
3005 if role.is_empty() {
3006 return Err(ToolError::invalid_input(
3007 "Workflow gate blocks_role must not be empty",
3008 ));
3009 }
3010 }
3011 if gate.on != GateOn::RoleComplete {
3012 return Err(ToolError::invalid_input(
3013 "Workflow role_start gates are not supported; use role_complete with an explicit prerequisite role",
3014 ));
3015 }
3016 }
3017 Ok(())
3018 }
3019
3020 fn count_plan_leaves(nodes: &[WorkflowNode], total: &mut usize) {
3021 for node in nodes {
3022 match node {
3023 WorkflowNode::Leaf(_) => *total += 1,
3024 WorkflowNode::BranchSet(spec) => count_plan_leaves(&spec.children, total),
3025 WorkflowNode::Sequence(spec) => count_plan_leaves(&spec.children, total),
3026 WorkflowNode::Reduce(_)
3027 | WorkflowNode::TeacherReview(_)
3028 | WorkflowNode::LoopUntil(_)
3029 | WorkflowNode::Cond(_)
3030 | WorkflowNode::Expand(_) => {}
3031 }
3032 }
3033 }
3034
3035 fn read_workflow_source_path(
3036 path: &str,
3037 context: &ToolContext,
3038 ) -> Result<WorkflowSource, ToolError> {
3039 let raw = Path::new(path);
3040 let joined = if raw.is_absolute() {
3041 raw.to_path_buf()
3042 } else {
3043 context.workspace.join(raw)
3044 };
3045 let canonical = joined.canonicalize().map_err(|err| {
3046 ToolError::invalid_input(format!(
3047 "Failed to resolve workflow source_path '{path}': {err}"
3048 ))
3049 })?;
3050 if !context.trust_mode {
3051 let workspace = context
3052 .workspace
3053 .canonicalize()
3054 .unwrap_or_else(|_| context.workspace.clone());
3055 // The user-global saved-workflow store is a first-class source
3056 // alongside the workspace: `~/.codewhale/workflows/*.workflow.js`
3057 // definitions surface as slash commands and must launch from any
3058 // workspace without trust_mode.
3059 let home_store = crate::config::effective_home_dir()
3060 .map(|home| home.join(".codewhale").join("workflows"))
3061 .and_then(|dir| dir.canonicalize().ok());
3062 let inside_home_store = home_store
3063 .as_deref()
3064 .is_some_and(|dir| canonical.starts_with(dir));
3065 if !canonical.starts_with(&workspace) && !inside_home_store {
3066 return Err(ToolError::permission_denied(format!(
3067 "workflow source_path must stay inside the workspace or ~/.codewhale/workflows: {}",
3068 canonical.display()
3069 )));
3070 }
3071 }
3072 let source = std::fs::read_to_string(&canonical).map_err(|err| {
3073 ToolError::execution_failed(format!(
3074 "Failed to read workflow source_path '{}': {err}",
3075 canonical.display()
3076 ))
3077 })?;
3078 workflow_source_from_raw(source, Some(canonical))
3079 }
3080
3081 fn workflow_source_from_raw(
3082 source: String,
3083 path: Option<PathBuf>,
3084 ) -> Result<WorkflowSource, ToolError> {
3085 let adapted = adapt_workflow_source(&source, path.as_deref())?;
3086 Ok(WorkflowSource {
3087 source: adapted.source,
3088 path,
3089 spec: adapted.spec,
3090 })
3091 }
3092
3093 struct AdaptedWorkflowSource {
3094 source: String,
3095 spec: Option<WorkflowSpec>,
3096 }
3097
3098 fn adapt_workflow_source(
3099 source: &str,
3100 path: Option<&Path>,
3101 ) -> Result<AdaptedWorkflowSource, ToolError> {
3102 if !looks_like_declarative_workflow(source) {
3103 return Ok(AdaptedWorkflowSource {
3104 source: source.to_string(),
3105 spec: None,
3106 });
3107 }
3108
3109 let identifier = path
3110 .map(|path| path.display().to_string())
3111 .unwrap_or_else(|| "<inline workflow>".to_string());
3112 let extension = path
3113 .and_then(Path::extension)
3114 .and_then(|extension| extension.to_str())
3115 .unwrap_or_default();
3116 let spec = if extension.eq_ignore_ascii_case("ts") {
3117 compile_typescript_workflow(&identifier, source)
3118 } else {
3119 compile_javascript_workflow(&identifier, source)
3120 }
3121 .map_err(|err| {
3122 ToolError::invalid_input(format!(
3123 "Failed to compile declarative Workflow source '{identifier}': {err}"
3124 ))
3125 })?;
3126
3127 let lowered = lower_declarative_workflow_to_imperative_js(&spec)?;
3128 Ok(AdaptedWorkflowSource {
3129 source: lowered,
3130 spec: Some(spec),
3131 })
3132 }
3133
3134 fn looks_like_declarative_workflow(source: &str) -> bool {
3135 // Match a top-level `workflow(...)` / `export default workflow(...)` call on
3136 // any line, ignoring leading indentation, so an indented (non-column-0)
3137 // declarative call is still recognized rather than misrun as an imperative
3138 // script (#dogfood 0.8.67).
3139 source.lines().any(|line| {
3140 let trimmed = line.trim_start();
3141 trimmed.starts_with("workflow(") || trimmed.starts_with("export default workflow(")
3142 })
3143 }
3144
3145 fn lower_declarative_workflow_to_imperative_js(spec: &WorkflowSpec) -> Result<String, ToolError> {
3146 let mut lowerer = DeclarativeWorkflowLowerer::default();
3147 lowerer.line("\"use strict\";");
3148 lowerer.line("const __results = Object.create(null);");
3149 lowerer.line(format!(
3150 "phase({});",
3151 js_string(&format!("workflow: {}", spec.goal))
3152 ));
3153 for node in &spec.nodes {
3154 lowerer.lower_node(node, None)?;
3155 }
3156 lowerer.line("return __results;");
3157 Ok(lowerer.finish())
3158 }
3159
3160 #[derive(Default)]
3161 struct DeclarativeWorkflowLowerer {
3162 source: String,
3163 next_var: usize,
3164 }
3165
3166 impl DeclarativeWorkflowLowerer {
3167 fn finish(self) -> String {
3168 self.source
3169 }
3170
3171 fn line(&mut self, line: impl AsRef<str>) {
3172 self.source.push_str(line.as_ref());
3173 self.source.push('\n');
3174 }
3175
3176 fn next_temp(&mut self, prefix: &str) -> String {
3177 let value = format!("__{prefix}_{}", self.next_var);
3178 self.next_var += 1;
3179 value
3180 }
3181
3182 fn lower_node(&mut self, node: &WorkflowNode, phase: Option<&str>) -> Result<(), ToolError> {
3183 match node {
3184 WorkflowNode::Leaf(spec) => self.lower_leaf(spec, phase, /* parallel */ false),
3185 WorkflowNode::BranchSet(spec) => self.lower_branch(spec),
3186 WorkflowNode::Sequence(spec) => self.lower_sequence(spec),
3187 WorkflowNode::Reduce(spec) => self.lower_reduce(spec),
3188 WorkflowNode::TeacherReview(_) => Err(unsupported_declarative_node("teacher_review")),
3189 WorkflowNode::LoopUntil(_) => Err(unsupported_declarative_node("loop_until")),
3190 WorkflowNode::Cond(_) => Err(unsupported_declarative_node("cond")),
3191 WorkflowNode::Expand(_) => Err(unsupported_declarative_node("expand")),
3192 }
3193 }
3194
3195 fn lower_leaf(
3196 &mut self,
3197 spec: &LeafSpec,
3198 phase: Option<&str>,
3199 parallel: bool,
3200 ) -> Result<(), ToolError> {
3201 self.line(format!(
3202 "__results[{}] = await task({});",
3203 js_string(&spec.id),
3204 leaf_task_options_expression(spec, phase, parallel)?
3205 ));
3206 Ok(())
3207 }
3208
3209 fn lower_branch(&mut self, spec: &BranchSpec) -> Result<(), ToolError> {
3210 self.line(format!("phase({});", js_string(&spec.id)));
3211 if spec.parallel {
3212 let mut leaves = Vec::new();
3213 for child in &spec.children {
3214 let WorkflowNode::Leaf(leaf) = child else {
3215 return Err(ToolError::invalid_input(format!(
3216 "Declarative Workflow adapter only supports leaf children inside parallel branch '{}'",
3217 spec.id
3218 )));
3219 };
3220 leaves.push(leaf);
3221 }
3222 // #4124: use Workflow `parallel()` (all-settled / partial success)
3223 // instead of raw Promise.all, which aborts siblings on first failure.
3224 let temp = self.next_temp("parallel");
3225 self.line(format!("const {temp} = await parallel(["));
3226 for leaf in &leaves {
3227 // Parallel write-capable children default to worktree isolation
3228 // (#4120) unless the plan explicitly sets isolation: shared.
3229 self.line(format!(
3230 " () => task({}),",
3231 leaf_task_options_expression(leaf, Some(&spec.id), /* parallel */ true)?
3232 ));
3233 }
3234 self.line("]);");
3235 for (index, leaf) in leaves.iter().enumerate() {
3236 self.line(format!(
3237 "__results[{}] = {temp}[{index}];",
3238 js_string(&leaf.id)
3239 ));
3240 }
3241 return Ok(());
3242 }
3243
3244 for child in &spec.children {
3245 self.lower_node(child, Some(&spec.id))?;
3246 }
3247 Ok(())
3248 }
3249
3250 fn lower_sequence(&mut self, spec: &SequenceSpec) -> Result<(), ToolError> {
3251 self.line(format!("phase({});", js_string(&spec.id)));
3252 for child in &spec.children {
3253 self.lower_node(child, Some(&spec.id))?;
3254 }
3255 Ok(())
3256 }
3257
3258 fn lower_reduce(&mut self, spec: &ReduceSpec) -> Result<(), ToolError> {
3259 let inputs = result_inputs_expression(&spec.inputs);
3260 self.line(format!(
3261 "__results[{}] = await task({});",
3262 js_string(&spec.id),
3263 task_options_expression(
3264 format!(
3265 "{} + \"\\n\\nInputs:\\n\" + {inputs}",
3266 js_string(&spec.prompt)
3267 ),
3268 Some("plan"),
3269 None,
3270 None,
3271 spec.model_policy.model.as_deref(),
3272 false,
3273 None,
3274 None,
3275 None,
3276 Some("read_only"),
3277 &[],
3278 &spec.id,
3279 Some("reduce"),
3280 None,
3281 None,
3282 )
3283 ));
3284 Ok(())
3285 }
3286 }
3287
3288 fn unsupported_declarative_node(kind: &str) -> ToolError {
3289 ToolError::invalid_input(format!(
3290 "Declarative Workflow adapter does not yet support {kind} nodes"
3291 ))
3292 }
3293
3294 fn leaf_description(spec: &LeafSpec) -> String {
3295 let mut description = spec.prompt.trim().to_string();
3296 let mut metadata = Vec::new();
3297 metadata.push(format!("Workflow leaf id: {}", spec.id));
3298 metadata.push(format!("Mode: {}", task_mode_name(spec.mode)));
3299 if !spec.file_scope.is_empty() {
3300 metadata.push(format!("File scope: {}", spec.file_scope.join(", ")));
3301 }
3302 if !spec.depends_on_results.is_empty() {
3303 metadata.push(format!(
3304 "Depends on results: {}",
3305 spec.depends_on_results.join(", ")
3306 ));
3307 }
3308 if spec.budget != BudgetSpec::default() {
3309 let mut budget = Vec::new();
3310 if let Some(max_steps) = spec.budget.max_steps {
3311 budget.push(format!("max_steps={max_steps}"));
3312 }
3313 if let Some(timeout_secs) = spec.budget.timeout_secs {
3314 budget.push(format!("timeout_secs={timeout_secs}"));
3315 }
3316 if let Some(max_parallel) = spec.budget.max_parallel {
3317 budget.push(format!("max_parallel={max_parallel}"));
3318 }
3319 if let Some(max_tokens) = spec.budget.max_tokens {
3320 budget.push(format!("max_tokens={max_tokens}"));
3321 }
3322 if !budget.is_empty() {
3323 metadata.push(format!("Budget: {}", budget.join(", ")));
3324 }
3325 }
3326 if !metadata.is_empty() {
3327 description.push_str("\n\nWorkflow metadata:\n");
3328 for item in metadata {
3329 description.push_str("- ");
3330 description.push_str(&item);
3331 description.push('\n');
3332 }
3333 }
3334 description
3335 }
3336
3337 fn leaf_task_options_expression(
3338 spec: &LeafSpec,
3339 phase: Option<&str>,
3340 parallel: bool,
3341 ) -> Result<String, ToolError> {
3342 validate_leaf_runtime_contract(spec)?;
3343 let worktree = leaf_wants_worktree(spec, parallel);
3344 let write_authority = match spec.mode {
3345 TaskMode::ReadOnly => "read_only",
3346 TaskMode::ReadWrite if worktree => "worktree_write",
3347 TaskMode::ReadWrite => "workspace_write",
3348 };
3349 let write_roots = if spec.mode == TaskMode::ReadWrite {
3350 spec.file_scope
3351 .iter()
3352 .map(|scope| codewhale_workflow::normalize_file_scope_root(scope))
3353 .collect::<Vec<_>>()
3354 } else {
3355 Vec::new()
3356 };
3357 Ok(task_options_expression(
3358 leaf_description_expression(spec),
3359 leaf_subagent_type(spec),
3360 spec.role.as_deref(),
3361 spec.profile.as_deref(),
3362 spec.model_policy.model.as_deref(),
3363 // Parallel write-capable children default to worktree isolation (#4120).
3364 // Explicit isolation: shared is the approved same-worktree override.
3365 worktree,
3366 spec.budget.max_tokens,
3367 spec.budget.max_steps,
3368 spec.budget.timeout_secs,
3369 Some(write_authority),
3370 &write_roots,
3371 &spec.id,
3372 phase,
3373 leaf_allowed_tools(spec)?,
3374 spec.cwd.as_deref(),
3375 ))
3376 }
3377
3378 fn validate_leaf_runtime_contract(spec: &LeafSpec) -> Result<(), ToolError> {
3379 if spec.mode == TaskMode::ReadOnly && spec.permissions.allow_write {
3380 return Err(ToolError::invalid_input(format!(
3381 "Workflow leaf '{}' is read_only but requests allow_write permissions",
3382 spec.id
3383 )));
3384 }
3385 if spec.mode == TaskMode::ReadWrite && spec.file_scope.is_empty() {
3386 return Err(ToolError::invalid_input(format!(
3387 "Workflow leaf '{}' is read_write but declares no file_scope for its bounded write claim",
3388 spec.id
3389 )));
3390 }
3391 for scope in &spec.file_scope {
3392 let normalized = codewhale_workflow::normalize_file_scope_root(scope);
3393 if normalized.is_empty() || normalized.contains('*') {
3394 return Err(ToolError::invalid_input(format!(
3395 "Workflow leaf '{}' has unsupported file_scope '{}'; use a concrete path or a trailing /* or /** directory scope",
3396 spec.id, scope
3397 )));
3398 }
3399 }
3400 // A Fleet role and its authority posture are independent. In particular,
3401 // acceptance workflows must be able to resolve the `implementer` role to
3402 // its saved profile while narrowing that child to the read-only tool set.
3403 // `leaf_allowed_tools` enforces the mode below; rejecting the combination
3404 // made verification-only role/gate dogfood impossible.
3405 if spec.mode == TaskMode::ReadWrite
3406 && matches!(
3407 spec.agent_type,
3408 AgentType::Explore | AgentType::Plan | AgentType::Review | AgentType::Verifier
3409 )
3410 {
3411 return Err(ToolError::invalid_input(format!(
3412 "Workflow leaf '{}' is read_write but uses read-only agent_type {}",
3413 spec.id,
3414 agent_type_name(spec.agent_type)
3415 )));
3416 }
3417 if spec.mode == TaskMode::ReadOnly
3418 && spec
3419 .permissions
3420 .allowed_tools
3421 .iter()
3422 .any(|tool| is_write_or_shell_tool(tool))
3423 {
3424 return Err(ToolError::invalid_input(format!(
3425 "Workflow leaf '{}' is read_only but requests write/shell allowed_tools",
3426 spec.id
3427 )));
3428 }
3429 if spec.permissions.deny_all_tools && !spec.permissions.allowed_tools.is_empty() {
3430 return Err(ToolError::invalid_input(format!(
3431 "Workflow leaf '{}' cannot combine deny_all_tools with allowed_tools",
3432 spec.id
3433 )));
3434 }
3435 Ok(())
3436 }
3437
3438 fn leaf_description_expression(spec: &LeafSpec) -> String {
3439 let description = js_string(&leaf_description(spec));
3440 if spec.depends_on_results.is_empty() {
3441 return description;
3442 }
3443 let inputs = result_inputs_expression(&spec.depends_on_results);
3444 // parallel() retains null for a failed slot. A dependent worker must
3445 // not start with an empty stand-in for that result; reducers can still
3446 // inspect partial fan-out through their separate input projection.
3447 let required =
3448 serde_json::to_string(&spec.depends_on_results).expect("dependency IDs serialize to JSON");
3449 format!(
3450 "({required}.forEach(id => {{ if (__results[id] == null) throw new Error('Required Workflow result unavailable: ' + id); }}), {description} + \"\\n\\nInputs:\\n\" + {inputs})"
3451 )
3452 }
3453
3454 fn result_inputs_expression(inputs: &[String]) -> String {
3455 let entries = inputs
3456 .iter()
3457 .map(|input| format!("[{}, __results[{}]]", js_string(input), js_string(input)))
3458 .collect::<Vec<_>>()
3459 .join(", ");
3460 format!(
3461 "[{entries}].map(([id, value]) => \"--- \" + id + \" ---\\n\" + String(value ?? \"\")).join(\"\\n\\n\")"
3462 )
3463 }
3464
3465 fn leaf_subagent_type(spec: &LeafSpec) -> Option<&'static str> {
3466 // A named Fleet profile owns the child's runtime type. Emitting the IR's
3467 // default `general` here makes role-only leaves look like an explicit type
3468 // override and can conflict with the resolved roster member (for example,
3469 // scout -> explore). Preserve non-General types because those represent an
3470 // authored override and the spawn path must still validate compatibility.
3471 if (spec.role.is_some() || spec.profile.is_some()) && spec.agent_type == AgentType::General {
3472 return None;
3473 }
3474 if spec.mode == TaskMode::ReadOnly && spec.agent_type == AgentType::General {
3475 return Some("review");
3476 }
3477 // A read_only leaf must not *name* a write-capable type. `type` is a claim
3478 // about what the child can do, and claiming it while the leaf narrows the
3479 // child to read-only tools is the contradiction #5123 asks the spawn path
3480 // to reject. The leaf's role/profile already carries the identity roster
3481 // resolution needs, so drop the redundant type and let the role speak —
3482 // this is the `implementer` role narrowed to verification-only work that
3483 // `validate_leaf_runtime_contract` deliberately allows.
3484 if spec.mode == TaskMode::ReadOnly
3485 && spec.agent_type == AgentType::Implementer
3486 && (spec.role.is_some() || spec.profile.is_some())
3487 {
3488 return None;
3489 }
3490 Some(agent_type_name(spec.agent_type))
3491 }
3492
3493 fn leaf_allowed_tools(spec: &LeafSpec) -> Result<Option<Vec<String>>, ToolError> {
3494 if spec.permissions.deny_all_tools {
3495 return Ok(Some(Vec::new()));
3496 }
3497 if !spec.permissions.allowed_tools.is_empty() {
3498 return Ok(Some(spec.permissions.allowed_tools.clone()));
3499 }
3500 if spec.mode != TaskMode::ReadOnly {
3501 return Ok(None);
3502 }
3503 Ok(Some(
3504 read_only_allowed_tools(spec.agent_type)
3505 .iter()
3506 .map(|tool| (*tool).to_string())
3507 .collect(),
3508 ))
3509 }
3510
3511 fn read_only_allowed_tools(agent_type: AgentType) -> &'static [&'static str] {
3512 match agent_type {
3513 AgentType::Verifier => &["File"],
3514 _ => &["File"],
3515 }
3516 }
3517
3518 fn is_write_or_shell_tool(tool: &str) -> bool {
3519 // One list, owned by the workflow crate. This used to be a second copy
3520 // that drifted from `elevation.rs`'s — see `codewhale_workflow::is_write_tool`.
3521 codewhale_workflow::is_write_tool(tool) || codewhale_workflow::is_shell_tool(tool)
3522 }
3523
3524 // Pre-existing builder that grew `allowed_tools`; each arg maps 1:1 onto one
3525 // optional field of the generated JS options literal.
3526 #[allow(clippy::too_many_arguments)]
3527 fn task_options_expression(
3528 description_expr: String,
3529 subagent_type: Option<&str>,
3530 role: Option<&str>,
3531 profile: Option<&str>,
3532 model: Option<&str>,
3533 worktree: bool,
3534 token_budget: Option<u64>,
3535 max_steps: Option<u32>,
3536 wall_time_secs: Option<u64>,
3537 write_authority: Option<&str>,
3538 write_roots: &[String],
3539 label: &str,
3540 phase: Option<&str>,
3541 allowed_tools: Option<Vec<String>>,
3542 cwd: Option<&str>,
3543 ) -> String {
3544 let mut fields = vec![format!("description: {description_expr}")];
3545 if let Some(subagent_type) = subagent_type {
3546 fields.push(format!("type: {}", js_string(subagent_type)));
3547 }
3548 fields.push(format!("label: {}", js_string(label)));
3549 if let Some(phase) = phase {
3550 fields.push(format!("phase: {}", js_string(phase)));
3551 }
3552 if let Some(cwd) = cwd {
3553 fields.push(format!("cwd: {}", js_string(cwd)));
3554 }
3555 if let Some(role) = role {
3556 fields.push(format!("role: {}", js_string(role)));
3557 }
3558 if let Some(profile) = profile {
3559 fields.push(format!("profile: {}", js_string(profile)));
3560 }
3561 if let Some(model) = model {
3562 fields.push(format!("model: {}", js_string(model)));
3563 }
3564 if worktree {
3565 fields.push("worktree: true".to_string());
3566 }
3567 if let Some(token_budget) = token_budget {
3568 fields.push(format!("tokenBudget: {token_budget}"));
3569 }
3570 // Workflow's saved budget uses zero for no additional step cap. Omit
3571 // that sentinel at the Agent boundary, where an explicit cap must be
3572 // positive and may only narrow inherited authority.
3573 if let Some(max_steps) = max_steps.filter(|steps| *steps > 0) {
3574 fields.push(format!("maxSteps: {max_steps}"));
3575 }
3576 if let Some(wall_time_secs) = wall_time_secs {
3577 fields.push(format!("wallTimeSecs: {wall_time_secs}"));
3578 }
3579 if let Some(write_authority) = write_authority {
3580 fields.push(format!("writeAuthority: {}", js_string(write_authority)));
3581 }
3582 if !write_roots.is_empty() {
3583 fields.push(format!(
3584 "writeRoots: {}",
3585 serde_json::to_string(write_roots).expect("serializing write roots cannot fail")
3586 ));
3587 }
3588 if let Some(allowed_tools) = allowed_tools {
3589 fields.push(format!(
3590 "allowedTools: {}",
3591 serde_json::to_string(&allowed_tools).expect("serializing tool names cannot fail")
3592 ));
3593 }
3594 format!("{{ {} }}", fields.join(", "))
3595 }
3596
3597 fn js_string(value: &str) -> String {
3598 serde_json::to_string(value).expect("serializing JS string cannot fail")
3599 }
3600
3601 fn agent_type_name(agent_type: AgentType) -> &'static str {
3602 match agent_type {
3603 AgentType::General => "general",
3604 AgentType::Explore => "explore",
3605 AgentType::Plan => "plan",
3606 AgentType::Review => "review",
3607 AgentType::Implementer => "implementer",
3608 AgentType::Verifier => "verifier",
3609 }
3610 }
3611
3612 fn task_mode_name(mode: TaskMode) -> &'static str {
3613 match mode {
3614 TaskMode::ReadOnly => "read_only",
3615 TaskMode::ReadWrite => "read_write",
3616 }
3617 }
3618
3619 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
3620 enum ExplicitGateVerdict {
3621 Approve,
3622 Reject,
3623 }
3624
3625 /// Recognize only a standalone verdict token on the first non-empty line.
3626 ///
3627 /// This deliberately does not interpret prose, Markdown bullets, or verdict
3628 /// words later in an otherwise successful child response. Existing workflows
3629 /// whose children return ordinary prose therefore remain pass-on-success,
3630 /// while review-style children can fail closed with `BLOCK` or `FAIL`.
3631 fn explicit_gate_verdict(output: Option<&str>) -> Option<ExplicitGateVerdict> {
3632 let first_meaningful = output?
3633 .lines()
3634 .map(str::trim)
3635 .find(|line| !line.is_empty())?;
3636 if first_meaningful.eq_ignore_ascii_case("APPROVE")
3637 || first_meaningful.eq_ignore_ascii_case("PASS")
3638 {
3639 Some(ExplicitGateVerdict::Approve)
3640 } else if first_meaningful.eq_ignore_ascii_case("BLOCK")
3641 || first_meaningful.eq_ignore_ascii_case("FAIL")
3642 {
3643 Some(ExplicitGateVerdict::Reject)
3644 } else {
3645 None
3646 }
3647 }
3648
3649 fn has_gate_artifact_body(output: Option<&str>) -> bool {
3650 let Some(output) = output else {
3651 return false;
3652 };
3653 let mut meaningful_lines = output
3654 .lines()
3655 .map(str::trim)
3656 .filter(|line| !line.is_empty());
3657 // A declared artifact needs both a body label and at least one concrete
3658 // entry after the verdict. This keeps `APPROVE\nok` from promoting a
3659 // placeholder while remaining format-agnostic for arbitrary artifact kinds.
3660 meaningful_lines.next();
3661 meaningful_lines.next().is_some() && meaningful_lines.next().is_some()
3662 }
3663
3664 fn gate_outcome_for_completed_role(
3665 record: &RuntimeTaskRecord,
3666 require_explicit_verdict: bool,
3667 artifact_kind: Option<&str>,
3668 ) -> GateOutcome {
3669 match record.status {
3670 IrWorkflowRunStatus::Succeeded => match explicit_gate_verdict(record.output.as_deref()) {
3671 Some(ExplicitGateVerdict::Reject) => GateOutcome::Fail {
3672 reason: record
3673 .output
3674 .clone()
3675 .unwrap_or_else(|| "child returned an explicit rejection verdict".into()),
3676 },
3677 Some(ExplicitGateVerdict::Approve)
3678 if require_explicit_verdict
3679 && artifact_kind.is_some()
3680 && !has_gate_artifact_body(record.output.as_deref()) =>
3681 {
3682 GateOutcome::Fail {
3683 reason: format!(
3684 "task {} approved without the required {} artifact body",
3685 record.agent_id,
3686 artifact_kind.unwrap_or("gate")
3687 ),
3688 }
3689 }
3690 Some(ExplicitGateVerdict::Approve) => GateOutcome::Pass,
3691 None if require_explicit_verdict => GateOutcome::Fail {
3692 reason: format!(
3693 "task {} completed without the required first-line gate verdict; expected exactly APPROVE, PASS, BLOCK, or FAIL",
3694 record.agent_id
3695 ),
3696 },
3697 None => GateOutcome::Pass,
3698 },
3699 _ => GateOutcome::Fail {
3700 reason: record.output.clone().unwrap_or_else(|| {
3701 format!("task {} ended as {:?}", record.agent_id, record.status)
3702 }),
3703 },
3704 }
3705 }
3706
3707 #[derive(Debug, Clone)]
3708 struct RuntimeTaskRecord {
3709 agent_id: String,
3710 label: Option<String>,
3711 role: Option<String>,
3712 status: IrWorkflowRunStatus,
3713 output: Option<String>,
3714 schema_error: Option<String>,
3715 usage: Option<WorkflowTaskUsage>,
3716 }
3717
3718 impl RuntimeTaskRecord {
3719 /// Whether this record ended in a terminal state that produced no usable
3720 /// result — the states the slot ledger counts as a failed task.
3721 ///
3722 /// `BudgetExceeded` is the named gap this exists to close: a child that
3723 /// dies of budget exhaustion is a failed task for the all-failed rule,
3724 /// not an invisible one. `ReplayDiverged` means the leaf's replay did
3725 /// not reproduce its recorded result — no output either. `Cancelled` is
3726 /// deliberately excluded: it is the run's own stop, not lost work, and
3727 /// run-level cancellation is finalized before this ledger is consulted.
3728 fn failed_for_ledger(&self) -> bool {
3729 matches!(
3730 self.status,
3731 IrWorkflowRunStatus::Failed
3732 | IrWorkflowRunStatus::BudgetExceeded
3733 | IrWorkflowRunStatus::ReplayDiverged
3734 )
3735 }
3736 }
3737
3738 /// Bound on the workflow completion pump inbox (#6147): one completion per
3739 /// terminated workflow child, drained by the pump task.
3740 const WORKFLOW_COMPLETION_CHANNEL_CAPACITY: usize = 64;
3741
3742 struct SubAgentWorkflowDriver {
3743 run_id: String,
3744 owner_session_id: String,
3745 manager: SharedSubAgentManager,
3746 runtime: SubAgentRuntime,
3747 parent_cancel_token: CancellationToken,
3748 state: Arc<WorkflowWorkspaceState>,
3749 completion_tx: mpsc::Sender<SubAgentCompletion>,
3750 completion_state: Arc<Mutex<CompletionState>>,
3751 child_ids: Arc<Mutex<Vec<String>>>,
3752 /// Monotonic 0-based child admission counter for `workflow_child_index`.
3753 child_counter: AtomicU32,
3754 max_children: u32,
3755 /// Latest `phase(...)` title observed on this run (used when a task omits
3756 /// an explicit `phase` option).
3757 current_phase: Mutex<Option<String>>,
3758 task_records: Arc<Mutex<HashMap<String, RuntimeTaskRecord>>>,
3759 /// Fan-outs that resolved with every slot failed (`ProgressEvent::FanoutAllSlotsFailed`),
3760 /// script throws included. Consulted by the terminal slot ledger: those
3761 /// fan-outs can leave no task record at all.
3762 dead_fanouts: AtomicU32,
3763 dropped_slots: AtomicU32,
3764 total_budget: Option<u64>,
3765 last_budget_event: Arc<Mutex<Option<BudgetSnapshot>>>,
3766 /// Workflow-owned gates installed for this run (#4179).
3767 gate_specs: Arc<Vec<GateSpec>>,
3768 /// Lane-scoped gate and handoff state keyed by run id.
3769 gate_board: Arc<Mutex<LaneGateBoard>>,
3770 /// Caps concurrently live `task()` children for this run (product: 16).
3771 concurrent_gate: Arc<Semaphore>,
3772 /// Held permits for in-flight children; released on completion/cancel.
3773 spawn_permits: Mutex<HashMap<String, OwnedSemaphorePermit>>,
3774 /// Optional named Fleet roster for resolving Workflow task roles (#4177/#4178).
3775 fleet_name: Option<String>,
3776 /// The Fleet this Workflow is bound to, frozen at start. For an exact
3777 /// fleet this holds the immutable snapshot every task launch reads from,
3778 /// which is why editing `fleets/<name>.toml` mid-run cannot move a route.
3779 fleet: WorkflowFleetBinding,
3780 /// Workspace root, for durable `responseSchema` raw-output artifacts
3781 /// written beside the run report (#5583).
3782 workspace: PathBuf,
3783 }
3784
3785 /// A single-consumer handoff remains reserved during admission. If routing,
3786 /// spawning, or cancellation drops that admission, the same board retains the
3787 /// evidence for the next attempt.
3788 struct WorkflowHandoffReservation {
3789 board: Arc<Mutex<LaneGateBoard>>,
3790 artifacts: Vec<HandoffArtifact>,
3791 }
3792
3793 impl WorkflowHandoffReservation {
3794 fn commit(mut self) -> Vec<HandoffArtifact> {
3795 std::mem::take(&mut self.artifacts)
3796 }
3797 }
3798
3799 impl Drop for WorkflowHandoffReservation {
3800 fn drop(&mut self) {
3801 if self.artifacts.is_empty() {
3802 return;
3803 }
3804 let mut board = self
3805 .board
3806 .lock()
3807 .unwrap_or_else(|poison| poison.into_inner());
3808 // Consumption is newest-first; restore original chronological order.
3809 for artifact in self.artifacts.drain(..).rev() {
3810 board.artifacts.push(artifact);
3811 }
3812 }
3813 }
3814
3815 impl SubAgentWorkflowDriver {
3816 #[allow(clippy::too_many_arguments)]
3817 fn new(
3818 run_id: String,
3819 owner_session_id: String,
3820 manager: SharedSubAgentManager,
3821 mut runtime: SubAgentRuntime,
3822 state: Arc<WorkflowWorkspaceState>,
3823 total_budget: Option<u64>,
3824 fleet: WorkflowFleetBinding,
3825 gate_specs: Vec<GateSpec>,
3826 workspace: PathBuf,
3827 ) -> Arc<Self> {
3828 let fleet_name = fleet.name();
3829 let workflow_cfg = workflow_config_for(&runtime);
3830 let parent_cancel_token = runtime.cancel_token.clone();
3831 // Workflow cancellation owns only this token subtree, never the
3832 // caller's token or its unrelated direct children.
3833 runtime.cancel_token = runtime.cancel_token.child_token();
3834 runtime.context.cancel_token = Some(runtime.cancel_token.clone());
3835 let (completion_tx, completion_rx) = mpsc::channel(WORKFLOW_COMPLETION_CHANNEL_CAPACITY);
3836 let mut gate_board = LaneGateBoard::new(run_id.clone());
3837 gate_board.install_gates(&gate_specs);
3838 let driver = Arc::new(Self {
3839 run_id,
3840 owner_session_id,
3841 manager,
3842 runtime,
3843 parent_cancel_token,
3844 state,
3845 completion_tx,
3846 completion_state: Arc::new(Mutex::new(CompletionState::default())),
3847 child_ids: Arc::new(Mutex::new(Vec::new())),
3848 child_counter: AtomicU32::new(0),
3849 max_children: workflow_cfg.max_children.min(WORKFLOW_LIFETIME_CAP as u32),
3850 current_phase: Mutex::new(None),
3851 task_records: Arc::new(Mutex::new(HashMap::new())),
3852 dead_fanouts: AtomicU32::new(0),
3853 dropped_slots: AtomicU32::new(0),
3854 total_budget,
3855 last_budget_event: Arc::new(Mutex::new(None)),
3856 gate_specs: Arc::new(gate_specs),
3857 gate_board: Arc::new(Mutex::new(gate_board)),
3858 concurrent_gate: Arc::new(Semaphore::new(
3859 (workflow_cfg.max_concurrent as usize).min(WORKFLOW_MAX_CONCURRENT),
3860 )),
3861 spawn_permits: Mutex::new(HashMap::new()),
3862 fleet_name,
3863 fleet,
3864 workspace,
3865 });
3866 spawn_completion_pump(driver.clone(), completion_rx);
3867 driver
3868 }
3869
3870 fn force_cancel_all(&self) {
3871 self.concurrent_gate.close();
3872 self.runtime.cancel_token.cancel();
3873 let ids = self
3874 .child_ids
3875 .lock()
3876 .map(|ids| ids.clone())
3877 .unwrap_or_default();
3878 if let Ok(mut permits) = self.spawn_permits.lock() {
3879 permits.clear();
3880 }
3881 cancel_child_agents(self.manager.clone(), ids);
3882 if let Ok(mut state) = self.completion_state.lock() {
3883 for (_, waiter) in state.waiters.drain() {
3884 let _ = waiter.send(TaskCompletion::Cancelled);
3885 }
3886 }
3887 }
3888
3889 fn ensure_admission_open(&self) -> Result<(), DriverError> {
3890 if self.concurrent_gate.is_closed() || self.runtime.cancel_token.is_cancelled() {
3891 Err(DriverError::Rejected("workflow admission cancelled".into()))
3892 } else {
3893 Ok(())
3894 }
3895 }
3896
3897 fn finalize_running_tasks_cancelled(&self) {
3898 let ids = self
3899 .child_ids
3900 .lock()
3901 .map(|ids| ids.clone())
3902 .unwrap_or_default();
3903 for id in &ids {
3904 self.record_task_completion(id, &TaskCompletion::Cancelled, None);
3905 }
3906 }
3907
3908 fn record_child(&self, agent_id: &str) {
3909 if let Ok(mut ids) = self.child_ids.lock()
3910 && !ids.iter().any(|id| id == agent_id)
3911 {
3912 ids.push(agent_id.to_string());
3913 }
3914 if let Ok(mut runs) = self.state.runs.lock()
3915 && let Some(record) = runs.get_mut(&self.run_id)
3916 && !record.child_ids.iter().any(|id| id == agent_id)
3917 {
3918 record.child_ids.push(agent_id.to_string());
3919 }
3920 }
3921
3922 fn current_budget_snapshot(&self) -> BudgetSnapshot {
3923 // Token-budget enforcement was removed (#6189): nothing stops a run.
3924 // The snapshot survives for the declared spec ceiling only; spent is
3925 // no longer tracked per scope.
3926 BudgetSnapshot {
3927 total: self.total_budget,
3928 spent: 0,
3929 }
3930 }
3931
3932 /// Return the first authoritative gate failure after the VM has no more
3933 /// children to admit. Intermediate blocks already reject the downstream
3934 /// spawn; this final check gives a terminal role's BLOCK verdict the same
3935 /// fail-closed semantics.
3936 fn terminal_gate_failure(&self) -> Option<String> {
3937 let board = match self.gate_board.lock() {
3938 Ok(board) => board,
3939 Err(_) => {
3940 return Some(
3941 "workflow gate board was unavailable during terminal finalization".to_string(),
3942 );
3943 }
3944 };
3945 self.gate_specs.iter().find_map(|spec| {
3946 let state = board.gates.get(&spec.id).unwrap_or(&GateState::Pending);
3947 (!matches!(state, GateState::Passed)).then(|| {
3948 format!(
3949 "workflow gate `{}` ended {}: {}",
3950 spec.id,
3951 state.as_str(),
3952 gate_state_reason(state)
3953 )
3954 })
3955 })
3956 }
3957
3958 fn record_run_event(&self, event: WorkflowUiEvent) {
3959 let recorded = if let Ok(mut runs) = self.state.runs.lock()
3960 && let Some(record) = runs.get_mut(&self.run_id)
3961 {
3962 // A terminal run's event list is the cancel/complete receipt.
3963 // Racing VM and child completions must not append after that.
3964 if record.status != WorkflowRunStatus::Running {
3965 return;
3966 }
3967 record.push_event(event.clone());
3968 true
3969 } else {
3970 false
3971 };
3972 if recorded {
3973 self.state.record_event(&self.run_id, &event);
3974 }
3975 // #4122: stream typed events live into the panel + history card.
3976 self.emit_ui_event(&event);
3977 }
3978
3979 /// Publish a flattened WorkflowUiEvent on the engine event bus so the TUI
3980 /// can hydrate the panel while the tool is still running.
3981 fn emit_ui_event(&self, event: &WorkflowUiEvent) {
3982 let Some(tx) = self.runtime.event_tx.as_ref() else {
3983 return;
3984 };
3985 let Ok(mut value) = serde_json::to_value(event) else {
3986 return;
3987 };
3988 if let Some(obj) = value.as_object_mut() {
3989 obj.insert("run_id".to_string(), json!(self.run_id));
3990 }
3991 let _ = tx.try_send(Event::WorkflowUi {
3992 owner_session_id: self.owner_session_id.clone(),
3993 run_id: self.run_id.clone(),
3994 event: value,
3995 });
3996 }
3997
3998 fn record_budget_snapshot(&self, snapshot: BudgetSnapshot) {
3999 let changed = if let Ok(mut last) = self.last_budget_event.lock() {
4000 if last.as_ref() == Some(&snapshot) {
4001 false
4002 } else {
4003 *last = Some(snapshot);
4004 true
4005 }
4006 } else {
4007 false
4008 };
4009 let event = WorkflowUiEvent::new(&self.owner_session_id, budget_event_kind(snapshot));
4010 if changed {
4011 self.record_run_event(event);
4012 } else {
4013 // The VM polls the budget before it can admit its first child.
4014 // Keep that live path warm even when no token value changed, but
4015 // do not journal an unbounded stream of identical snapshots.
4016 self.emit_ui_event(&event);
4017 }
4018 }
4019
4020 fn prepare_request_for_gates(
4021 &self,
4022 request: &mut TaskRequest,
4023 ) -> Result<Vec<HandoffArtifact>, DriverError> {
4024 let Some(role) = request.role.as_deref().filter(|role| !role.is_empty()) else {
4025 return Ok(Vec::new());
4026 };
4027 if self.gate_specs.is_empty() {
4028 return Ok(Vec::new());
4029 }
4030
4031 let (blocked, handoffs) = {
4032 let mut board = self
4033 .gate_board
4034 .lock()
4035 .map_err(|_| DriverError::Rejected("workflow gate board lock poisoned".into()))?;
4036 let blocked = board.role_is_blocked(&self.gate_specs, role).cloned();
4037 // Handoffs are consumed (removed from the board) as they are
4038 // delivered — but only when the role is actually admitted. A
4039 // blocked task must leave them in place for the retry after the
4040 // gate clears.
4041 let handoffs = if blocked.is_none() {
4042 board.consume_handoffs_for(role, 4)
4043 } else {
4044 Vec::new()
4045 };
4046 (blocked, handoffs)
4047 };
4048
4049 if let Some(state) = blocked {
4050 return Err(DriverError::Rejected(format!(
4051 "workflow gate blocks role `{role}`: {}",
4052 gate_state_reason(&state)
4053 )));
4054 }
4055
4056 if !handoffs.is_empty() {
4057 append_handoff_context(request, &handoffs);
4058 }
4059 Ok(handoffs)
4060 }
4061
4062 fn update_gate_status(&self, status: Vec<GateStatusLine>) {
4063 let snapshot = if let Ok(mut runs) = self.state.runs.lock()
4064 && let Some(record) = runs.get_mut(&self.run_id)
4065 {
4066 record.gate_status = status;
4067 Some(record.clone())
4068 } else {
4069 None
4070 };
4071 if let Some(record) = snapshot {
4072 self.state.record_snapshot(&record);
4073 }
4074 }
4075
4076 fn evaluate_gates_for_completed_role(&self, record: &RuntimeTaskRecord) {
4077 let Some(role) = record.role.as_deref().filter(|role| !role.is_empty()) else {
4078 return;
4079 };
4080 if self.gate_specs.is_empty() {
4081 return;
4082 }
4083 let specs = self
4084 .gate_specs
4085 .iter()
4086 .filter(|spec| spec.on == GateOn::RoleComplete && spec.role.eq_ignore_ascii_case(role))
4087 .cloned()
4088 .collect::<Vec<_>>();
4089 if specs.is_empty() {
4090 return;
4091 }
4092
4093 let mut events = Vec::new();
4094 let mut next_status = Vec::new();
4095 if let Ok(mut board) = self.gate_board.lock() {
4096 for spec in specs {
4097 let outcome = gate_outcome_for_completed_role(
4098 record,
4099 spec.require_explicit_verdict,
4100 spec.artifact_kind.as_deref(),
4101 );
4102 let mut state = match board.evaluate(&spec, outcome.clone()) {
4103 Ok(state) => state,
4104 Err(err) => {
4105 let state = GateState::Blocked {
4106 reason: err.to_string(),
4107 };
4108 // Evaluation errors must become authoritative board state.
4109 // Otherwise the emitted receipt can say `blocked` while the
4110 // admission check still sees the gate as pending.
4111 board.gates.insert(spec.id.clone(), state.clone());
4112 state
4113 }
4114 };
4115 let mut promotion = None;
4116 if matches!(state, GateState::Passed)
4117 && let (Some(kind), Some(to_role)) =
4118 (spec.artifact_kind.as_deref(), spec.blocks_role.as_deref())
4119 {
4120 let artifact = HandoffArtifact {
4121 // Gate ids are authored input and are not guaranteed unique.
4122 // Use an opaque id so every promotion has a stable, distinct
4123 // identity even when a malformed workflow repeats a gate id.
4124 id: format!("handoff_{}", Uuid::new_v4()),
4125 lane_id: self.run_id.clone(),
4126 from_role: spec.role.clone(),
4127 to_role: to_role.to_string(),
4128 kind: kind.to_string(),
4129 payload: record.output.clone().unwrap_or_default(),
4130 created_at: now_ms().to_string(),
4131 };
4132 match board.record_handoff(artifact.clone()) {
4133 Ok(()) => {
4134 promotion = Some(WorkflowUiEvent::new(
4135 &self.owner_session_id,
4136 WorkflowUiEventKind::HandoffPromoted {
4137 artifact_id: artifact.id,
4138 gate_id: spec.id.clone(),
4139 kind: artifact.kind,
4140 from_role: artifact.from_role,
4141 to_role: artifact.to_role,
4142 producer_task_id: record.agent_id.clone(),
4143 },
4144 ));
4145 }
4146 Err(err) => {
4147 state = GateState::Blocked {
4148 reason: format!(
4149 "gate passed but its handoff could not be recorded: {err}"
4150 ),
4151 };
4152 board.gates.insert(spec.id.clone(), state.clone());
4153 }
4154 }
4155 }
4156 events.push(WorkflowUiEvent::new(
4157 &self.owner_session_id,
4158 WorkflowUiEventKind::GateUpdated {
4159 gate_id: spec.id.clone(),
4160 role: spec.role.clone(),
4161 gate: gate_kind_label(spec.gate).to_string(),
4162 state: state.as_str().to_string(),
4163 blocked_role: spec.blocks_role.clone(),
4164 blocked_reason: state.blocked_reason().map(str::to_string),
4165 },
4166 ));
4167 if let Some(event) = promotion {
4168 events.push(event);
4169 }
4170 }
4171 next_status = board.status_summary();
4172 }
4173 if !events.is_empty() || !next_status.is_empty() {
4174 self.update_gate_status(next_status);
4175 }
4176 for event in events {
4177 self.record_run_event(event);
4178 }
4179 }
4180
4181 fn record_task_started(
4182 &self,
4183 agent_id: &str,
4184 request: &TaskRequest,
4185 metadata: &WorkflowTaskSpawnMetadata,
4186 result: &crate::tools::subagent::SubAgentResult,
4187 fleet_receipt: Option<codewhale_workflow::FleetTaskReceipt>,
4188 ) {
4189 // Prefer typed spawn metadata over request fields so panel/history never
4190 // need to re-derive labels from the child prompt (#4119).
4191 let label = metadata
4192 .workflow_task_label
4193 .clone()
4194 .or_else(|| request.label.clone());
4195 self.record_run_event(WorkflowUiEvent::new(
4196 &self.owner_session_id,
4197 WorkflowUiEventKind::TaskStarted(Box::new(WorkflowTaskStartedEvent {
4198 task_id: agent_id.to_string(),
4199 label,
4200 role: request.role.as_deref().map(public_role_label),
4201 profile: request.profile.clone(),
4202 model: request.model.clone(),
4203 strength: request.model_strength.clone(),
4204 thinking: request.thinking.clone(),
4205 // #4039: both sides of the reasoning receipt come from the
4206 // spawn metadata the runtime minted, never from the request or
4207 // from current session config.
4208 requested_reasoning: metadata
4209 .requested_reasoning
4210 .clone()
4211 .or_else(|| request.thinking.clone()),
4212 effective_reasoning: metadata.effective_reasoning.clone(),
4213 // Prefer spawn metadata (fleet-resolved); fall back to request.
4214 //
4215 // An exact-Fleet receipt overrides both, because the spawn
4216 // metadata's role is the roster profile's **Runtime posture** —
4217 // the closed role policy selected after member identity — and displaying
4218 // that where the member's role belongs silently renames the
4219 // operator's `auditor` to `scout`. The posture is not lost: it
4220 // rides the receipt as its own field.
4221 resolved_role: displayed_resolved_role(
4222 fleet_receipt.as_ref(),
4223 metadata.resolved_role.as_deref(),
4224 request.role.as_deref(),
4225 ),
4226 resolved_profile: metadata
4227 .resolved_profile
4228 .clone()
4229 .or_else(|| request.profile.clone()),
4230 resolved_provider: metadata.resolved_provider.clone(),
4231 resolved_model: metadata.resolved_model.clone(),
4232 route_source: metadata.route_source.clone(),
4233 child_route: Some(metadata.child_route.clone()),
4234 worktree: request.worktree,
4235 workspace: result.workspace.clone(),
4236 git_branch: result.git_branch.clone(),
4237 parent_task_id: metadata.parent_task_id.clone(),
4238 depth: metadata.depth,
4239 workflow_run_id: metadata.workflow_run_id.clone(),
4240 workflow_phase_id: metadata.workflow_phase_id.clone(),
4241 workflow_task_label: metadata.workflow_task_label.clone(),
4242 workflow_child_index: metadata.workflow_child_index,
4243 fleet_receipt: fleet_receipt.clone(),
4244 })),
4245 ));
4246 // Also surface the decision as a run log line, so the receipt is
4247 // *visible* in the panel and transcript rather than only structured on
4248 // an event a UI has to know to unpack.
4249 if let Some(receipt) = fleet_receipt {
4250 self.record_run_event(WorkflowUiEvent::new(
4251 &self.owner_session_id,
4252 WorkflowUiEventKind::Log {
4253 message: format!("Fleet route {}", receipt.line()),
4254 },
4255 ));
4256 }
4257 }
4258
4259 /// Preserve a routing receipt whose task never became a child.
4260 ///
4261 /// A receipt normally rides the `task_started` event, which a failed spawn
4262 /// never emits. Recording it here keeps the run's history complete: the
4263 /// decision happened, the tokens were spent, and — if the Router ran on
4264 /// another provider — a bounded summary already left the host. Silence
4265 /// would make all three unrecoverable.
4266 fn record_orphaned_fleet_receipt(
4267 &self,
4268 receipt: &codewhale_workflow::FleetTaskReceipt,
4269 error: &str,
4270 ) {
4271 self.record_run_event(WorkflowUiEvent::new(
4272 &self.owner_session_id,
4273 WorkflowUiEventKind::Log {
4274 message: orphaned_fleet_receipt_line(receipt, error),
4275 },
4276 ));
4277 }
4278
4279 fn record_task_request(&self, agent_id: &str, request: &TaskRequest) {
4280 if let Ok(mut records) = self.task_records.lock() {
4281 records
4282 .entry(agent_id.to_string())
4283 .or_insert_with(|| RuntimeTaskRecord {
4284 agent_id: agent_id.to_string(),
4285 label: request.label.clone(),
4286 role: request.role.clone(),
4287 status: IrWorkflowRunStatus::Running,
4288 output: None,
4289 schema_error: None,
4290 usage: None,
4291 });
4292 }
4293 let pending_completion = self
4294 .completion_state
4295 .lock()
4296 .ok()
4297 .and_then(|state| state.pending.get(agent_id).cloned());
4298 if let Some(completion) = pending_completion {
4299 self.record_task_completion(agent_id, &completion.completion, completion.usage);
4300 } else if self.runtime.cancel_token.is_cancelled() {
4301 self.record_task_completion(agent_id, &TaskCompletion::Cancelled, None);
4302 }
4303 }
4304
4305 fn record_task_completion(
4306 &self,
4307 agent_id: &str,
4308 completion: &TaskCompletion,
4309 usage: Option<WorkflowTaskUsage>,
4310 ) {
4311 let mut terminal_event = None;
4312 let mut completed_record = None;
4313 if let Ok(mut records) = self.task_records.lock()
4314 && let Some(record) = records.get_mut(agent_id)
4315 {
4316 if usage.is_some() {
4317 record.usage = usage;
4318 }
4319 if record.status == IrWorkflowRunStatus::Running {
4320 let (status, output) = task_completion_status(completion);
4321 record.status = status;
4322 record.output = output;
4323 terminal_event = Some(WorkflowUiEvent::new(
4324 &self.owner_session_id,
4325 WorkflowUiEventKind::TaskCompleted {
4326 task_id: agent_id.to_string(),
4327 status,
4328 usage: record.usage.clone(),
4329 },
4330 ));
4331 completed_record = Some(record.clone());
4332 }
4333 }
4334 if let Some(event) = terminal_event {
4335 self.record_run_event(event);
4336 }
4337 if let Some(record) = completed_record.as_ref() {
4338 // A role-complete gate is caused by this terminal transition, so its
4339 // durable task receipt must precede gate evaluation and promotion.
4340 self.evaluate_gates_for_completed_role(record);
4341 }
4342 }
4343
4344 /// A rejected dispatch never produces a child agent, and inside
4345 /// `parallel()` the JS throw collapses to a `null` slot; this ledger keeps
4346 /// the rejection visible on the run record and result payload (#5035).
4347 fn record_dispatch_failure(
4348 &self,
4349 label: Option<String>,
4350 phase: Option<String>,
4351 message: String,
4352 ) {
4353 let failure = WorkflowDispatchFailure {
4354 at_ms: now_ms(),
4355 label,
4356 phase,
4357 message,
4358 };
4359 let slot = failure
4360 .label
4361 .as_deref()
4362 .or(failure.phase.as_deref())
4363 .unwrap_or("task");
4364 let progress_line = format!("dispatch failed for {slot}: {}", failure.message);
4365 let ui_event = WorkflowUiEvent::new(
4366 &self.owner_session_id,
4367 WorkflowUiEventKind::TaskDispatchFailed {
4368 label: failure.label.clone(),
4369 phase: failure.phase.clone(),
4370 message: failure.message.clone(),
4371 },
4372 );
4373 if let Ok(mut runs) = self.state.runs.lock()
4374 && let Some(record) = runs.get_mut(&self.run_id)
4375 {
4376 if record.status != WorkflowRunStatus::Running {
4377 return;
4378 }
4379 record.push_progress(progress_line.clone());
4380 record.push_event(ui_event.clone());
4381 record.push_dispatch_failure(failure);
4382 }
4383 self.state.record_progress(&self.run_id, &progress_line);
4384 self.state.record_event(&self.run_id, &ui_event);
4385 self.emit_ui_event(&ui_event);
4386 }
4387
4388 fn record_schema_validation_failure(&self, agent_id: &str, message: String) {
4389 if let Ok(mut records) = self.task_records.lock()
4390 && let Some(record) = records.get_mut(agent_id)
4391 {
4392 record.status = IrWorkflowRunStatus::Failed;
4393 record.schema_error = Some(message.clone());
4394 record.output = Some(message);
4395 }
4396 }
4397
4398 /// Bounded preview + durable artifact for a failed `responseSchema`
4399 /// attempt's raw reply (#5583). The record keeps a preview; the full
4400 /// text goes to a `.txt` artifact beside the run report when it is
4401 /// larger than the preview, so the journal stays small while the
4402 /// evidence stays durable.
4403 fn schema_raw_receipt(
4404 &self,
4405 task_id: &str,
4406 attempt: u32,
4407 raw: &str,
4408 ) -> (String, Option<String>) {
4409 let preview = bounded_raw_preview(raw);
4410 let artifact = if raw.chars().count() > SCHEMA_RAW_PREVIEW_CHARS {
4411 write_schema_raw_artifact(&self.workspace, &self.run_id, task_id, attempt, raw)
4412 } else {
4413 None
4414 };
4415 (preview, artifact)
4416 }
4417
4418 fn task_records_snapshot(&self) -> Vec<RuntimeTaskRecord> {
4419 self.task_records
4420 .lock()
4421 .map(|records| records.values().cloned().collect())
4422 .unwrap_or_default()
4423 }
4424
4425 /// Count a fan-out that resolved with every slot failed. The prelude's
4426 /// run-log breadcrumb already names it for operators; this counter is
4427 /// what lets the terminal slot ledger act on it.
4428 fn record_dead_fanout(&self) {
4429 self.dead_fanouts.fetch_add(1, Ordering::SeqCst);
4430 }
4431
4432 fn dead_fanout_count(&self) -> u32 {
4433 self.dead_fanouts.load(Ordering::SeqCst)
4434 }
4435
4436 /// Count one slot a settled fan-out dropped while others survived; the
4437 /// per-slot breadcrumb already narrates it, this counter makes the loss
4438 /// visible to the terminal slot ledger.
4439 fn record_dropped_slot(&self) {
4440 self.dropped_slots.fetch_add(1, Ordering::SeqCst);
4441 }
4442
4443 fn dropped_slot_count(&self) -> u32 {
4444 self.dropped_slots.load(Ordering::SeqCst)
4445 }
4446
4447 fn add_waiter_or_complete(&self, agent_id: String, waiter: oneshot::Sender<TaskCompletion>) {
4448 let mut state = self
4449 .completion_state
4450 .lock()
4451 .unwrap_or_else(|poison| poison.into_inner());
4452 if let Some(completion) = state.pending.remove(&agent_id) {
4453 drop(state);
4454 // The child may have completed before its permit was registered.
4455 // Consume that receipt and release the late-installed permit too.
4456 if let Ok(mut permits) = self.spawn_permits.lock() {
4457 permits.remove(&agent_id);
4458 }
4459 // Its receipt may also have arrived just after record_task_request
4460 // checked pending state. Replaying here closes that same race for
4461 // the durable task status; terminal transitions are idempotent.
4462 self.record_task_completion(&agent_id, &completion.completion, completion.usage);
4463 let _ = waiter.send(completion.completion);
4464 } else if self.runtime.cancel_token.is_cancelled() {
4465 if let Ok(mut permits) = self.spawn_permits.lock() {
4466 permits.remove(&agent_id);
4467 }
4468 let _ = waiter.send(TaskCompletion::Cancelled);
4469 } else {
4470 state.waiters.insert(agent_id, waiter);
4471 }
4472 }
4473
4474 fn deliver_completion(
4475 &self,
4476 agent_id: String,
4477 completion: TaskCompletion,
4478 usage: Option<WorkflowTaskUsage>,
4479 ) {
4480 self.record_task_completion(&agent_id, &completion, usage.clone());
4481 if let Ok(mut permits) = self.spawn_permits.lock() {
4482 permits.remove(&agent_id);
4483 }
4484 let mut state = self
4485 .completion_state
4486 .lock()
4487 .unwrap_or_else(|poison| poison.into_inner());
4488 if let Some(waiter) = state.waiters.remove(&agent_id) {
4489 let _ = waiter.send(completion);
4490 } else {
4491 state
4492 .pending
4493 .insert(agent_id, PendingCompletion { completion, usage });
4494 }
4495 }
4496 }
4497
4498 #[derive(Clone)]
4499 struct PendingCompletion {
4500 completion: TaskCompletion,
4501 usage: Option<WorkflowTaskUsage>,
4502 }
4503
4504 #[derive(Default)]
4505 struct CompletionState {
4506 waiters: HashMap<String, oneshot::Sender<TaskCompletion>>,
4507 pending: HashMap<String, PendingCompletion>,
4508 }
4509
4510 impl SubAgentWorkflowDriver {
4511 /// The admission half of [`WorkflowDriver::spawn_task`]; every `Err` it
4512 /// returns is recorded as a dispatch failure by the trait wrapper.
4513 async fn spawn_task_admitted(
4514 &self,
4515 mut request: TaskRequest,
4516 ) -> Result<SpawnedTask, DriverError> {
4517 self.ensure_admission_open()?;
4518 // Exact fleets resolve from the frozen snapshot; legacy role maps keep
4519 // their previous path unchanged.
4520 //
4521 // The exact path is deliberately split in two. **Binding** resolves the
4522 // member, its frozen route, and its clamped authority, and contacts
4523 // nobody. **Routing** — the half that may call the fleet's reasoning
4524 // router, spend the operator's tokens, and disclose a bounded summary
4525 // to another provider — happens only after this task has passed its
4526 // gates and holds a concurrency slot. A task that is rejected or
4527 // capacity-blocked therefore costs nothing and reveals nothing.
4528 let exact_binding = if let Some(operation) = self.fleet.exact() {
4529 // The depth budget is the other failure the spawn boundary can be
4530 // predicted to raise, and it does not depend on the member. Check
4531 // it here so an over-deep task is refused for free rather than
4532 // after a routing request has already been paid for.
4533 if self.runtime.would_exceed_depth() {
4534 return Err(DriverError::Rejected(format!(
4535 "fleet `{}`: sub-agent depth limit reached (depth {}, max {}); this task \
4536 cannot spawn a child, so it is refused before the reasoning router is asked \
4537 anything.",
4538 operation.snapshot().fleet().qualified(),
4539 self.runtime.spawn_depth,
4540 self.runtime.max_spawn_depth,
4541 )));
4542 }
4543 Some(bind_exact_fleet_task_request(
4544 operation,
4545 crate::tools::subagent::session_permission_ceiling(&self.runtime),
4546 &mut request,
4547 )?)
4548 } else {
4549 apply_named_fleet_to_task_request(self.fleet.legacy_roles(), &mut request).map_err(
4550 |err| {
4551 if let Some(fleet) = self.fleet_name.as_deref() {
4552 DriverError::Rejected(format!(
4553 "Fleet `{fleet}` role resolution failed: {err}"
4554 ))
4555 } else {
4556 err
4557 }
4558 },
4559 )?;
4560 None
4561 };
4562 // Wait for a concurrent slot (max 16 live children per run).
4563 let permit = self
4564 .concurrent_gate
4565 .clone()
4566 .acquire_owned()
4567 .await
4568 .map_err(|_| DriverError::Rejected("workflow concurrent admission closed".into()))?;
4569 self.ensure_admission_open()?;
4570 let workflow_child_index = self
4571 .child_counter
4572 .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |count| {
4573 (count < self.max_children).then_some(count + 1)
4574 })
4575 .map_err(|_| DriverError::Rejected(format!(
4576 "workflow.max_children limit ({}) reached; reduce remaining work or start a new reviewed plan",
4577 self.max_children
4578 )))?;
4579 let handoffs = WorkflowHandoffReservation {
4580 board: self.gate_board.clone(),
4581 artifacts: self.prepare_request_for_gates(&mut request)?,
4582 };
4583
4584 // Admitted. Only now may the reasoning router be consulted.
4585 let fleet_receipt = match (self.fleet.exact(), exact_binding.as_ref()) {
4586 (Some(operation), Some(binding)) => {
4587 match route_admitted_exact_task(operation, binding, &mut request).await {
4588 Ok(receipt) => Some(receipt),
4589 Err(err) => {
4590 drop(permit);
4591 return Err(err);
4592 }
4593 }
4594 }
4595 _ => None,
4596 };
4597 if let Err(err) = self.ensure_admission_open() {
4598 if let Some(receipt) = &fleet_receipt {
4599 self.record_orphaned_fleet_receipt(receipt, &err.to_string());
4600 }
4601 return Err(err);
4602 }
4603
4604 let runtime = self
4605 .runtime
4606 .clone()
4607 .with_parent_completion_tx(self.completion_tx.clone());
4608 let request_record = request.clone();
4609 let workflow_phase_id = request
4610 .phase
4611 .as_ref()
4612 .map(|phase| phase.trim())
4613 .filter(|phase| !phase.is_empty())
4614 .map(str::to_string)
4615 .or_else(|| {
4616 self.current_phase
4617 .lock()
4618 .ok()
4619 .and_then(|phase| phase.clone())
4620 });
4621 let workflow_task_label = request
4622 .label
4623 .as_ref()
4624 .map(|label| label.trim())
4625 .filter(|label| !label.is_empty())
4626 .map(str::to_string);
4627 let identity = WorkflowTaskSpawnIdentity {
4628 workflow_run_id: self.run_id.clone(),
4629 workflow_phase_id,
4630 workflow_task_label,
4631 workflow_child_index,
4632 exact_fleet_binding: exact_binding.clone(),
4633 // The Fleet decision travels to the spawn boundary as a value that
4634 // boundary re-checks, rather than as trust in the caller.
4635 fleet_authority_fingerprint: fleet_receipt
4636 .as_ref()
4637 .and_then(|receipt| receipt.authority_fingerprint.clone()),
4638 };
4639 let result =
4640 match spawn_workflow_task(request, self.manager.clone(), runtime, identity).await {
4641 Ok(result) => result,
4642 Err(err) => {
4643 drop(permit);
4644 // The Router decision was already made and already paid
4645 // for. Dropping the receipt with the failed spawn would
4646 // erase the only record that a routing request was spent —
4647 // and, when a bounded summary crossed to another provider,
4648 // the only disclosure that it did. It survives the failure.
4649 if let Some(receipt) = fleet_receipt {
4650 self.record_orphaned_fleet_receipt(&receipt, &err.to_string());
4651 }
4652 return Err(DriverError::Rejected(err.to_string()));
4653 }
4654 };
4655 let task_id = result.result.agent_id.clone();
4656 if let Err(err) = self.ensure_admission_open() {
4657 let _ = self.manager.write().await.cancel_agent(&task_id);
4658 if let Some(receipt) = &fleet_receipt {
4659 self.record_orphaned_fleet_receipt(receipt, &err.to_string());
4660 }
4661 return Err(err);
4662 }
4663 if let Ok(mut permits) = self.spawn_permits.lock() {
4664 permits.insert(task_id.clone(), permit);
4665 }
4666 self.record_child(&task_id);
4667 self.record_task_started(
4668 &task_id,
4669 &request_record,
4670 &result.metadata,
4671 &result.result,
4672 fleet_receipt,
4673 );
4674 for artifact in handoffs.commit() {
4675 self.record_run_event(WorkflowUiEvent::new(
4676 &self.owner_session_id,
4677 WorkflowUiEventKind::HandoffConsumed {
4678 artifact_id: artifact.id,
4679 kind: artifact.kind,
4680 from_role: artifact.from_role,
4681 to_role: artifact.to_role,
4682 consumer_task_id: task_id.clone(),
4683 },
4684 ));
4685 }
4686 self.record_task_request(&task_id, &request_record);
4687 let (tx, rx) = oneshot::channel();
4688 self.add_waiter_or_complete(task_id.clone(), tx);
4689 Ok(SpawnedTask {
4690 task_id,
4691 completion: rx,
4692 })
4693 }
4694 }
4695
4696 #[async_trait]
4697 impl WorkflowDriver for SubAgentWorkflowDriver {
4698 async fn spawn_task(&self, request: TaskRequest) -> Result<SpawnedTask, DriverError> {
4699 let label = request
4700 .label
4701 .as_deref()
4702 .map(str::trim)
4703 .filter(|label| !label.is_empty())
4704 .map(str::to_string);
4705 let phase = request
4706 .phase
4707 .as_deref()
4708 .map(str::trim)
4709 .filter(|phase| !phase.is_empty())
4710 .map(str::to_string)
4711 .or_else(|| {
4712 self.current_phase
4713 .lock()
4714 .ok()
4715 .and_then(|phase| phase.clone())
4716 });
4717 let result = self.spawn_task_admitted(request).await;
4718 if let Err(err) = &result {
4719 self.record_dispatch_failure(label, phase, err.to_string());
4720 }
4721 result
4722 }
4723
4724 fn cancel_all(&self) {
4725 self.force_cancel_all();
4726 }
4727
4728 fn budget(&self) -> BudgetSnapshot {
4729 let snapshot = self.current_budget_snapshot();
4730 self.record_budget_snapshot(snapshot);
4731 snapshot
4732 }
4733
4734 fn progress(&self, event: ProgressEvent) {
4735 let mut schema_error = None;
4736 let mut schema_repair = None;
4737 let (message, ui_event) = match event {
4738 // Pre-spawn rejections share the dispatch-failure ledger so the
4739 // completion classifier sees every requested slot, whether the VM
4740 // or the driver refused it.
4741 ProgressEvent::TaskRejected {
4742 label,
4743 phase,
4744 message,
4745 } => {
4746 self.record_dispatch_failure(label, phase, message);
4747 return;
4748 }
4749 // R9: a dead fan-out is a status signal, not a narrated line —
4750 // the prelude already logged the operator breadcrumb. Counted on
4751 // the driver so the terminal slot ledger can refuse to record a
4752 // run where nothing survived as a plain success.
4753 ProgressEvent::FanoutAllSlotsFailed { .. } => {
4754 self.record_dead_fanout();
4755 return;
4756 }
4757 ProgressEvent::FanoutSlotDropped { .. } => {
4758 self.record_dropped_slot();
4759 return;
4760 }
4761 ProgressEvent::Log { message } => (
4762 format!("log: {message}"),
4763 WorkflowUiEvent::new(&self.owner_session_id, WorkflowUiEventKind::Log { message }),
4764 ),
4765 ProgressEvent::Phase { title } => {
4766 if let Ok(mut current) = self.current_phase.lock() {
4767 *current = Some(title.clone());
4768 }
4769 (
4770 format!("phase: {title}"),
4771 WorkflowUiEvent::new(
4772 &self.owner_session_id,
4773 WorkflowUiEventKind::PhaseStarted { title },
4774 ),
4775 )
4776 }
4777 ProgressEvent::TaskSchemaValidationFailed {
4778 task_id,
4779 kind,
4780 attempt,
4781 message,
4782 raw,
4783 raw_truncated,
4784 } => {
4785 self.record_schema_validation_failure(&task_id, message.clone());
4786 let (raw_preview, artifact) = self.schema_raw_receipt(&task_id, attempt, &raw);
4787 schema_error = Some(WorkflowSchemaError {
4788 task_id: task_id.clone(),
4789 message: message.clone(),
4790 kind,
4791 attempt,
4792 raw_preview,
4793 raw_truncated,
4794 artifact,
4795 });
4796 (
4797 format!(
4798 "schema validation failed for {task_id} (attempt {attempt}): {message}"
4799 ),
4800 WorkflowUiEvent::new(
4801 &self.owner_session_id,
4802 WorkflowUiEventKind::TaskSchemaValidationFailed { task_id, message },
4803 ),
4804 )
4805 }
4806 // #5583: a failed decode with a repair still to come. Receipted
4807 // on the schema-repair ledger (visible even when the repair
4808 // succeeds); the live surfaces see it as a progress line so
4809 // operators know a bounded re-ask is happening.
4810 ProgressEvent::TaskSchemaRepairAttempted {
4811 task_id,
4812 kind,
4813 attempt,
4814 message,
4815 raw,
4816 raw_truncated,
4817 } => {
4818 let (raw_preview, artifact) = self.schema_raw_receipt(&task_id, attempt, &raw);
4819 schema_repair = Some(WorkflowSchemaRepairAttempt {
4820 task_id: task_id.clone(),
4821 kind,
4822 attempt,
4823 message: message.clone(),
4824 raw_preview,
4825 raw_truncated,
4826 artifact,
4827 });
4828 let note = format!(
4829 "schema decode failed for {task_id} (attempt {attempt}): {message}; \
4830 dispatching a bounded repair"
4831 );
4832 (
4833 format!("log: {note}"),
4834 WorkflowUiEvent::new(
4835 &self.owner_session_id,
4836 WorkflowUiEventKind::Log { message: note },
4837 ),
4838 )
4839 }
4840 };
4841 if let Ok(mut runs) = self.state.runs.lock()
4842 && let Some(record) = runs.get_mut(&self.run_id)
4843 {
4844 if record.status != WorkflowRunStatus::Running {
4845 return;
4846 }
4847 record.push_progress(message.clone());
4848 record.push_event(ui_event.clone());
4849 if let Some(schema_error) = schema_error {
4850 record.schema_errors.push(schema_error);
4851 }
4852 if let Some(schema_repair) = schema_repair {
4853 record.schema_repair_count = record.schema_repair_count.saturating_add(1);
4854 record.schema_repairs.push(schema_repair);
4855 }
4856 }
4857 self.state.record_progress(&self.run_id, &message);
4858 self.state.record_event(&self.run_id, &ui_event);
4859 // #4122: phase/schema/log progress streams into the live panel path.
4860 self.emit_ui_event(&ui_event);
4861 }
4862 }
4863
4864 fn budget_event_kind(snapshot: BudgetSnapshot) -> WorkflowUiEventKind {
4865 WorkflowUiEventKind::BudgetUpdated {
4866 total: snapshot.total,
4867 spent: snapshot.spent,
4868 remaining: snapshot.remaining(),
4869 }
4870 }
4871
4872 fn gate_kind_label(kind: GateKind) -> &'static str {
4873 match kind {
4874 GateKind::Verify => "verify",
4875 GateKind::Review => "review",
4876 GateKind::Approve => "approve",
4877 }
4878 }
4879
4880 fn gate_state_reason(state: &GateState) -> String {
4881 state
4882 .blocked_reason()
4883 .map(str::to_string)
4884 .unwrap_or_else(|| state.as_str().to_string())
4885 }
4886
4887 fn append_handoff_context(request: &mut TaskRequest, handoffs: &[HandoffArtifact]) {
4888 request
4889 .description
4890 .push_str("\n\nWorkflow handoff artifacts available for this role:\n");
4891 for artifact in handoffs {
4892 request.description.push_str(&format!(
4893 "- id: {} kind: {} from: {} to: {}\n payload: {}\n",
4894 artifact.id,
4895 artifact.kind,
4896 artifact.from_role,
4897 artifact.to_role,
4898 compact_handoff_payload(&artifact.payload, WORKFLOW_HANDOFF_MAX_CHARS)
4899 ));
4900 }
4901 }
4902
4903 fn compact_handoff_payload(payload: &str, max_chars: usize) -> String {
4904 let trimmed = payload.trim();
4905 if trimmed.chars().count() <= max_chars {
4906 return trimmed.to_string();
4907 }
4908 let mut out = trimmed.chars().take(max_chars).collect::<String>();
4909 out.push_str("...");
4910 out
4911 }
4912
4913 fn task_completion_status(completion: &TaskCompletion) -> (IrWorkflowRunStatus, Option<String>) {
4914 match completion {
4915 TaskCompletion::Completed { text } => (IrWorkflowRunStatus::Succeeded, Some(text.clone())),
4916 TaskCompletion::Failed { message } => (IrWorkflowRunStatus::Failed, Some(message.clone())),
4917 TaskCompletion::Cancelled => (IrWorkflowRunStatus::Cancelled, None),
4918 TaskCompletion::BudgetExhausted { message } => {
4919 (IrWorkflowRunStatus::BudgetExceeded, Some(message.clone()))
4920 }
4921 }
4922 }
4923
4924 /// Sum per-task telemetry into run-wide totals for `run_completed` (#2974).
4925 /// Returns `None` when no task contributed telemetry (e.g. a plan that ran
4926 /// zero children) so the event stays byte-identical to its pre-#2974 shape.
4927 fn run_usage_totals(records: &[RuntimeTaskRecord]) -> Option<WorkflowRunUsage> {
4928 let mut usages = records.iter().filter_map(|record| record.usage.as_ref());
4929 let mut totals = WorkflowRunUsage::from_task(usages.next()?);
4930 for usage in usages {
4931 totals.add_task(usage);
4932 }
4933 Some(totals)
4934 }
4935
4936 /// Convert captured task telemetry into the shared `WorkflowUsage` aggregate
4937 /// used by the workflow execution record (#2974).
4938 fn workflow_usage_from_task(usage: &WorkflowTaskUsage) -> WorkflowUsage {
4939 WorkflowUsage {
4940 input_tokens: usage.input_tokens,
4941 output_tokens: usage.output_tokens,
4942 cost_microusd: usage.cost_microusd,
4943 }
4944 }
4945
4946 fn execution_from_declarative_spec(
4947 spec: &WorkflowSpec,
4948 records: Vec<RuntimeTaskRecord>,
4949 terminal_status: WorkflowRunStatus,
4950 ) -> IrWorkflowExecution {
4951 let by_label = records
4952 .into_iter()
4953 .filter_map(|record| record.label.clone().map(|label| (label, record)))
4954 .collect::<HashMap<_, _>>();
4955 let mut execution = IrWorkflowExecution::default();
4956 for node in &spec.nodes {
4957 push_execution_node(node, &by_label, &mut execution);
4958 }
4959 let mut leaf_usage = execution.leaf_results.iter().map(|leaf| leaf.usage);
4960 execution.usage = leaf_usage
4961 .next()
4962 .map_or_else(WorkflowUsage::default, |first| {
4963 leaf_usage.fold(first, |mut totals, usage| {
4964 totals.input_tokens = sum_optional_usage(totals.input_tokens, usage.input_tokens);
4965 totals.output_tokens =
4966 sum_optional_usage(totals.output_tokens, usage.output_tokens);
4967 totals.cost_microusd =
4968 sum_optional_usage(totals.cost_microusd, usage.cost_microusd);
4969 totals
4970 })
4971 });
4972 match terminal_status {
4973 WorkflowRunStatus::Completed | WorkflowRunStatus::Degraded => {}
4974 WorkflowRunStatus::Failed => mark_ir_status(&mut execution, IrWorkflowRunStatus::Failed),
4975 WorkflowRunStatus::Cancelled => {
4976 mark_ir_status(&mut execution, IrWorkflowRunStatus::Cancelled);
4977 }
4978 WorkflowRunStatus::Running => {
4979 execution.status = IrWorkflowRunStatus::Running;
4980 }
4981 }
4982 execution
4983 }
4984
4985 fn push_execution_node(
4986 node: &WorkflowNode,
4987 records: &HashMap<String, RuntimeTaskRecord>,
4988 execution: &mut IrWorkflowExecution,
4989 ) {
4990 match node {
4991 WorkflowNode::Leaf(spec) => push_leaf_execution(spec, records, execution),
4992 WorkflowNode::BranchSet(spec) => push_branch_execution(spec, records, execution),
4993 WorkflowNode::Sequence(spec) => push_sequence_execution(spec, records, execution),
4994 WorkflowNode::Reduce(spec) => push_control_execution(
4995 spec.id.as_str(),
4996 ControlNodeKind::Reduce,
4997 records.get(&spec.id),
4998 spec.inputs.clone(),
4999 Some(spec.prompt.clone()),
5000 execution,
5001 ),
5002 WorkflowNode::TeacherReview(spec) => push_control_execution(
5003 spec.id.as_str(),
5004 ControlNodeKind::TeacherReview,
5005 records.get(&spec.id),
5006 spec.candidates.clone(),
5007 Some("teacher review not lowered by the production adapter".to_string()),
5008 execution,
5009 ),
5010 WorkflowNode::LoopUntil(spec) => push_control_execution(
5011 spec.id.as_str(),
5012 ControlNodeKind::LoopUntil,
5013 records.get(&spec.id),
5014 spec.children.iter().map(declarative_node_id).collect(),
5015 Some("loop_until not lowered by the production adapter".to_string()),
5016 execution,
5017 ),
5018 WorkflowNode::Cond(spec) => push_control_execution(
5019 spec.id.as_str(),
5020 ControlNodeKind::Cond,
5021 records.get(&spec.id),
5022 spec.then_nodes
5023 .iter()
5024 .chain(spec.else_nodes.iter())
5025 .map(declarative_node_id)
5026 .collect(),
5027 Some("cond not lowered by the production adapter".to_string()),
5028 execution,
5029 ),
5030 WorkflowNode::Expand(spec) => push_control_execution(
5031 spec.id.as_str(),
5032 ControlNodeKind::Expand,
5033 records.get(&spec.id),
5034 Vec::new(),
5035 Some(format!("expand not lowered from {}", spec.source)),
5036 execution,
5037 ),
5038 }
5039 }
5040
5041 fn push_leaf_execution(
5042 spec: &LeafSpec,
5043 records: &HashMap<String, RuntimeTaskRecord>,
5044 execution: &mut IrWorkflowExecution,
5045 ) {
5046 let record = records.get(&spec.id);
5047 let status = record
5048 .map(|record| record.status)
5049 .unwrap_or(IrWorkflowRunStatus::Pending);
5050 mark_ir_status(execution, status);
5051 execution.leaf_results.push(LeafResult {
5052 leaf_id: spec.id.clone(),
5053 task_id: record
5054 .map(|record| record.agent_id.clone())
5055 .unwrap_or_else(|| spec.id.clone()),
5056 role: spec.role.clone(),
5057 profile: spec.profile.clone(),
5058 status,
5059 usage: record
5060 .and_then(|record| record.usage.as_ref())
5061 .map(workflow_usage_from_task)
5062 .unwrap_or_default(),
5063 memo_usage: WorkflowMemoUsage::default(),
5064 output: record.and_then(|record| record.output.clone()),
5065 artifacts: Vec::new(),
5066 schema_error: record.and_then(|record| record.schema_error.clone()),
5067 });
5068 }
5069
5070 fn push_branch_execution(
5071 spec: &BranchSpec,
5072 records: &HashMap<String, RuntimeTaskRecord>,
5073 execution: &mut IrWorkflowExecution,
5074 ) {
5075 let before = execution.leaf_results.len();
5076 for child in &spec.children {
5077 push_execution_node(child, records, execution);
5078 }
5079 let status = aggregate_ir_status(
5080 execution.leaf_results[before..]
5081 .iter()
5082 .map(|result| result.status),
5083 );
5084 mark_ir_status(execution, status);
5085 execution.branch_results.push(BranchResult {
5086 branch_id: spec.id.clone(),
5087 task_id: spec.id.clone(),
5088 status,
5089 usage: WorkflowUsage::default(),
5090 memo_usage: WorkflowMemoUsage::default(),
5091 artifacts: Vec::new(),
5092 notes: Some("production driver branch receipt from child task outcomes".to_string()),
5093 });
5094 execution.control_node_results.push(ControlNodeResult {
5095 node_id: spec.id.clone(),
5096 kind: ControlNodeKind::BranchSet,
5097 status,
5098 selected_children: spec.children.iter().map(declarative_node_id).collect(),
5099 summary: Some("branch set lowered into production child tasks".to_string()),
5100 });
5101 }
5102
5103 fn push_sequence_execution(
5104 spec: &SequenceSpec,
5105 records: &HashMap<String, RuntimeTaskRecord>,
5106 execution: &mut IrWorkflowExecution,
5107 ) {
5108 let before_leaf = execution.leaf_results.len();
5109 let before_control = execution.control_node_results.len();
5110 for child in &spec.children {
5111 push_execution_node(child, records, execution);
5112 }
5113 let status = aggregate_ir_status(
5114 execution.leaf_results[before_leaf..]
5115 .iter()
5116 .map(|result| result.status)
5117 .chain(
5118 execution.control_node_results[before_control..]
5119 .iter()
5120 .map(|result| result.status),
5121 ),
5122 );
5123 mark_ir_status(execution, status);
5124 execution.control_node_results.push(ControlNodeResult {
5125 node_id: spec.id.clone(),
5126 kind: ControlNodeKind::Sequence,
5127 status,
5128 selected_children: spec.children.iter().map(declarative_node_id).collect(),
5129 summary: Some("sequence lowered in declaration order".to_string()),
5130 });
5131 }
5132
5133 fn push_control_execution(
5134 node_id: &str,
5135 kind: ControlNodeKind,
5136 record: Option<&RuntimeTaskRecord>,
5137 selected_children: Vec<String>,
5138 fallback_summary: Option<String>,
5139 execution: &mut IrWorkflowExecution,
5140 ) {
5141 let status = record
5142 .map(|record| record.status)
5143 .unwrap_or(IrWorkflowRunStatus::Pending);
5144 mark_ir_status(execution, status);
5145 execution.control_node_results.push(ControlNodeResult {
5146 node_id: node_id.to_string(),
5147 kind,
5148 status,
5149 selected_children,
5150 summary: record
5151 .and_then(|record| record.output.clone())
5152 .or(fallback_summary),
5153 });
5154 }
5155
5156 fn aggregate_ir_status(
5157 statuses: impl IntoIterator<Item = IrWorkflowRunStatus>,
5158 ) -> IrWorkflowRunStatus {
5159 let mut saw_pending = false;
5160 let mut saw_running = false;
5161 for status in statuses {
5162 match status {
5163 IrWorkflowRunStatus::BudgetExceeded => return IrWorkflowRunStatus::BudgetExceeded,
5164 IrWorkflowRunStatus::Cancelled => return IrWorkflowRunStatus::Cancelled,
5165 IrWorkflowRunStatus::Failed | IrWorkflowRunStatus::ReplayDiverged => {
5166 return IrWorkflowRunStatus::Failed;
5167 }
5168 IrWorkflowRunStatus::Running => saw_running = true,
5169 IrWorkflowRunStatus::Pending => saw_pending = true,
5170 IrWorkflowRunStatus::Succeeded => {}
5171 }
5172 }
5173 if saw_running {
5174 IrWorkflowRunStatus::Running
5175 } else if saw_pending {
5176 IrWorkflowRunStatus::Pending
5177 } else {
5178 IrWorkflowRunStatus::Succeeded
5179 }
5180 }
5181
5182 fn mark_ir_status(execution: &mut IrWorkflowExecution, status: IrWorkflowRunStatus) {
5183 match status {
5184 IrWorkflowRunStatus::Failed | IrWorkflowRunStatus::ReplayDiverged => {
5185 execution.mark_failed()
5186 }
5187 IrWorkflowRunStatus::Cancelled => execution.mark_cancelled(),
5188 IrWorkflowRunStatus::BudgetExceeded => execution.mark_budget_exceeded(),
5189 IrWorkflowRunStatus::Running => {
5190 if execution.status == IrWorkflowRunStatus::Succeeded {
5191 execution.status = IrWorkflowRunStatus::Running;
5192 }
5193 }
5194 IrWorkflowRunStatus::Pending => {
5195 if execution.status == IrWorkflowRunStatus::Succeeded {
5196 execution.status = IrWorkflowRunStatus::Pending;
5197 }
5198 }
5199 IrWorkflowRunStatus::Succeeded => {}
5200 }
5201 }
5202
5203 fn declarative_node_id(node: &WorkflowNode) -> String {
5204 match node {
5205 WorkflowNode::BranchSet(spec) => spec.id.clone(),
5206 WorkflowNode::Leaf(spec) => spec.id.clone(),
5207 WorkflowNode::Sequence(spec) => spec.id.clone(),
5208 WorkflowNode::Reduce(spec) => spec.id.clone(),
5209 WorkflowNode::TeacherReview(spec) => spec.id.clone(),
5210 WorkflowNode::LoopUntil(spec) => spec.id.clone(),
5211 WorkflowNode::Cond(spec) => spec.id.clone(),
5212 WorkflowNode::Expand(spec) => spec.id.clone(),
5213 }
5214 }
5215
5216 fn spawn_completion_pump(
5217 driver: Arc<SubAgentWorkflowDriver>,
5218 mut rx: mpsc::Receiver<SubAgentCompletion>,
5219 ) {
5220 spawn_supervised(
5221 "workflow-completion-pump",
5222 std::panic::Location::caller(),
5223 async move {
5224 loop {
5225 let completion = tokio::select! {
5226 biased;
5227 _ = driver.runtime.cancel_token.cancelled() => break,
5228 completion = rx.recv() => completion,
5229 };
5230 let Some(completion) = completion else {
5231 break;
5232 };
5233 let agent_id = completion.agent_id.clone();
5234 let (task_completion, usage) =
5235 completion_from_manager(driver.manager.clone(), &agent_id, completion.payload)
5236 .await;
5237 driver.deliver_completion(agent_id, task_completion, usage);
5238 }
5239 },
5240 );
5241 }
5242
5243 /// Resolve a child's terminal completion from the manager, reading it exactly
5244 /// once.
5245 ///
5246 /// Every production path that publishes a child's completion does so inside
5247 /// `SubAgentManager::finish_terminal_result`, which calls
5248 /// `SubAgentTerminalDeliveryContext::deliver` (`subagent/mod.rs:2241`; the
5249 /// `try_send` that wakes this pump is at `:2254`) and then commits the terminal
5250 /// status through `update_from_result_with_persist` — both within the same
5251 /// `&mut self` call, so its caller holds the write guard on this same
5252 /// `Arc<RwLock<_>>` across the pair. See the six call sites at
5253 /// `subagent/mod.rs:4761`, `:5883`, `:6592`, `:8025`, `:11453` (the panic path,
5254 /// guarded at `:11442`) and `:11734` (guarded at `:11705`). A reader therefore
5255 /// cannot acquire the lock between the wake and the commit, so the first read
5256 /// after a completion arrives already observes the terminal status. Polling for
5257 /// it bought nothing and cost the serial pump up to a second of head-of-line
5258 /// blocking per child (#6211).
5259 ///
5260 /// Known limitation: this does not cover ids the manager never records —
5261 /// notably the workflow `run_id` that `finish_workflow_controller` sends for a
5262 /// detached nested workflow. Those fail closed here rather than being waited
5263 /// on.
5264 async fn completion_from_manager(
5265 manager: SharedSubAgentManager,
5266 agent_id: &str,
5267 fallback_payload: String,
5268 ) -> (TaskCompletion, Option<WorkflowTaskUsage>) {
5269 let snapshot_and_usage = {
5270 let manager = manager.read().await;
5271 let snapshot = manager.get_result(agent_id).ok();
5272 let usage = snapshot
5273 .as_ref()
5274 .filter(|snapshot| snapshot.status != SubAgentStatus::Running)
5275 .map(|snapshot| task_usage_from_manager(&manager, agent_id, snapshot));
5276 let verification = manager
5277 .get_worker_record(agent_id)
5278 .map(|record| record.verification);
5279 (snapshot, usage, verification)
5280 };
5281 if let (Some(snapshot), usage, verification) = snapshot_and_usage
5282 && snapshot.status != SubAgentStatus::Running
5283 {
5284 let completion = match snapshot.status {
5285 SubAgentStatus::Completed
5286 if verification.as_ref().is_some_and(|receipt| {
5287 matches!(
5288 receipt.status.as_str(),
5289 "deliverable_missing" | "claim_mismatch"
5290 )
5291 }) =>
5292 {
5293 TaskCompletion::Failed {
5294 message: format!(
5295 "Sub-agent delivery verification failed: {}",
5296 truncate_chars(
5297 &verification.expect("matched failed receipt").summary,
5298 1_000
5299 )
5300 ),
5301 }
5302 }
5303 SubAgentStatus::Completed => TaskCompletion::Completed {
5304 text: snapshot.result.clone().unwrap_or(fallback_payload),
5305 },
5306 SubAgentStatus::Failed(ref message) => TaskCompletion::Failed {
5307 message: message.clone(),
5308 },
5309 SubAgentStatus::Interrupted(ref message) => TaskCompletion::Failed {
5310 message: message.clone(),
5311 },
5312 SubAgentStatus::Cancelled => TaskCompletion::Cancelled,
5313 SubAgentStatus::BudgetExhausted => TaskCompletion::BudgetExhausted {
5314 message: "sub-agent budget exhausted".to_string(),
5315 },
5316 SubAgentStatus::Running => unreachable!("guarded above"),
5317 };
5318 return (completion, usage);
5319 }
5320 (
5321 TaskCompletion::Failed {
5322 message: format!(
5323 "sub-agent '{agent_id}' had no terminal manager record when its completion was delivered"
5324 ),
5325 },
5326 None,
5327 )
5328 }
5329
5330 /// Capture per-worker telemetry at terminal delivery (#2974): provider-reported
5331 /// tokens from the worker ledger, the model/tool step count and duration from
5332 /// the agent snapshot, and a durable artifact reference for the full output.
5333 fn task_usage_from_manager(
5334 manager: &SubAgentManager,
5335 agent_id: &str,
5336 snapshot: &SubAgentResult,
5337 ) -> WorkflowTaskUsage {
5338 let record = manager.get_worker_record(agent_id);
5339 let usage = record.as_ref().map(|record| &record.usage);
5340 let result_ref = record.as_ref().and_then(|record| {
5341 record
5342 .artifacts
5343 .iter()
5344 .find(|artifact| artifact.kind == "transcript")
5345 .or_else(|| record.artifacts.last())
5346 .map(|artifact| artifact.target.clone())
5347 });
5348 let input_tokens = usage.and_then(|usage| usage.input_tokens);
5349 let output_tokens = usage.and_then(|usage| usage.output_tokens);
5350 let total_tokens = usage.and_then(|usage| usage.total_tokens);
5351 // #4039: the worker ledger leaves these fields `None` until it receives a
5352 // typed provider usage envelope. Presence, not magnitude, is the receipt:
5353 // a provider-reported zero is still a real observation and must survive.
5354 let reported = provider_usage_was_reported(input_tokens, output_tokens, total_tokens);
5355 WorkflowTaskUsage {
5356 input_tokens: reported.then_some(input_tokens).flatten(),
5357 output_tokens: reported.then_some(output_tokens).flatten(),
5358 total_tokens: reported.then_some(total_tokens).flatten(),
5359 cost_microusd: usage.and_then(|usage| usage.cost_microusd),
5360 tool_calls: Some(snapshot.steps_taken),
5361 duration_ms: Some(snapshot.duration_ms),
5362 result_ref,
5363 token_source: reported.then_some(WorkflowTokenSource::ProviderReported),
5364 }
5365 }
5366
5367 fn provider_usage_was_reported(
5368 input_tokens: Option<u64>,
5369 output_tokens: Option<u64>,
5370 total_tokens: Option<u64>,
5371 ) -> bool {
5372 [input_tokens, output_tokens, total_tokens]
5373 .iter()
5374 .any(Option::is_some)
5375 }
5376
5377 fn cancel_child_agents(manager: SharedSubAgentManager, ids: Vec<String>) {
5378 if ids.is_empty() {
5379 return;
5380 }
5381 if let Ok(mut manager_guard) = manager.try_write() {
5382 for id in ids {
5383 let _ = manager_guard.cancel_agent(&id);
5384 }
5385 return;
5386 }
5387 if tokio::runtime::Handle::try_current().is_ok() {
5388 spawn_supervised(
5389 "workflow-cancel-children",
5390 std::panic::Location::caller(),
5391 async move {
5392 let mut manager_guard = manager.write().await;
5393 for id in ids {
5394 let _ = manager_guard.cancel_agent(&id);
5395 }
5396 },
5397 );
5398 }
5399 }
5400
5401 fn lock_mutex<T>(mutex: &Mutex<T>) -> Result<MutexGuard<'_, T>, ToolError> {
5402 mutex
5403 .lock()
5404 .map_err(|_| ToolError::execution_failed("workflow state lock poisoned"))
5405 }
5406
5407 fn now_ms() -> u64 {
5408 SystemTime::now()
5409 .duration_since(UNIX_EPOCH)
5410 .unwrap_or_default()
5411 .as_millis()
5412 .try_into()
5413 .unwrap_or(u64::MAX)
5414 }
5415
5416 mod journal;
5417
5418 use journal::{WorkflowWorkspaceState, peek_shared_workflow_state, shared_workflow_state};
5419
5420 /// Bounded, read-only projection of one workflow run for the human-only
5421 /// `/structcopy` command (#2033).
5422 ///
5423 /// Built on the existing [`WorkflowRunSummary`] projection so retention and
5424 /// truncation accounting (`events_total` / `events_dropped`) stay in exactly
5425 /// one place. Two extra constraints beyond the model-facing summary:
5426 /// `source_path` collapses to a bare file-name label so no filesystem path
5427 /// leaves the process, and raw event/hook payloads never enter the
5428 /// projection. Returns `None` when `run_id` is unknown to this session;
5429 /// never creates workspace state or touches the journal.
5430 pub(crate) fn structcopy_run_projection(
5431 workspace: &Path,
5432 run_id: &str,
5433 owner_session_id: Option<&str>,
5434 ) -> Option<Value> {
5435 let owner_session_id = owner_session_id?;
5436 let state = peek_shared_workflow_state(workspace)?;
5437 let runs = state
5438 .runs
5439 .lock()
5440 .unwrap_or_else(|poison| poison.into_inner());
5441 let record = runs
5442 .get(run_id)
5443 .filter(|record| record.owner_session_id.as_deref() == Some(owner_session_id))?;
5444 let mut value = serde_json::to_value(record.summary()).ok()?;
5445 if let Some(object) = value.as_object_mut() {
5446 let source_file = record
5447 .source_path
5448 .as_ref()
5449 .and_then(|path| path.file_name())
5450 .and_then(|name| name.to_str())
5451 .map(|name| Value::String(name.to_string()))
5452 .unwrap_or(Value::Null);
5453 object.remove("source_path");
5454 object.insert("source_file".to_string(), source_file);
5455 // `WorkflowRunSummary` predates truthful unavailable-state rendering
5456 // and uses zero defaults when no execution projection exists. A
5457 // structural export must not turn that absence into measured zeros.
5458 if record.execution.is_none() {
5459 object.insert("leaf_count".to_string(), Value::Null);
5460 object.insert("branch_count".to_string(), Value::Null);
5461 object.insert("control_count".to_string(), Value::Null);
5462 }
5463 }
5464 Some(value)
5465 }
5466
5467 /// One workflow run as the human-facing `/workflow` command reads it: a
5468 /// bounded projection of the run record with no raw event payloads and no
5469 /// filesystem paths beyond the source file name.
5470 #[derive(Debug, Clone, PartialEq, Eq)]
5471 pub(crate) struct HostWorkflowRunLine {
5472 pub run_id: String,
5473 /// Human-facing stage derived from the durable owner record and its typed
5474 /// event tail: queued | running | waiting | completed | degraded | failed |
5475 /// cancelled. This is presentation state, not a second lifecycle owner.
5476 pub status: &'static str,
5477 /// Whether the canonical owner record is still nonterminal. Controls key
5478 /// off this bit instead of reverse-parsing the display stage.
5479 pub active: bool,
5480 /// The run's goal, its workflow id, or the source file name.
5481 pub label: String,
5482 pub started_at_ms: u64,
5483 pub completed_at_ms: Option<u64>,
5484 pub child_count: usize,
5485 pub last_progress: Option<String>,
5486 pub error: Option<String>,
5487 }
5488
5489 fn host_workflow_stage(record: &WorkflowRunRecord) -> &'static str {
5490 match record.status {
5491 WorkflowRunStatus::Completed => return "completed",
5492 WorkflowRunStatus::Degraded => return "degraded",
5493 WorkflowRunStatus::Failed => return "failed",
5494 WorkflowRunStatus::Cancelled => return "cancelled",
5495 WorkflowRunStatus::Running => {}
5496 }
5497
5498 // The owner journal remains the sole lifecycle source. These finer
5499 // nonterminal stages come only from its typed event stream: before the VM
5500 // reports real activity the accepted run is queued; while one or more
5501 // children remain open the VM is waiting on those agents; otherwise it is
5502 // actively running host/script work. A truncated tail may forget a child's
5503 // start event, in which case we safely show the coarser `running` state.
5504 let mut open_children = HashSet::new();
5505 let mut work_started = !record.progress.is_empty();
5506 for event in &record.events {
5507 match &event.kind {
5508 WorkflowUiEventKind::TaskStarted(started) => {
5509 work_started = true;
5510 open_children.insert(started.task_id.clone());
5511 }
5512 WorkflowUiEventKind::TaskCompleted { task_id, .. } => {
5513 work_started = true;
5514 open_children.remove(task_id);
5515 }
5516 WorkflowUiEventKind::RunStarted { .. }
5517 | WorkflowUiEventKind::RunCompleted { .. }
5518 | WorkflowUiEventKind::RunCancelled { .. } => {}
5519 _ => work_started = true,
5520 }
5521 }
5522 if !open_children.is_empty() {
5523 "waiting"
5524 } else if work_started {
5525 "running"
5526 } else {
5527 "queued"
5528 }
5529 }
5530
5531 fn host_run_line(record: &WorkflowRunRecord) -> HostWorkflowRunLine {
5532 let summary = record.summary();
5533 let label = summary
5534 .workflow_goal
5535 .clone()
5536 .or_else(|| summary.workflow_id.clone())
5537 .or_else(|| {
5538 summary
5539 .source_path
5540 .as_ref()
5541 .and_then(|path| path.file_name())
5542 .map(|name| name.to_string_lossy().into_owned())
5543 })
5544 .unwrap_or_else(|| "workflow".to_string());
5545 HostWorkflowRunLine {
5546 run_id: summary.run_id,
5547 status: host_workflow_stage(record),
5548 active: summary.status == WorkflowRunStatus::Running,
5549 label,
5550 started_at_ms: summary.started_at_ms,
5551 completed_at_ms: summary.completed_at_ms,
5552 child_count: summary.child_count,
5553 last_progress: summary.last_progress,
5554 error: summary.error,
5555 }
5556 }
5557
5558 /// Live workspace state if this process already has it, otherwise the
5559 /// existing run journal. Never creates `.codewhale/` or the ledger.
5560 fn host_workflow_state(workspace: &Path) -> Option<Arc<WorkflowWorkspaceState>> {
5561 if let Some(state) = peek_shared_workflow_state(workspace) {
5562 return Some(state);
5563 }
5564 // No live state yet: hydrate only if a journal already exists so
5565 // status/cancel can see runs from a previous process without creating
5566 // files in a workspace that never ran a workflow.
5567 workflow_journal_exists(workspace).then(|| shared_workflow_state(workspace))
5568 }
5569
5570 /// Cancel hydrates an on-disk journal without the restart-orphan Failed
5571 /// rewrite so a running line with no live controller can still be cancelled.
5572 fn host_workflow_state_for_cancel(workspace: &Path) -> Option<Arc<WorkflowWorkspaceState>> {
5573 if let Some(state) = peek_shared_workflow_state(workspace) {
5574 return Some(state);
5575 }
5576 workflow_journal_exists(workspace)
5577 .then(|| WorkflowWorkspaceState::open_preserving_running(workspace))
5578 }
5579
5580 fn workflow_journal_exists(workspace: &Path) -> bool {
5581 workspace
5582 .join(journal::CODEWHALE_DIR)
5583 .join(journal::WORKFLOW_RUNS_FILE)
5584 .is_file()
5585 }
5586
5587 /// Every workflow run this workspace knows about (live and journaled),
5588 /// oldest first. Read-only: never creates the journal or workspace state.
5589 /// The `/workflow status` command reads this directly so status never costs
5590 /// a model turn.
5591 pub(crate) fn host_workflow_runs(
5592 workspace: &Path,
5593 owner_session_id: Option<&str>,
5594 ) -> Vec<HostWorkflowRunLine> {
5595 let Some(owner_session_id) = owner_session_id else {
5596 return Vec::new();
5597 };
5598 let Some(state) = host_workflow_state(workspace) else {
5599 return Vec::new();
5600 };
5601 let runs = state
5602 .runs
5603 .lock()
5604 .unwrap_or_else(|poison| poison.into_inner());
5605 let mut lines: Vec<HostWorkflowRunLine> = runs
5606 .values()
5607 .filter(|record| record.owner_session_id.as_deref() == Some(owner_session_id))
5608 .map(host_run_line)
5609 .collect();
5610 lines.sort_by_key(|line| line.started_at_ms);
5611 lines
5612 }
5613
5614 /// One child-agent row of a workflow run as the host reads it: the typed
5615 /// spawn label, the resolved role/model, the phase it was admitted under,
5616 /// and a terminal state derived from the task-completed event (`running`
5617 /// until one exists). Same bounded projection rules as
5618 /// [`HostWorkflowRunLine`]: no raw event payloads, no filesystem paths.
5619 #[derive(Debug, Clone, PartialEq, Eq)]
5620 pub(crate) struct HostWorkflowChildRow {
5621 pub task_id: String,
5622 pub label: Option<String>,
5623 pub role: Option<String>,
5624 pub model: Option<String>,
5625 pub phase: Option<String>,
5626 pub state: &'static str,
5627 }
5628
5629 /// Expanded host projection for the `/workflows` run manager: the run line
5630 /// plus the phase order, the child-agent roster, and the retained progress
5631 /// tail, all derived from the same run record the journal persists. Derived
5632 /// from the bounded event tail, so a long run shows the newest phases and
5633 /// children, not the whole history. Read-only: never creates the journal or
5634 /// workspace state.
5635 #[derive(Debug, Clone, PartialEq, Eq)]
5636 pub(crate) struct HostWorkflowRunDetail {
5637 pub line: HostWorkflowRunLine,
5638 pub phases: Vec<String>,
5639 pub children: Vec<HostWorkflowChildRow>,
5640 pub progress_tail: Vec<String>,
5641 pub has_result: bool,
5642 }
5643
5644 fn host_task_state(status: IrWorkflowRunStatus) -> &'static str {
5645 match status {
5646 IrWorkflowRunStatus::Pending => "pending",
5647 IrWorkflowRunStatus::Running => "running",
5648 IrWorkflowRunStatus::Succeeded => "succeeded",
5649 IrWorkflowRunStatus::Failed => "failed",
5650 IrWorkflowRunStatus::Cancelled => "cancelled",
5651 IrWorkflowRunStatus::BudgetExceeded => "budget_exceeded",
5652 IrWorkflowRunStatus::ReplayDiverged => "replay_diverged",
5653 }
5654 }
5655
5656 fn host_run_detail(record: &WorkflowRunRecord) -> HostWorkflowRunDetail {
5657 let line = host_run_line(record);
5658 let mut phases: Vec<String> = Vec::new();
5659 let mut children: Vec<HostWorkflowChildRow> = Vec::new();
5660 for event in &record.events {
5661 match &event.kind {
5662 WorkflowUiEventKind::PhaseStarted { title } => {
5663 if phases.last().map(String::as_str) != Some(title.as_str()) {
5664 phases.push(title.clone());
5665 }
5666 }
5667 WorkflowUiEventKind::TaskStarted(started) => children.push(HostWorkflowChildRow {
5668 task_id: started.task_id.clone(),
5669 label: started
5670 .workflow_task_label
5671 .clone()
5672 .or_else(|| started.label.clone()),
5673 role: started
5674 .resolved_role
5675 .clone()
5676 .or_else(|| started.role.clone()),
5677 model: if started.resolved_model.is_empty() {
5678 None
5679 } else {
5680 Some(started.resolved_model.clone())
5681 },
5682 phase: started.workflow_phase_id.clone(),
5683 state: "running",
5684 }),
5685 WorkflowUiEventKind::TaskCompleted {
5686 task_id, status, ..
5687 } => {
5688 if let Some(row) = children.iter_mut().find(|row| row.task_id == *task_id) {
5689 row.state = host_task_state(*status);
5690 }
5691 }
5692 _ => {}
5693 }
5694 }
5695 HostWorkflowRunDetail {
5696 line,
5697 phases,
5698 children,
5699 progress_tail: record
5700 .progress
5701 .iter()
5702 .rev()
5703 .take(HOST_RUN_PROGRESS_TAIL)
5704 .rev()
5705 .cloned()
5706 .collect(),
5707 has_result: record.result.is_some(),
5708 }
5709 }
5710
5711 /// Every workflow run this workspace knows about (live and journaled) with
5712 /// the detail the `/workflows` manager renders, oldest first. Read-only:
5713 /// never creates the journal or workspace state.
5714 pub(crate) fn host_workflow_run_details(
5715 workspace: &Path,
5716 owner_session_id: Option<&str>,
5717 ) -> Vec<HostWorkflowRunDetail> {
5718 let Some(owner_session_id) = owner_session_id else {
5719 return Vec::new();
5720 };
5721 let Some(state) = host_workflow_state(workspace) else {
5722 return Vec::new();
5723 };
5724 let runs = state
5725 .runs
5726 .lock()
5727 .unwrap_or_else(|poison| poison.into_inner());
5728 let mut details: Vec<HostWorkflowRunDetail> = runs
5729 .values()
5730 .filter(|record| record.owner_session_id.as_deref() == Some(owner_session_id))
5731 .map(host_run_detail)
5732 .collect();
5733 details.sort_by_key(|detail| detail.line.started_at_ms);
5734 details
5735 }
5736
5737 /// Cancel a running workflow directly from the host (the `/workflow cancel`
5738 /// command and the panel's cancel control), without a model turn. Returns
5739 /// the run's projection after cancellation, or a plain reason when the run
5740 /// is unknown. A run that already finished is reported as it is. After a
5741 /// restart, a journaled running line with no live controller is cancelled
5742 /// in the journal rather than rejected as unknown or controller-missing.
5743 pub(crate) fn host_cancel_workflow(
5744 workspace: &Path,
5745 run_id: &str,
5746 owner_session_id: Option<&str>,
5747 ) -> Result<HostWorkflowRunLine, String> {
5748 let Some(owner_session_id) = owner_session_id else {
5749 return Err(format!("Unknown workflow run '{run_id}'."));
5750 };
5751 let Some(state) = host_workflow_state_for_cancel(workspace) else {
5752 return Err(format!("Unknown workflow run '{run_id}'."));
5753 };
5754 match cancel_workflow_run(run_id, state.clone(), owner_session_id) {
5755 Ok(_) => {
5756 let runs = state
5757 .runs
5758 .lock()
5759 .unwrap_or_else(|poison| poison.into_inner());
5760 runs.get(run_id)
5761 .map(host_run_line)
5762 .ok_or_else(|| format!("Unknown workflow run '{run_id}'."))
5763 }
5764 Err(err) => Err(err.to_string()),
5765 }
5766 }
5767
5768 /// Seed a minimal run record so `/structcopy` tests can exercise the
5769 /// workflow projection without standing up the JS VM.
5770 #[cfg(test)]
5771 pub(crate) fn structcopy_test_seed_run(workspace: &Path, run_id: &str, owner_session_id: &str) {
5772 let state = shared_workflow_state(workspace);
5773 let record = WorkflowRunRecord::new(
5774 run_id.to_string(),
5775 Some(owner_session_id.to_string()),
5776 None,
5777 None,
5778 None,
5779 );
5780 state
5781 .runs
5782 .lock()
5783 .unwrap_or_else(|poison| poison.into_inner())
5784 .insert(run_id.to_string(), record);
5785 }
5786
5787 /// Reconcile workflow bindings after the journal has replayed restart
5788 /// recovery. The journal owns lifecycle truth; the graph only receives its
5789 /// monotonic projection.
5790 pub(crate) fn reconcile_persisted_workflow_bindings(
5791 work: &SharedWorkRuntime,
5792 session_id: &str,
5793 workspace: &Path,
5794 ) -> Result<usize, String> {
5795 let state = shared_workflow_state(workspace);
5796 let records = state
5797 .runs
5798 .lock()
5799 .unwrap_or_else(|poison| poison.into_inner())
5800 .values()
5801 .cloned()
5802 .collect::<Vec<_>>();
5803 let candidates = work
5804 .reconcilable_durable_bindings(Some(session_id))
5805 .into_iter()
5806 .filter(|external| external.starts_with("workflow:"))
5807 .collect::<std::collections::HashSet<_>>();
5808 let mut seen = std::collections::HashSet::new();
5809 let mut changed = 0usize;
5810 for record in records {
5811 if record.owner_session_id.as_deref() != Some(session_id) {
5812 continue;
5813 }
5814 let external = format!("workflow:{}", record.run_id);
5815 if !candidates.contains(&external) {
5816 continue;
5817 }
5818 seen.insert(external.clone());
5819 let lifecycle = WorkflowWorkLifecycle {
5820 work: work.clone(),
5821 session_id: session_id.to_string(),
5822 external,
5823 };
5824 changed += usize::from(lifecycle.reconcile_record(&record)?);
5825 }
5826 for external in candidates.difference(&seen) {
5827 changed += usize::from(work.reconcile_observation(
5828 session_id,
5829 external,
5830 OperationObservation::OwnerMissing {
5831 checked_at: i64::try_from(now_ms()).unwrap_or(i64::MAX),
5832 },
5833 )?);
5834 }
5835 Ok(changed)
5836 }
5837
5838 #[cfg(test)]
5839 mod tests {
5840 use super::*;
5841 use crate::client::CodewhaleClient;
5842 use crate::tools::ToolRegistryBuilder;
5843 use crate::tools::subagent::{SubAgentRuntime, new_shared_subagent_manager};
5844 use axum::{Json, Router, routing::post};
5845 use codewhale_workflow::{IsolationMode, leaf_is_write_capable};
5846 use std::sync::atomic::{AtomicUsize, Ordering};
5847
5848 #[test]
5849 fn a_clean_slot_ledger_leaves_the_run_completed() {
5850 let ledger = SlotLedger {
5851 tasks: 3,
5852 failed_tasks: 0,
5853 rejected: 0,
5854 dead_fanouts: 0,
5855 dropped_slots: 0,
5856 any_child_ran: true,
5857 retained_detail: None,
5858 };
5859 assert_eq!(ledger.classify(), None);
5860 }
5861
5862 #[test]
5863 fn a_run_with_no_tasks_at_all_stays_completed() {
5864 // A script that orchestrates nothing is not a dropped-slot failure.
5865 assert_eq!(SlotLedger::default().classify(), None);
5866 }
5867
5868 #[test]
5869 fn some_failed_slots_degrade_the_run_without_failing_it() {
5870 let (status, message) = SlotLedger {
5871 tasks: 3,
5872 failed_tasks: 1,
5873 rejected: 0,
5874 dead_fanouts: 0,
5875 dropped_slots: 0,
5876 any_child_ran: true,
5877 retained_detail: None,
5878 }
5879 .classify()
5880 .expect("a dropped slot must be classified");
5881 assert_eq!(status, WorkflowRunStatus::Degraded);
5882 assert!(message.contains("1 of 3 task(s) failed"), "{message}");
5883 }
5884
5885 #[test]
5886 fn a_run_whose_every_task_failed_cannot_report_success() {
5887 // R9: `parallel()`'s settled default resolves each dead slot to
5888 // `null`, so the script returns cleanly. The run must not.
5889 let (status, message) = SlotLedger {
5890 tasks: 4,
5891 failed_tasks: 4,
5892 rejected: 0,
5893 dead_fanouts: 0,
5894 dropped_slots: 0,
5895 any_child_ran: true,
5896 retained_detail: None,
5897 }
5898 .classify()
5899 .expect("a fully-failed fan-out must be classified");
5900 assert_eq!(status, WorkflowRunStatus::Failed);
5901 assert!(
5902 message.contains("no task produced a result: all 4 task(s) failed"),
5903 "{message}"
5904 );
5905 assert_ne!(
5906 owner_state_for_run_status(status),
5907 OwnerState::Completed,
5908 "a run that lost every task must never project as completed"
5909 );
5910 }
5911
5912 #[test]
5913 fn a_fully_failed_fan_out_also_names_its_rejected_dispatches() {
5914 let (status, message) = SlotLedger {
5915 tasks: 2,
5916 failed_tasks: 2,
5917 rejected: 1,
5918 dead_fanouts: 0,
5919 dropped_slots: 0,
5920 any_child_ran: true,
5921 retained_detail: Some("admission cap".to_string()),
5922 }
5923 .classify()
5924 .expect("a fully-failed fan-out must be classified");
5925 assert_eq!(status, WorkflowRunStatus::Failed);
5926 assert!(
5927 message.contains("all 2 task(s) failed")
5928 && message.contains("1 dispatch(es) were rejected"),
5929 "{message}"
5930 );
5931 }
5932
5933 #[test]
5934 fn every_dispatch_rejected_still_fails_before_any_child_ran() {
5935 let (status, message) = SlotLedger {
5936 tasks: 0,
5937 failed_tasks: 0,
5938 rejected: 2,
5939 dead_fanouts: 0,
5940 dropped_slots: 0,
5941 any_child_ran: false,
5942 retained_detail: Some("depth ceiling".to_string()),
5943 }
5944 .classify()
5945 .expect("an all-rejected run must be classified");
5946 assert_eq!(status, WorkflowRunStatus::Failed);
5947 assert!(
5948 message.contains("no child agents ran") && message.contains("depth ceiling"),
5949 "{message}"
5950 );
5951 }
5952
5953 #[test]
5954 fn budget_exhausted_children_count_as_failed_for_the_all_failed_rule() {
5955 // R9 blocker: a fan-out whose every child died of budget exhaustion
5956 // used to classify as a plain completion because `failed_tasks`
5957 // counted only `Failed` records. The records below are built through
5958 // the real completion mapping, not hand-set counters.
5959 let completions = [
5960 TaskCompletion::BudgetExhausted {
5961 message: "shared pool drained".to_string(),
5962 },
5963 TaskCompletion::BudgetExhausted {
5964 message: "shared pool drained".to_string(),
5965 },
5966 ];
5967 let task_records: Vec<RuntimeTaskRecord> = completions
5968 .iter()
5969 .enumerate()
5970 .map(|(index, completion)| {
5971 let (status, output) = task_completion_status(completion);
5972 RuntimeTaskRecord {
5973 agent_id: format!("agent_{index}"),
5974 label: None,
5975 role: None,
5976 status,
5977 output,
5978 schema_error: None,
5979 usage: None,
5980 }
5981 })
5982 .collect();
5983 let ledger = SlotLedger {
5984 tasks: task_records.len(),
5985 failed_tasks: task_records
5986 .iter()
5987 .filter(|task| task.failed_for_ledger())
5988 .count(),
5989 rejected: 0,
5990 // The fan-out those children died in also reported itself dead.
5991 dead_fanouts: 1,
5992 dropped_slots: 0,
5993 any_child_ran: true,
5994 retained_detail: None,
5995 };
5996 assert_eq!(task_records[0].status, IrWorkflowRunStatus::BudgetExceeded);
5997 let (status, message) = ledger
5998 .classify()
5999 .expect("an all-budget-exhausted run must be classified");
6000 assert_eq!(status, WorkflowRunStatus::Failed);
6001 assert!(
6002 message.contains("all 2 task(s) failed")
6003 && message.contains("1 fan-out(s) lost every slot"),
6004 "{message}"
6005 );
6006 assert_ne!(
6007 owner_state_for_run_status(status),
6008 OwnerState::Completed,
6009 "a run that lost every child to budget exhaustion must never project as completed"
6010 );
6011 }
6012
6013 #[test]
6014 fn a_fan_out_of_script_throws_with_no_task_records_fails_the_run() {
6015 // R9 blocker: thunks that throw without calling `task()` leave no
6016 // task record and no dispatch failure — only the structured
6017 // every-slot-failed count. That run produced nothing.
6018 let (status, message) = SlotLedger {
6019 tasks: 0,
6020 failed_tasks: 0,
6021 rejected: 0,
6022 dead_fanouts: 1,
6023 dropped_slots: 0,
6024 any_child_ran: false,
6025 retained_detail: None,
6026 }
6027 .classify()
6028 .expect("a dead script-throw fan-out must be classified");
6029 assert_eq!(status, WorkflowRunStatus::Failed);
6030 assert!(
6031 message.contains("no work survived") && message.contains("1 fan-out(s)"),
6032 "{message}"
6033 );
6034 }
6035
6036 #[test]
6037 fn a_dead_fan_out_beside_surviving_work_degrades_instead_of_failing() {
6038 // Some work survived elsewhere in the run: keep the output, refuse
6039 // the plain-success label, but do not claim nothing completed.
6040 let (status, message) = SlotLedger {
6041 tasks: 1,
6042 failed_tasks: 0,
6043 rejected: 0,
6044 dead_fanouts: 1,
6045 dropped_slots: 0,
6046 any_child_ran: true,
6047 retained_detail: None,
6048 }
6049 .classify()
6050 .expect("a dead fan-out beside surviving work must be classified");
6051 assert_eq!(status, WorkflowRunStatus::Degraded);
6052 assert!(
6053 message.contains("1 fan-out(s) lost every slot")
6054 && message.contains("result may be partial"),
6055 "{message}"
6056 );
6057 }
6058
6059 #[test]
6060 fn an_empty_fan_out_is_not_a_dead_fan_out() {
6061 // `parallel([])` declares no slots; a run that orchestrates nothing
6062 // stays completed (mirrors `a_run_with_no_tasks_at_all_stays_completed`).
6063 assert_eq!(SlotLedger::default().classify(), None);
6064 }
6065
6066 #[test]
6067 fn settled_runs_leave_a_report_artifact_under_codewhale_reports() {
6068 let tmp = tempfile::tempdir().expect("tempdir");
6069 let mut record = WorkflowRunRecord::new(
6070 "workflow_report_1".to_string(),
6071 Some("session-test".to_string()),
6072 None,
6073 None,
6074 None,
6075 );
6076 record.status = WorkflowRunStatus::Completed;
6077 record.workflow_goal = Some("prove the report artifact".to_string());
6078 record.push_progress("phase: scan".to_string());
6079 record.result = Some(serde_json::json!({"confirmed": 2}));
6080
6081 write_run_report_artifact(tmp.path(), &record);
6082
6083 let path = tmp
6084 .path()
6085 .join(".codewhale")
6086 .join("reports")
6087 .join("workflow_report_1.md");
6088 let body = std::fs::read_to_string(&path).expect("report written");
6089 assert!(body.contains("# Workflow run workflow_report_1"), "{body}");
6090 assert!(body.contains("status: Completed"), "{body}");
6091 assert!(body.contains("prove the report artifact"), "{body}");
6092 assert!(body.contains("phase: scan"), "{body}");
6093 assert!(body.contains("\"confirmed\": 2"), "{body}");
6094 }
6095
6096 #[test]
6097 fn running_runs_write_no_report_artifact() {
6098 let tmp = tempfile::tempdir().expect("tempdir");
6099 let record = WorkflowRunRecord::new(
6100 "workflow_report_2".to_string(),
6101 Some("session-test".to_string()),
6102 None,
6103 None,
6104 None,
6105 );
6106 write_run_report_artifact(tmp.path(), &record);
6107 assert!(
6108 !tmp.path().join(".codewhale").join("reports").exists(),
6109 "running runs must not leave report files"
6110 );
6111 }
6112
6113 #[test]
6114 fn source_path_accepts_the_home_workflow_store_and_rejects_elsewhere() {
6115 let _lock = crate::test_support::lock_test_env();
6116 let tmp = tempfile::tempdir().expect("tempdir");
6117 let home = tmp.path().join("home");
6118 let store = home.join(".codewhale").join("workflows");
6119 std::fs::create_dir_all(&store).expect("store");
6120 let _home_guard = crate::test_support::EnvVarGuard::set("HOME", &home);
6121 let _userprofile_guard = crate::test_support::EnvVarGuard::set("USERPROFILE", &home);
6122
6123 let saved = store.join("triage.workflow.js");
6124 std::fs::write(&saved, "phase('scan');\n").expect("write saved workflow");
6125 let elsewhere = tmp.path().join("outside.workflow.js");
6126 std::fs::write(&elsewhere, "phase('scan');\n").expect("write outside workflow");
6127
6128 let workspace = tmp.path().join("ws");
6129 std::fs::create_dir_all(&workspace).expect("workspace");
6130 let context = ToolContext::new(workspace);
6131
6132 let resolved = read_workflow_source_path(saved.to_str().expect("utf8 path"), &context)
6133 .expect("home workflow store is a first-class source");
6134 assert!(resolved.source.contains("phase('scan')"));
6135
6136 let err = read_workflow_source_path(elsewhere.to_str().expect("utf8 path"), &context)
6137 .expect_err("arbitrary outside paths stay denied");
6138 assert!(
6139 err.to_string()
6140 .contains("workspace or ~/.codewhale/workflows"),
6141 "{err}"
6142 );
6143 }
6144
6145 #[test]
6146 fn restored_workflow_binding_consumes_journal_recovery() {
6147 let tmp = tempfile::tempdir().expect("tempdir");
6148 let state = WorkflowWorkspaceState::open(tmp.path());
6149 let record = WorkflowRunRecord::new(
6150 "workflow_restore".to_string(),
6151 Some("restored-workflow-session".to_string()),
6152 None,
6153 None,
6154 None,
6155 );
6156 state.record_snapshot(&record);
6157
6158 let work = crate::work_graph::new_shared_work_runtime(
6159 crate::tools::todo::new_shared_todo_list(),
6160 crate::tools::plan::new_shared_plan_state(),
6161 );
6162 work.register_operation(
6163 "restored-workflow-session",
6164 OperationIntent::new(
6165 "workflow:workflow_restore",
6166 "restored workflow",
6167 true,
6168 "workflow",
6169 "restore-test",
6170 ),
6171 )
6172 .expect("register saved workflow binding");
6173 work.reconcile_operation(
6174 "restored-workflow-session",
6175 OperationOwnerSnapshot::new("workflow:workflow_restore", OwnerState::Running, 1, 1),
6176 )
6177 .expect("saved running owner state");
6178 work.register_operation(
6179 "restored-workflow-session",
6180 OperationIntent::new(
6181 "workflow:workflow_absent",
6182 "absent workflow",
6183 true,
6184 "workflow",
6185 "absent-restore-test",
6186 ),
6187 )
6188 .expect("register absent workflow binding");
6189 work.reconcile_operation(
6190 "restored-workflow-session",
6191 OperationOwnerSnapshot::new("workflow:workflow_absent", OwnerState::Running, 1, 1),
6192 )
6193 .expect("saved absent owner state");
6194
6195 assert_eq!(
6196 reconcile_persisted_workflow_bindings(&work, "restored-workflow-session", tmp.path(),),
6197 Ok(2)
6198 );
6199 let graph = work
6200 .capture(Some("restored-workflow-session"))
6201 .expect("capture restored workflow")
6202 .expect("graph")
6203 .graph;
6204 let operation = graph
6205 .nodes
6206 .iter()
6207 .find(|node| {
6208 node.binding
6209 .as_ref()
6210 .is_some_and(|binding| binding.external == "workflow:workflow_restore")
6211 })
6212 .expect("workflow operation");
6213 assert_eq!(operation.state, crate::work_graph::NodeState::Failed);
6214 assert_eq!(
6215 operation
6216 .binding
6217 .as_ref()
6218 .and_then(|binding| binding.last_observation.as_ref())
6219 .map(|observation| observation.seq),
6220 Some(2),
6221 "journal replay must advance the lost live owner before graph reconciliation"
6222 );
6223 let absent = graph
6224 .nodes
6225 .iter()
6226 .find(|node| {
6227 node.binding
6228 .as_ref()
6229 .is_some_and(|binding| binding.external == "workflow:workflow_absent")
6230 })
6231 .expect("absent workflow operation");
6232 assert_eq!(absent.state, crate::work_graph::NodeState::Stale);
6233 assert_eq!(
6234 reconcile_persisted_workflow_bindings(&work, "restored-workflow-session", tmp.path(),),
6235 Ok(0),
6236 "rechecking an already stale missing owner must be idempotent"
6237 );
6238 }
6239
6240 #[tokio::test]
6241 async fn cancellation_without_controller_marks_the_journal_cancelled() {
6242 let tmp = tempfile::tempdir().expect("tempdir");
6243 let state = WorkflowWorkspaceState::open(tmp.path());
6244 let record = WorkflowRunRecord::new(
6245 "workflow_missing_controller".to_string(),
6246 Some("missing-controller-session".to_string()),
6247 None,
6248 None,
6249 None,
6250 );
6251 state
6252 .runs
6253 .lock()
6254 .expect("runs lock")
6255 .insert(record.run_id.clone(), record.clone());
6256 state.record_snapshot(&record);
6257
6258 let work = crate::work_graph::new_shared_work_runtime(
6259 crate::tools::todo::new_shared_todo_list(),
6260 crate::tools::plan::new_shared_plan_state(),
6261 );
6262 work.register_operation(
6263 "missing-controller-session",
6264 OperationIntent::new(
6265 "workflow:workflow_missing_controller",
6266 "missing controller",
6267 true,
6268 "workflow",
6269 "missing-controller-test",
6270 ),
6271 )
6272 .expect("register workflow");
6273 work.reconcile_operation(
6274 "missing-controller-session",
6275 OperationOwnerSnapshot::new(
6276 "workflow:workflow_missing_controller",
6277 OwnerState::Running,
6278 1,
6279 1,
6280 ),
6281 )
6282 .expect("running workflow");
6283 state.attach_lifecycle(
6284 "workflow_missing_controller",
6285 WorkflowWorkLifecycle {
6286 work: work.clone(),
6287 session_id: "missing-controller-session".to_string(),
6288 external: "workflow:workflow_missing_controller".to_string(),
6289 },
6290 );
6291
6292 cancel_workflow(
6293 json!({"run_id": "workflow_missing_controller"}),
6294 state.clone(),
6295 "missing-controller-session",
6296 )
6297 .await
6298 .expect("controller-less cancel must still journal cancelled");
6299 let record = state
6300 .runs
6301 .lock()
6302 .expect("runs lock")
6303 .get("workflow_missing_controller")
6304 .cloned()
6305 .expect("workflow owner");
6306 assert_eq!(record.status, WorkflowRunStatus::Cancelled);
6307 assert_eq!(record.lifecycle_seq, 2);
6308 assert!(
6309 record
6310 .error
6311 .as_deref()
6312 .is_some_and(|error| error.contains("no live process")),
6313 "expected an honest nothing-live receipt, got {:?}",
6314 record.error
6315 );
6316 let operation = work
6317 .capture(Some("missing-controller-session"))
6318 .expect("capture")
6319 .expect("graph")
6320 .graph
6321 .nodes
6322 .into_iter()
6323 .find(|node| node.kind == crate::work_graph::NodeKind::Operation)
6324 .expect("workflow operation");
6325 assert_eq!(operation.state, crate::work_graph::NodeState::Cancelled);
6326 }
6327
6328 #[tokio::test]
6329 async fn workflow_controls_are_session_owned_and_legacy_records_fail_closed() {
6330 let tmp = tempfile::tempdir().expect("tempdir");
6331 let state = WorkflowWorkspaceState::open(tmp.path());
6332 let record = |run_id: &str, owner: Option<&str>| {
6333 WorkflowRunRecord::new(
6334 run_id.to_string(),
6335 owner.map(str::to_string),
6336 None,
6337 None,
6338 None,
6339 )
6340 };
6341 let legacy_record = {
6342 let mut value =
6343 serde_json::to_value(record("workflow-legacy", Some("session-from-newer-schema")))
6344 .expect("serialize legacy fixture");
6345 value
6346 .as_object_mut()
6347 .expect("record object")
6348 .remove("owner_session_id");
6349 serde_json::from_value::<WorkflowRunRecord>(value)
6350 .expect("legacy ownerless journal record parses")
6351 };
6352 assert!(legacy_record.owner_session_id.is_none());
6353 {
6354 let mut runs = state.runs.lock().expect("runs");
6355 runs.insert(
6356 "workflow-a".to_string(),
6357 record("workflow-a", Some("session-a")),
6358 );
6359 runs.insert(
6360 "workflow-b".to_string(),
6361 record("workflow-b", Some("session-b")),
6362 );
6363 runs.insert("workflow-legacy".to_string(), legacy_record);
6364 }
6365
6366 let listed =
6367 status_workflow(json!({}), state.clone(), "session-b").expect("session-owned list");
6368 let listed: Value = serde_json::from_str(&listed.content).expect("list json");
6369 assert_eq!(listed["count"], 1);
6370 assert_eq!(listed["runs"][0]["run_id"], "workflow-b");
6371
6372 let empty_dir = tempfile::tempdir().expect("empty tempdir");
6373 let empty_state = WorkflowWorkspaceState::open(empty_dir.path());
6374 let unknown_a = workflow_result_for("workflow-a", empty_state.clone(), "session-b")
6375 .expect_err("unknown A run");
6376 let foreign = workflow_result_for("workflow-a", state.clone(), "session-b")
6377 .expect_err("foreign run must be hidden");
6378 let unknown_legacy = workflow_result_for("workflow-legacy", empty_state, "session-b")
6379 .expect_err("unknown legacy-shaped id");
6380 let legacy = workflow_result_for("workflow-legacy", state.clone(), "session-b")
6381 .expect_err("legacy ownerless run must be hidden");
6382 assert_eq!(foreign.to_string(), unknown_a.to_string());
6383 assert_eq!(legacy.to_string(), unknown_legacy.to_string());
6384
6385 cancel_workflow(json!({"run_id": "workflow-a"}), state.clone(), "session-b")
6386 .await
6387 .expect_err("B cannot cancel A");
6388 cancel_workflow(
6389 json!({"run_id": "workflow-legacy"}),
6390 state.clone(),
6391 "session-b",
6392 )
6393 .await
6394 .expect_err("B cannot cancel ownerless legacy work");
6395 {
6396 let runs = state.runs.lock().expect("runs");
6397 assert_eq!(runs["workflow-a"].status, WorkflowRunStatus::Running);
6398 assert_eq!(runs["workflow-legacy"].status, WorkflowRunStatus::Running);
6399 }
6400
6401 cancel_workflow(json!({"run_id": "workflow-b"}), state.clone(), "session-b")
6402 .await
6403 .expect("B can cancel B");
6404 assert_eq!(
6405 state.runs.lock().expect("runs")["workflow-b"].status,
6406 WorkflowRunStatus::Cancelled
6407 );
6408 }
6409
6410 #[test]
6411 fn handoff_compaction_preserves_release_sized_evidence() {
6412 let payload = format!("APPROVE\n{}\nterminal: RunCompleted", "e".repeat(1_500));
6413
6414 assert_eq!(
6415 compact_handoff_payload(&payload, WORKFLOW_HANDOFF_MAX_CHARS),
6416 payload
6417 );
6418 }
6419
6420 #[test]
6421 fn handoff_compaction_still_caps_oversized_artifacts() {
6422 let payload = "e".repeat(WORKFLOW_HANDOFF_MAX_CHARS + 1);
6423 let compacted = compact_handoff_payload(&payload, WORKFLOW_HANDOFF_MAX_CHARS);
6424
6425 assert_eq!(compacted.chars().count(), WORKFLOW_HANDOFF_MAX_CHARS + 3);
6426 assert!(compacted.ends_with("..."));
6427 }
6428
6429 #[test]
6430 fn declarative_detection_matches_indented_and_nonleading_workflow_calls() {
6431 // column-0 forms
6432 assert!(looks_like_declarative_workflow("workflow({ tasks: [] })"));
6433 assert!(looks_like_declarative_workflow(
6434 "export default workflow({})"
6435 ));
6436 // #dogfood 0.8.67: a leading statement/comment followed by an INDENTED
6437 // top-level workflow( call must still be detected as declarative.
6438 assert!(looks_like_declarative_workflow(
6439 "// build the run\n workflow({\n tasks: [],\n })"
6440 ));
6441 // imperative scripts must not be misdetected as declarative
6442 assert!(!looks_like_declarative_workflow(
6443 "return await parallel([() => task({ description: \"x\" })]);"
6444 ));
6445 assert!(!looks_like_declarative_workflow("const x = myworkflow(1);"));
6446 }
6447
6448 #[test]
6449 fn workflow_action_defaults_to_start() {
6450 assert_eq!(
6451 parse_workflow_action(&json!({})).unwrap(),
6452 WorkflowAction::Start
6453 );
6454 assert_eq!(
6455 parse_workflow_action(&json!({"action": "run"})).unwrap(),
6456 WorkflowAction::Run
6457 );
6458 }
6459
6460 #[test]
6461 fn named_fleet_maps_workflow_role_to_profile_before_spawn() {
6462 let fleet = FleetRoleMap::from_pairs([
6463 ("scout", "scout"),
6464 ("implementer", "builder"),
6465 ("reviewer", "reviewer"),
6466 ("verifier", "verifier"),
6467 ("release_lead", "manager"),
6468 ])
6469 .expect("fleet");
6470 let mut request = TaskRequest {
6471 description: "fix it".to_string(),
6472 subagent_type: None,
6473 role: Some("implementer".to_string()),
6474 profile: None,
6475 model: None,
6476 model_strength: None,
6477 thinking: None,
6478 cwd: None,
6479 worktree: true,
6480 write_authority: Some("worktree_write".to_string()),
6481 write_roots: vec!["src".to_string()],
6482 exact_files: Vec::new(),
6483 coordination_contracts: vec!["test-contract".to_string()],
6484 dependencies: Vec::new(),
6485 acceptance: Vec::new(),
6486 allowed_tools: None,
6487 disallowed_tools: Vec::new(),
6488 max_depth: None,
6489 token_budget: None,
6490 max_steps: None,
6491 wall_time_secs: None,
6492 response_schema: None,
6493 schema_repair_attempts: None,
6494 label: Some("fix".to_string()),
6495 phase: Some("implement".to_string()),
6496 };
6497
6498 apply_named_fleet_to_task_request(Some(&fleet), &mut request).expect("resolve");
6499
6500 assert_eq!(request.role.as_deref(), Some("implement"));
6501 assert_eq!(request.profile.as_deref(), Some("builder"));
6502 }
6503
6504 // ── Exact named Fleet (schema = "exact") ────────────────────────────────
6505
6506 /// A Fleet that references a saved, reusable Reasoning Router service, and
6507 /// whose members' ids differ from their semantic roles — the case a gate
6508 /// keyed on a role has to keep working through.
6509 const EXACT_GLM_FLEET: &str = r#"
6510 name = "glm-pair"
6511 schema = "exact"
6512 reasoning_router = "luna-low"
6513
6514 [[members]]
6515 id = "implementer"
6516 role = "builder"
6517 provider = "zai"
6518 model = "glm-5"
6519 reasoning = "auto"
6520 permissions = "read_write"
6521
6522 [[members]]
6523 id = "auditor"
6524 role = "reviewer"
6525 provider = "zai"
6526 model = "glm-5"
6527 reasoning = "high"
6528 permissions = "read_only"
6529 "#;
6530
6531 fn exact_task_request(role: &str) -> TaskRequest {
6532 TaskRequest {
6533 description: "land the fix".to_string(),
6534 subagent_type: None,
6535 role: Some(role.to_string()),
6536 profile: None,
6537 model: None,
6538 model_strength: None,
6539 thinking: None,
6540 cwd: None,
6541 worktree: false,
6542 write_authority: None,
6543 write_roots: Vec::new(),
6544 exact_files: Vec::new(),
6545 coordination_contracts: Vec::new(),
6546 dependencies: Vec::new(),
6547 acceptance: Vec::new(),
6548 allowed_tools: None,
6549 disallowed_tools: Vec::new(),
6550 max_depth: None,
6551 token_budget: None,
6552 max_steps: None,
6553 wall_time_secs: None,
6554 response_schema: None,
6555 schema_repair_attempts: None,
6556 label: None,
6557 phase: None,
6558 }
6559 }
6560
6561 /// A task for a write-capable member. The spawn boundary refuses an
6562 /// unbounded write claim, so a write-capable exact task always carries a
6563 /// declared scope — the same contract, checked before the Router runs.
6564 fn exact_write_task_request(role: &str) -> TaskRequest {
6565 TaskRequest {
6566 write_roots: vec!["crates/tui".to_string()],
6567 ..exact_task_request(role)
6568 }
6569 }
6570
6571 fn exact_session() -> codewhale_workflow::PermissionCeiling {
6572 codewhale_workflow::PermissionCeiling {
6573 write: true,
6574 network_tool: true,
6575 shell: codewhale_workflow::ShellCeiling::Full,
6576 delegation_depth: codewhale_config::DEFAULT_SPAWN_DEPTH,
6577 tools: true,
6578 }
6579 }
6580
6581 fn exact_workflow_with(
6582 text: &str,
6583 router: Option<std::sync::Arc<crate::fleet::exact::StaticFleetRouter>>,
6584 ) -> crate::fleet::exact::ExactFleetWorkflow {
6585 let document = codewhale_workflow::FleetDocument::parse(text).expect("exact fleet parses");
6586 crate::fleet::exact::ExactFleetWorkflow::for_tests(
6587 &document,
6588 codewhale_workflow::QualifiedFleetId {
6589 name: "glm-pair".to_string(),
6590 origin: "workspace".to_string(),
6591 },
6592 router,
6593 )
6594 }
6595
6596 fn exact_workflow(text: &str) -> crate::fleet::exact::ExactFleetWorkflow {
6597 exact_workflow_with(
6598 text,
6599 Some(crate::fleet::exact::StaticFleetRouter::new(
6600 r#"{"reasoning":"max"}"#,
6601 )),
6602 )
6603 }
6604
6605 /// Binding resolves the member and Runtime authority; routing resolves reasoning.
6606 /// Both halves land on the request, in that order.
6607 #[tokio::test]
6608 async fn exact_fleet_task_launch_resolves_member_route_and_runtime_authority() {
6609 let operation = exact_workflow(EXACT_GLM_FLEET);
6610 let mut request = exact_write_task_request("builder");
6611
6612 let binding = bind_exact_fleet_task_request(&operation, exact_session(), &mut request)
6613 .expect("exact member resolves");
6614
6615 // Addressed by member id, so the frozen snapshot route (which carries
6616 // the exact provider pin and canonical wire model) is what the
6617 // spawn resolves…
6618 assert_eq!(request.profile.as_deref(), Some("implementer"));
6619 // …while the semantic role is preserved for gates and records.
6620 assert_eq!(request.role.as_deref(), Some("implement"));
6621 let member = operation
6622 .snapshot()
6623 .member("implementer")
6624 .expect("snapshot entry");
6625 assert_eq!(member.route.provider, "zai");
6626 assert_eq!(member.route.model, "glm-5");
6627
6628 // Runtime's role/parent intersection reached the request before routing.
6629 assert_eq!(request.write_authority.as_deref(), Some("workspace_write"));
6630 assert_eq!(
6631 request.max_depth,
6632 Some(codewhale_config::DEFAULT_SPAWN_DEPTH)
6633 );
6634 assert!(request.thinking.is_none(), "reasoning is not decided yet");
6635
6636 route_admitted_exact_task(&operation, &binding, &mut request)
6637 .await
6638 .expect("routing");
6639 // `auto` was resolved by the Router into a concrete tier — never the
6640 // literal sentinel, and never the legacy local heuristic.
6641 assert_eq!(request.thinking.as_deref(), Some("max"));
6642
6643 // A read-only member is launched read-only, with no router call.
6644 let mut auditor = exact_task_request("reviewer");
6645 let auditor_binding =
6646 bind_exact_fleet_task_request(&operation, exact_session(), &mut auditor)
6647 .expect("auditor resolves");
6648 route_admitted_exact_task(&operation, &auditor_binding, &mut auditor)
6649 .await
6650 .expect("routing");
6651 assert_eq!(auditor.write_authority.as_deref(), Some("read_only"));
6652 assert_eq!(auditor.thinking.as_deref(), Some("high"));
6653 assert_eq!(auditor.role.as_deref(), Some("reviewer"));
6654 assert_eq!(auditor.profile.as_deref(), Some("auditor"));
6655 }
6656
6657 /// The session's `[workflow]` table must decide the approval requirement;
6658 /// before this the tool consulted product defaults only, so a user who
6659 /// set `require_approval_for_writes = false` (documented in
6660 /// docs/AUTOMATIC_WORKFLOWS.md) still got the approval card.
6661 #[test]
6662 fn workflow_tool_honors_the_session_workflow_config() {
6663 let tmp = tempfile::tempdir().expect("tempdir");
6664 let ctx = ToolContext::new(tmp.path().to_path_buf());
6665 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
6666 let mut runtime = SubAgentRuntime::new(
6667 stub_client(),
6668 "deepseek-v4-flash".to_string(),
6669 ctx,
6670 true,
6671 None,
6672 manager.clone(),
6673 );
6674 let input = json!({
6675 "action": "start",
6676 "plan": {
6677 "goal": "write freely",
6678 "risk": "writes",
6679 "children": [{ "prompt": "edit", "type": "implementer" }]
6680 }
6681 });
6682 let tool = WorkflowTool::new(Arc::clone(&manager), runtime.clone());
6683 assert_eq!(
6684 tool.approval_requirement_for(&input),
6685 ApprovalRequirement::Required,
6686 "product default: writes need approval"
6687 );
6688
6689 let config = crate::config::Config {
6690 workflow: Some(codewhale_config::WorkflowConfigToml {
6691 require_approval_for_writes: false,
6692 ..Default::default()
6693 }),
6694 ..Default::default()
6695 };
6696 runtime.api_config = Some(Arc::new(config));
6697 let tool = WorkflowTool::new(Arc::clone(&manager), runtime.clone());
6698 assert_eq!(
6699 tool.approval_requirement_for(&input),
6700 ApprovalRequirement::Auto,
6701 "the session config must win"
6702 );
6703
6704 let read_only = json!({
6705 "action": "start",
6706 "plan": {
6707 "goal": "scout crates",
6708 "risk": "read_only",
6709 "children": [{ "prompt": "look", "type": "explore" }]
6710 }
6711 });
6712 let config = crate::config::Config {
6713 workflow: Some(codewhale_config::WorkflowConfigToml {
6714 auto_start_read_only: false,
6715 ..Default::default()
6716 }),
6717 ..Default::default()
6718 };
6719 runtime.api_config = Some(Arc::new(config));
6720 let tool = WorkflowTool::new(manager, runtime);
6721 assert_eq!(
6722 tool.approval_requirement_for(&read_only),
6723 ApprovalRequirement::Required,
6724 "auto_start_read_only = false must still ask"
6725 );
6726 }
6727
6728 /// The gate machinery keys on the **semantic role**. It must still fire
6729 /// when a member's id differs from that role — the exact failure an earlier
6730 /// pass introduced by stamping the profile id into `role`.
6731 #[tokio::test]
6732 #[allow(clippy::await_holding_lock)]
6733 async fn a_role_keyed_gate_still_fires_when_the_member_id_differs() {
6734 let tmp = tempfile::tempdir().expect("tempdir");
6735 let ctx = ToolContext::new(tmp.path().to_path_buf());
6736 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
6737 let runtime = SubAgentRuntime::new(
6738 stub_client(),
6739 "deepseek-v4-flash".to_string(),
6740 ctx.clone(),
6741 true,
6742 None,
6743 manager.clone(),
6744 );
6745 let state = WorkflowWorkspaceState::open(tmp.path());
6746 let run_id = "workflow_exact_gate".to_string();
6747 // The gate blocks the semantic role `implement`, whose member id is
6748 // the *different* string `implementer`.
6749 let gates = vec![GateSpec {
6750 id: "explore-findings".to_string(),
6751 role: "explore".to_string(),
6752 on: GateOn::RoleComplete,
6753 gate: GateKind::Approve,
6754 on_fail: codewhale_workflow::GateOnFail::Block,
6755 blocks_role: Some("implement".to_string()),
6756 max_retries: 0,
6757 artifact_kind: Some("findings".to_string()),
6758 require_explicit_verdict: false,
6759 }];
6760 state.runs.lock().expect("runs").insert(
6761 run_id.clone(),
6762 WorkflowRunRecord::new(
6763 run_id.clone(),
6764 Some("session-test".to_string()),
6765 None,
6766 None,
6767 None,
6768 ),
6769 );
6770 let driver = SubAgentWorkflowDriver::new(
6771 run_id.clone(),
6772 "session-test".to_string(),
6773 manager,
6774 runtime,
6775 state.clone(),
6776 None,
6777 WorkflowFleetBinding::None,
6778 gates,
6779 tmp.path().to_path_buf(),
6780 );
6781
6782 // The upstream explore agent fails, which puts the gate into a
6783 // blocking state.
6784 driver.evaluate_gates_for_completed_role(&RuntimeTaskRecord {
6785 agent_id: "explore-agent".to_string(),
6786 label: Some("explore".to_string()),
6787 role: Some("explore".to_string()),
6788 status: IrWorkflowRunStatus::Failed,
6789 output: None,
6790 schema_error: None,
6791 usage: None,
6792 });
6793
6794 let operation = exact_workflow(EXACT_GLM_FLEET);
6795 let mut request = exact_write_task_request("builder");
6796 bind_exact_fleet_task_request(&operation, exact_session(), &mut request).expect("bind");
6797
6798 assert_ne!(
6799 request.role.as_deref(),
6800 request.profile.as_deref(),
6801 "this fleet's ids and roles differ, which is the whole point"
6802 );
6803 assert_eq!(request.role.as_deref(), Some("implement"));
6804
6805 let err = driver
6806 .prepare_request_for_gates(&mut request)
6807 .expect_err("a blocking gate on `implement` must still see `implement`");
6808 assert!(err.to_string().contains("implement"), "{err}");
6809
6810 // The same gate does *not* block a task carrying the member id, which
6811 // is exactly why stamping the id into `role` silently disabled it.
6812 let mut by_id = exact_task_request("builder");
6813 by_id.role = Some("implementer".to_string());
6814 by_id.profile = None;
6815 assert!(
6816 driver.prepare_request_for_gates(&mut by_id).is_ok(),
6817 "the profile id is not the semantic role the gate keys on"
6818 );
6819 }
6820
6821 /// A task whose `role` and `profile` name different members is rejected
6822 /// rather than resolved by precedence.
6823 #[test]
6824 fn exact_fleet_rejects_a_conflicting_task_role_and_profile() {
6825 let operation = exact_workflow(EXACT_GLM_FLEET);
6826 let mut request = exact_task_request("reviewer");
6827 request.profile = Some("implementer".to_string());
6828
6829 let err = bind_exact_fleet_task_request(&operation, exact_session(), &mut request)
6830 .expect_err("conflicting identity");
6831 let message = format!("{err:?}");
6832 assert!(message.contains("different members"), "{message}");
6833 }
6834
6835 #[test]
6836 fn exact_fleet_rejects_task_level_route_overrides() {
6837 let operation = exact_workflow(EXACT_GLM_FLEET);
6838
6839 for mutate in [
6840 (|request: &mut TaskRequest| request.model = Some("glm-4".to_string()))
6841 as fn(&mut TaskRequest),
6842 |request: &mut TaskRequest| request.model_strength = Some("faster".to_string()),
6843 |request: &mut TaskRequest| request.thinking = Some("off".to_string()),
6844 ] {
6845 let mut request = exact_task_request("builder");
6846 mutate(&mut request);
6847 let err = bind_exact_fleet_task_request(&operation, exact_session(), &mut request)
6848 .expect_err("an exact fleet member may not be re-routed per task");
6849 let message = format!("{err:?}");
6850 assert!(
6851 message.contains("not allowed"),
6852 "override must be rejected, not ignored: {message}"
6853 );
6854 }
6855 }
6856
6857 /// A task must not be able to replace Runtime's role/parent authority by
6858 /// asking for a different agent type, tool surface, or write authority.
6859 #[test]
6860 fn exact_fleet_rejects_task_level_posture_widening() {
6861 let operation = exact_workflow(EXACT_GLM_FLEET);
6862
6863 for (field, mutate) in [
6864 (
6865 "subagent_type",
6866 (|request: &mut TaskRequest| {
6867 request.subagent_type = Some("general".to_string());
6868 }) as fn(&mut TaskRequest),
6869 ),
6870 ("write_authority", |request: &mut TaskRequest| {
6871 request.write_authority = Some("workspace_write".to_string());
6872 }),
6873 ] {
6874 // The Runtime reviewer posture is read-only.
6875 let mut request = exact_task_request("reviewer");
6876 mutate(&mut request);
6877 let err = bind_exact_fleet_task_request(&operation, exact_session(), &mut request)
6878 .expect_err("Runtime authority must win over a task option");
6879 let message = format!("{err:?}");
6880 assert!(
6881 message.contains(field),
6882 "{field} must be rejected: {message}"
6883 );
6884 }
6885
6886 // With no task options, Runtime's reviewer posture is what lands.
6887 let mut clean = exact_task_request("reviewer");
6888 bind_exact_fleet_task_request(&operation, exact_session(), &mut clean)
6889 .expect("clean launch");
6890 assert_eq!(clean.write_authority.as_deref(), Some("read_only"));
6891 assert_eq!(clean.subagent_type, None);
6892 // A task allowlist can only narrow; it never removes Runtime denials.
6893 let mut narrowed = exact_task_request("reviewer");
6894 narrowed.allowed_tools = Some(vec!["exec_shell".to_string()]);
6895 bind_exact_fleet_task_request(&operation, exact_session(), &mut narrowed)
6896 .expect("allowlist narrows without overriding the Runtime denylist");
6897 assert_eq!(narrowed.write_authority.as_deref(), Some("read_only"));
6898 assert!(
6899 narrowed
6900 .disallowed_tools
6901 .iter()
6902 .any(|tool| tool.eq_ignore_ascii_case("exec_shell"))
6903 );
6904 }
6905
6906 /// Runtime's reviewer posture becomes a real tool policy, and the legacy
6907 /// Fleet `permissions` key cannot turn network reach off or rewrite it.
6908 #[test]
6909 fn exact_fleet_runtime_authority_reaches_the_spawn_request() {
6910 let operation = exact_workflow(EXACT_GLM_FLEET);
6911 let mut request = exact_task_request("reviewer");
6912
6913 bind_exact_fleet_task_request(&operation, exact_session(), &mut request).expect("bind");
6914
6915 assert_eq!(
6916 request.allowed_tools, None,
6917 "Runtime reviewer inherits the parent tool surface"
6918 );
6919 for denied in ["write_file", "apply_patch", "exec_shell"] {
6920 assert!(
6921 request.disallowed_tools.iter().any(|name| name == denied),
6922 "{denied} must be denied: {:?}",
6923 request.disallowed_tools
6924 );
6925 }
6926 for available in ["Bash", "Web", "web.run", "fetch_url", "mcp*"] {
6927 assert!(
6928 !request
6929 .disallowed_tools
6930 .iter()
6931 .any(|name| name.eq_ignore_ascii_case(available)),
6932 "Runtime reviewer keeps {available}: {:?}",
6933 request.disallowed_tools
6934 );
6935 }
6936 }
6937
6938 #[test]
6939 fn legacy_fleet_permissions_cannot_change_runtime_authority() {
6940 let narrow = exact_workflow(EXACT_GLM_FLEET);
6941 let legacy_full_text =
6942 EXACT_GLM_FLEET.replace("permissions = \"read_only\"", "permissions = \"full\"");
6943 let legacy_full = exact_workflow(&legacy_full_text);
6944 let mut narrow_request = exact_task_request("reviewer");
6945 let mut full_request = exact_task_request("reviewer");
6946
6947 let narrow_binding =
6948 bind_exact_fleet_task_request(&narrow, exact_session(), &mut narrow_request)
6949 .expect("legacy narrow value loads");
6950 let full_binding =
6951 bind_exact_fleet_task_request(&legacy_full, exact_session(), &mut full_request)
6952 .expect("legacy full value loads");
6953
6954 assert_eq!(narrow_binding.authority, full_binding.authority);
6955 assert_eq!(narrow_request.write_authority, full_request.write_authority);
6956 assert_eq!(
6957 narrow_request.disallowed_tools,
6958 full_request.disallowed_tools
6959 );
6960 }
6961
6962 /// A Router decision is paid for before the child exists. If the spawn then
6963 /// fails, the receipt is the only record that tokens were spent and — for a
6964 /// cross-provider Router — that a bounded summary already left the host. It
6965 /// must survive the failure rather than being dropped with it.
6966 #[tokio::test]
6967 async fn a_routing_receipt_survives_a_failed_spawn() {
6968 let tmp = tempfile::tempdir().expect("tempdir");
6969 let ctx = ToolContext::new(tmp.path().to_path_buf());
6970 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
6971 let runtime = SubAgentRuntime::new(
6972 stub_client(),
6973 "deepseek-v4-flash".to_string(),
6974 ctx.clone(),
6975 true,
6976 None,
6977 manager.clone(),
6978 );
6979 let state = WorkflowWorkspaceState::open(tmp.path());
6980 let run_id = "workflow_orphaned_receipt".to_string();
6981 state.runs.lock().expect("runs").insert(
6982 run_id.clone(),
6983 WorkflowRunRecord::new(
6984 run_id.clone(),
6985 Some("session-test".to_string()),
6986 None,
6987 None,
6988 None,
6989 ),
6990 );
6991 let driver = SubAgentWorkflowDriver::new(
6992 run_id.clone(),
6993 "session-test".to_string(),
6994 manager,
6995 runtime,
6996 state.clone(),
6997 None,
6998 WorkflowFleetBinding::None,
6999 Vec::new(),
7000 tmp.path().to_path_buf(),
7001 );
7002
7003 let operation = exact_workflow(EXACT_GLM_FLEET);
7004 let mut request = exact_write_task_request("builder");
7005 let binding =
7006 bind_exact_fleet_task_request(&operation, exact_session(), &mut request).expect("bind");
7007 let receipt = route_admitted_exact_task(&operation, &binding, &mut request)
7008 .await
7009 .expect("routing");
7010
7011 driver.record_orphaned_fleet_receipt(&receipt, "Sub-agent depth limit reached");
7012
7013 let events = state
7014 .runs
7015 .lock()
7016 .expect("runs")
7017 .get(&run_id)
7018 .expect("run")
7019 .events
7020 .clone();
7021 let logged = events
7022 .iter()
7023 .filter_map(|event| match &event.kind {
7024 WorkflowUiEventKind::Log { message } => Some(message.clone()),
7025 _ => None,
7026 })
7027 .find(|message| message.contains("spawn_failed=true"))
7028 .expect("the receipt must outlive the failed spawn");
7029
7030 assert!(logged.contains("member=implementer"), "{logged}");
7031 assert!(logged.contains("source=fleet_router"), "{logged}");
7032 assert!(
7033 logged.contains("reasoning_router:workspace/luna-low"),
7034 "{logged}"
7035 );
7036 assert!(logged.contains("Sub-agent depth limit reached"), "{logged}");
7037 // Still content-free: a failure line is no excuse to echo the task.
7038 assert!(!logged.contains("land the fix"), "{logged}");
7039 }
7040
7041 /// A member's semantic role is what the operator named and what gates key
7042 /// on; the roster profile's role is the Runtime **posture** selected after
7043 /// identity resolution. The started event must show the first, not the
7044 /// second.
7045 #[tokio::test]
7046 async fn a_started_event_shows_the_members_role_not_its_permission_posture() {
7047 const AUDIT_FLEET: &str = r#"
7048 name = "glm-pair"
7049 schema = "exact"
7050
7051 [[members]]
7052 id = "auditor"
7053 role = "auditor"
7054 provider = "zai"
7055 model = "glm-5"
7056 reasoning = "high"
7057 permissions = "read_only"
7058 "#;
7059 let operation = exact_workflow_with(AUDIT_FLEET, None);
7060 // The member declares `permissions = "read_only"`, and an undeclared
7061 // role name no longer hands the child a write-capable posture (#5575),
7062 // so a declared write scope is now correctly refused at bind time.
7063 // Ask for no write scope, which is what this member actually has.
7064 let mut request = exact_task_request("auditor");
7065 let binding =
7066 bind_exact_fleet_task_request(&operation, exact_session(), &mut request).expect("bind");
7067 let receipt = route_admitted_exact_task(&operation, &binding, &mut request)
7068 .await
7069 .expect("routing");
7070
7071 // The binding authority — and therefore the spawn metadata — carries
7072 // the posture role, because that is what picked the child's tool
7073 // surface.
7074 let posture = binding.authority.posture_role.to_string();
7075 assert_eq!(posture, "explore");
7076 assert_eq!(receipt.posture_role.as_deref(), Some("explore"));
7077
7078 // What the run displays is the member's role, not that posture.
7079 assert_eq!(
7080 displayed_resolved_role(Some(&receipt), Some(&posture), request.role.as_deref()),
7081 Some("auditor".to_string()),
7082 "the panel must not rename the operator's member to its posture"
7083 );
7084
7085 // A non-Fleet task keeps the previous precedence untouched.
7086 assert_eq!(
7087 displayed_resolved_role(None, Some("builder"), Some("reviewer")),
7088 Some("builder".to_string())
7089 );
7090 assert_eq!(
7091 displayed_resolved_role(None, None, Some("reviewer")),
7092 Some("reviewer".to_string())
7093 );
7094 }
7095
7096 /// The spawn boundary refuses an unbounded write claim. A task that will
7097 /// hit that refusal must be stopped while it is still free — before the
7098 /// Router is asked anything — or the operator pays for a routing decision
7099 /// about work that could never have started.
7100 #[test]
7101 fn a_predictably_invalid_write_scope_is_rejected_before_the_router_runs() {
7102 let router = crate::fleet::exact::StaticFleetRouter::new(r#"{"reasoning":"max"}"#);
7103 let operation = exact_workflow_with(EXACT_GLM_FLEET, Some(router.clone()));
7104
7105 // Write-capable member, no declared scope: refused at the spawn
7106 // boundary, so refused here first.
7107 let mut unbounded = exact_task_request("builder");
7108 let err = bind_exact_fleet_task_request(&operation, exact_session(), &mut unbounded)
7109 .expect_err("an unbounded write claim never reaches a spawn");
7110 let message = format!("{err:?}");
7111 assert!(message.contains("write_roots"), "{message}");
7112
7113 // Read-only member declaring a write scope is the mirror error.
7114 let mut scoped_read_only = exact_task_request("reviewer");
7115 scoped_read_only.exact_files = vec!["crates/tui/src/main.rs".to_string()];
7116 let err = bind_exact_fleet_task_request(&operation, exact_session(), &mut scoped_read_only)
7117 .expect_err("a read-only member may not claim files");
7118 assert!(format!("{err:?}").contains("read-only"), "{err:?}");
7119
7120 // Neither spent a routing request.
7121 assert_eq!(
7122 router.call_count(),
7123 0,
7124 "validation that the spawn will fail must precede the router call"
7125 );
7126
7127 // The same task with a declared scope binds cleanly.
7128 let mut bounded = exact_write_task_request("builder");
7129 bind_exact_fleet_task_request(&operation, exact_session(), &mut bounded)
7130 .expect("a bounded write claim is valid");
7131 }
7132
7133 /// The parent posture wins over the saved Fleet, in the request the child
7134 /// actually receives.
7135 #[test]
7136 fn a_read_only_session_narrows_a_write_capable_exact_member() {
7137 let operation = exact_workflow(EXACT_GLM_FLEET);
7138 let session = codewhale_workflow::PermissionCeiling {
7139 write: false,
7140 network_tool: false,
7141 shell: codewhale_workflow::ShellCeiling::ReadOnly,
7142 delegation_depth: 0,
7143 tools: true,
7144 };
7145
7146 let mut request = exact_task_request("builder");
7147 bind_exact_fleet_task_request(&operation, session, &mut request).expect("bind");
7148
7149 assert_eq!(
7150 request.write_authority.as_deref(),
7151 Some("read_only"),
7152 "a saved read_write member must not write inside a read-only session"
7153 );
7154 assert_eq!(request.max_depth, Some(0));
7155 }
7156
7157 /// A task that never reaches admission must never reach the Router.
7158 #[test]
7159 fn a_rejected_or_unadmitted_task_spends_no_router_call() {
7160 let router = crate::fleet::exact::StaticFleetRouter::new(r#"{"reasoning":"max"}"#);
7161 let operation = exact_workflow_with(EXACT_GLM_FLEET, Some(router.clone()));
7162
7163 // Rejected by an override check, before the member is even resolved.
7164 let mut overridden = exact_task_request("builder");
7165 overridden.model = Some("glm-4".to_string());
7166 assert!(
7167 bind_exact_fleet_task_request(&operation, exact_session(), &mut overridden).is_err()
7168 );
7169
7170 // Rejected by member resolution.
7171 let mut unknown = exact_task_request("wizard");
7172 assert!(bind_exact_fleet_task_request(&operation, exact_session(), &mut unknown).is_err());
7173
7174 // Admitted-shaped but never routed: the capacity-blocked case.
7175 let mut queued = exact_write_task_request("builder");
7176 bind_exact_fleet_task_request(&operation, exact_session(), &mut queued).expect("bind");
7177
7178 assert_eq!(
7179 router.call_count(),
7180 0,
7181 "binding must never contact the reasoning router"
7182 );
7183 }
7184
7185 /// The routing decision must survive the run as a durable, visible receipt
7186 /// that carries no task content.
7187 #[tokio::test]
7188 async fn an_exact_fleet_launch_produces_a_durable_routing_receipt() {
7189 let operation = exact_workflow(EXACT_GLM_FLEET);
7190 let mut request = exact_write_task_request("builder");
7191
7192 let binding =
7193 bind_exact_fleet_task_request(&operation, exact_session(), &mut request).expect("bind");
7194 let receipt = route_admitted_exact_task(&operation, &binding, &mut request)
7195 .await
7196 .expect("routing");
7197
7198 assert_eq!(receipt.fleet, "workspace/glm-pair");
7199 assert_eq!(receipt.member_id, "implementer");
7200 assert_eq!(receipt.member_role, "implement");
7201 assert_eq!(receipt.provider, "zai");
7202 assert_eq!(receipt.model, "glm-5");
7203 assert_eq!(receipt.requested_reasoning, "auto");
7204 assert_eq!(receipt.effective_reasoning, "max");
7205 assert_eq!(receipt.selection_source, "fleet_router");
7206 let router = receipt.router.as_ref().expect("router identity");
7207 assert_eq!(router.service_kind, "reasoning_router");
7208 assert_eq!(router.qualified(), "workspace/luna-low");
7209 let call = router.call.as_ref().expect("call disclosure");
7210 assert_eq!(call.requested, "low");
7211 assert_eq!(call.effective, "low");
7212
7213 // It rides the durable task_started event, and older consumers that
7214 // never saw this field still deserialize.
7215 let event = WorkflowUiEvent::new(
7216 "session-test",
7217 WorkflowUiEventKind::TaskStarted(Box::new(WorkflowTaskStartedEvent {
7218 task_id: "t1".to_string(),
7219 label: None,
7220 role: request.role.clone(),
7221 profile: request.profile.clone(),
7222 model: None,
7223 strength: None,
7224 thinking: request.thinking.clone(),
7225 // The #4039 reasoning fields carry the same requested →
7226 // effective pair the receipt records, so the event and its
7227 // receipt cannot disagree about what the child ran at.
7228 requested_reasoning: Some(receipt.requested_reasoning.clone()),
7229 effective_reasoning: Some(receipt.effective_reasoning.clone()),
7230 resolved_role: None,
7231 resolved_profile: None,
7232 resolved_provider: "zai".to_string(),
7233 resolved_model: "glm-5".to_string(),
7234 route_source: "fleet".to_string(),
7235 child_route: None,
7236 worktree: false,
7237 workspace: None,
7238 git_branch: None,
7239 parent_task_id: None,
7240 depth: 0,
7241 workflow_run_id: None,
7242 workflow_phase_id: None,
7243 workflow_task_label: None,
7244 workflow_child_index: None,
7245 fleet_receipt: Some(receipt.clone()),
7246 })),
7247 );
7248 let payload = serde_json::to_value(&event).expect("serialize");
7249 let rendered = payload.to_string();
7250 for expected in [
7251 "fleet_receipt",
7252 "\"selection_source\":\"fleet_router\"",
7253 "\"provider_effective_reasoning\"",
7254 "\"requested_reasoning\":\"auto\"",
7255 "gpt-5.6-luna",
7256 "\"service_kind\":\"reasoning_router\"",
7257 ] {
7258 assert!(
7259 rendered.contains(expected),
7260 "{expected} missing: {rendered}"
7261 );
7262 }
7263 // No absolute paths, secrets, or task text on a durable event.
7264 for forbidden in ["/Users/", "/home/", ".toml", "api_key", "land the fix"] {
7265 assert!(!rendered.contains(forbidden), "{forbidden} in {rendered}");
7266 }
7267
7268 // The visible one-line form names every side of the decision.
7269 let line = receipt.line();
7270 for expected in [
7271 "requested=auto",
7272 "effective=max",
7273 "source=fleet_router",
7274 "reasoning_router:workspace/luna-low",
7275 "router_call_requested=low",
7276 ] {
7277 assert!(line.contains(expected), "{expected} missing from {line}");
7278 }
7279 }
7280
7281 #[test]
7282 fn exact_fleet_rejects_an_unknown_member() {
7283 let operation = exact_workflow(EXACT_GLM_FLEET);
7284 let mut request = exact_task_request("wizard");
7285
7286 let err = bind_exact_fleet_task_request(&operation, exact_session(), &mut request)
7287 .expect_err("unknown member");
7288 let message = format!("{err:?}");
7289 assert!(message.contains("wizard"), "{message}");
7290 assert!(message.contains("implementer"), "{message}");
7291
7292 // The Reasoning Router is never dispatchable as a worker.
7293 let mut router_request = exact_task_request("luna-low");
7294 assert!(
7295 bind_exact_fleet_task_request(&operation, exact_session(), &mut router_request)
7296 .is_err(),
7297 "the reasoning router must not be launchable as a worker"
7298 );
7299 }
7300
7301 /// One saved Router profile, referenced by two different Fleets.
7302 #[tokio::test]
7303 async fn one_reasoning_router_profile_serves_two_fleets() {
7304 let first = exact_workflow(EXACT_GLM_FLEET);
7305 let second = {
7306 let text = EXACT_GLM_FLEET.replace("name = \"glm-pair\"", "name = \"glm-solo\"");
7307 let document =
7308 codewhale_workflow::FleetDocument::parse(&text).expect("second fleet parses");
7309 crate::fleet::exact::ExactFleetWorkflow::for_tests(
7310 &document,
7311 codewhale_workflow::QualifiedFleetId {
7312 name: "glm-solo".to_string(),
7313 origin: "workspace".to_string(),
7314 },
7315 Some(crate::fleet::exact::StaticFleetRouter::new(
7316 r#"{"reasoning":"low"}"#,
7317 )),
7318 )
7319 };
7320
7321 assert_eq!(
7322 first.snapshot().router(),
7323 second.snapshot().router(),
7324 "both fleets reference the identical captured router service"
7325 );
7326 assert_ne!(
7327 first.snapshot().fleet().qualified(),
7328 second.snapshot().fleet().qualified()
7329 );
7330
7331 // Both actually route through it, and both receipts name it.
7332 for (operation, expected) in [(&first, "max"), (&second, "low")] {
7333 let mut request = exact_write_task_request("builder");
7334 let binding = bind_exact_fleet_task_request(operation, exact_session(), &mut request)
7335 .expect("bind");
7336 let receipt = route_admitted_exact_task(operation, &binding, &mut request)
7337 .await
7338 .expect("routing");
7339 assert_eq!(receipt.effective_reasoning, expected);
7340 assert_eq!(
7341 receipt.router.as_ref().expect("router").qualified(),
7342 "workspace/luna-low"
7343 );
7344 }
7345 }
7346
7347 /// Editing the saved Fleet mid-run must not move the running Workflow's
7348 /// routes. The snapshot owns copies; only the next Workflow sees the edit.
7349 #[tokio::test]
7350 async fn editing_the_fleet_file_after_start_does_not_move_a_running_route() {
7351 let tmp = tempfile::tempdir().expect("tmp");
7352 let fleets = tmp.path().join("fleets");
7353 std::fs::create_dir_all(&fleets).expect("fleets dir");
7354 let path = fleets.join("glm-pair.toml");
7355 std::fs::write(&path, EXACT_GLM_FLEET).expect("write fleet");
7356
7357 let roots = vec![codewhale_workflow::FleetSearchRoot::new(
7358 "workspace",
7359 tmp.path(),
7360 )];
7361 let (document, id) =
7362 codewhale_workflow::FleetDocument::load_by_name("glm-pair", &roots).expect("load");
7363 let operation = crate::fleet::exact::ExactFleetWorkflow::for_tests(
7364 &document,
7365 id,
7366 Some(crate::fleet::exact::StaticFleetRouter::new(
7367 r#"{"reasoning":"max"}"#,
7368 )),
7369 );
7370 let started_hash = operation.snapshot().content_hash().to_string();
7371
7372 // The operator rewrites the saved Fleet mid-run.
7373 std::fs::write(
7374 &path,
7375 EXACT_GLM_FLEET
7376 .replace(
7377 "model = \"glm-5\"\nreasoning = \"auto\"",
7378 "model = \"glm-4\"\nreasoning = \"off\"",
7379 )
7380 .replace("permissions = \"read_write\"", "permissions = \"full\""),
7381 )
7382 .expect("rewrite fleet");
7383
7384 let mut request = exact_write_task_request("builder");
7385 let binding = bind_exact_fleet_task_request(&operation, exact_session(), &mut request)
7386 .expect("bind after the edit");
7387 route_admitted_exact_task(&operation, &binding, &mut request)
7388 .await
7389 .expect("launch after the edit");
7390
7391 let member = operation
7392 .snapshot()
7393 .member("implementer")
7394 .expect("snapshot entry");
7395 assert_eq!(
7396 member.route.model, "glm-5",
7397 "the in-flight snapshot must keep the model it started with"
7398 );
7399 assert_eq!(request.thinking.as_deref(), Some("max"));
7400 assert_eq!(
7401 request.write_authority.as_deref(),
7402 Some("workspace_write"),
7403 "the edit must not widen the running ceiling to `full`"
7404 );
7405 assert_eq!(operation.snapshot().content_hash(), started_hash);
7406
7407 // A fresh Workflow does see the edit.
7408 let (reloaded, reloaded_id) =
7409 codewhale_workflow::FleetDocument::load_by_name("glm-pair", &roots).expect("reload");
7410 let next = crate::fleet::exact::ExactFleetWorkflow::for_tests(
7411 &reloaded,
7412 reloaded_id,
7413 Some(crate::fleet::exact::StaticFleetRouter::new(
7414 r#"{"reasoning":"max"}"#,
7415 )),
7416 );
7417 assert_eq!(
7418 next.snapshot()
7419 .member("implementer")
7420 .expect("snapshot entry")
7421 .route
7422 .model,
7423 "glm-4"
7424 );
7425 assert_ne!(next.snapshot().content_hash(), started_hash);
7426 }
7427
7428 #[test]
7429 fn named_fleet_rejects_unknown_workflow_role_before_spawn() {
7430 let fleet = FleetRoleMap::from_pairs([("scout", "scout")]).expect("fleet");
7431 let mut request = TaskRequest {
7432 description: "fix it".to_string(),
7433 subagent_type: None,
7434 role: Some("wizard".to_string()),
7435 profile: None,
7436 model: None,
7437 model_strength: None,
7438 thinking: None,
7439 cwd: None,
7440 worktree: false,
7441 write_authority: Some("read_only".to_string()),
7442 write_roots: Vec::new(),
7443 exact_files: Vec::new(),
7444 coordination_contracts: Vec::new(),
7445 dependencies: Vec::new(),
7446 acceptance: Vec::new(),
7447 allowed_tools: None,
7448 disallowed_tools: Vec::new(),
7449 max_depth: None,
7450 token_budget: None,
7451 max_steps: None,
7452 wall_time_secs: None,
7453 response_schema: None,
7454 schema_repair_attempts: None,
7455 label: None,
7456 phase: None,
7457 };
7458
7459 let err = apply_named_fleet_to_task_request(Some(&fleet), &mut request)
7460 .expect_err("unknown role should fail");
7461 assert!(
7462 err.to_string().contains("unknown fleet role `wizard`"),
7463 "{err}"
7464 );
7465 }
7466
7467 #[test]
7468 fn declarative_leaf_budget_reaches_task_runtime_options() {
7469 let source = r#"
7470 workflow({
7471 "goal": "bound one child",
7472 "nodes": [{
7473 "agent": {
7474 "id": "bounded",
7475 "prompt": "Inspect bounded evidence.",
7476 "budget": { "max_tokens": 5000, "max_steps": 4, "timeout_secs": 90 }
7477 }
7478 }]
7479 });
7480 "#;
7481
7482 let adapted = adapt_workflow_source(source, None).expect("lower bounded leaf");
7483 assert!(
7484 adapted.source.contains("tokenBudget: 5000"),
7485 "{}",
7486 adapted.source
7487 );
7488 assert!(adapted.source.contains("maxSteps: 4"), "{}", adapted.source);
7489 assert!(
7490 adapted.source.contains("wallTimeSecs: 90"),
7491 "{}",
7492 adapted.source
7493 );
7494 }
7495
7496 #[tokio::test]
7497 #[allow(clippy::await_holding_lock)]
7498 async fn declarative_max_steps_zero_runs_without_a_turn_cap() {
7499 let _retry_guard = workflow_test_retry_guard();
7500 let tmp = tempfile::tempdir().expect("tempdir");
7501 let ctx = ToolContext::new(tmp.path().to_path_buf());
7502 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
7503 let (client, calls) = fake_chat_client("Completed without a turn cap.").await;
7504 let runtime = SubAgentRuntime::new(
7505 client,
7506 "deepseek-v4-flash".to_string(),
7507 ctx.clone(),
7508 true,
7509 None,
7510 manager.clone(),
7511 );
7512 let tool = WorkflowTool::new(manager, runtime);
7513
7514 let result = tool
7515 .execute(
7516 json!({
7517 "action": "run",
7518 "script": r#"
7519 workflow({
7520 "goal": "prove zero means an unbounded child loop",
7521 "nodes": [{
7522 "agent": {
7523 "id": "zero-step",
7524 "prompt": "Complete this task.",
7525 "budget": { "max_steps": 0, "timeout_secs": 90 }
7526 }
7527 }]
7528 });
7529 "#
7530 }),
7531 &ctx,
7532 )
7533 .await
7534 .expect("workflow returns its terminal receipt");
7535 let payload: Value = serde_json::from_str(&result.content).expect("workflow JSON");
7536
7537 assert_eq!(
7538 calls.load(Ordering::SeqCst),
7539 1,
7540 "an unbounded child must be allowed to start a model turn"
7541 );
7542 assert_eq!(payload["status"], "completed", "{payload}");
7543 assert_eq!(
7544 payload["execution"]["leaf_results"][0]["status"], "succeeded",
7545 "{payload}"
7546 );
7547 }
7548
7549 #[tokio::test]
7550 #[allow(clippy::await_holding_lock)]
7551 async fn role_only_leaf_omits_type_and_resolves_through_named_fleet() {
7552 let _retry_guard = workflow_test_retry_guard();
7553 let _env_lock = crate::test_support::lock_test_env();
7554 let tmp = tempfile::tempdir().expect("tempdir");
7555 let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", tmp.path());
7556 let fleet_dir = tmp.path().join("fleets");
7557 std::fs::create_dir_all(&fleet_dir).expect("fleet dir");
7558 std::fs::write(
7559 fleet_dir.join("role-only-test.toml"),
7560 r#"
7561 name = "role-only-test"
7562
7563 [roles]
7564 scout = "scout"
7565 reviewer = "reviewer"
7566 "#,
7567 )
7568 .expect("role-only fleet");
7569 let source = r#"
7570 export default workflow({
7571 "goal": "resolve a role-only child",
7572 "nodes": [
7573 {
7574 "agent": {
7575 "id": "scout-source",
7576 "prompt": "Inspect the source without editing.",
7577 "role": "scout",
7578 "mode": "read_only"
7579 }
7580 }
7581 ]
7582 });
7583 "#;
7584
7585 let adapted = adapt_workflow_source(source, None).expect("lower role-only workflow");
7586 assert!(adapted.source.contains("role: \"scout\""));
7587 assert!(
7588 !adapted.source.contains("type:"),
7589 "Fleet-addressed leaves must defer runtime type to the roster:\n{}",
7590 adapted.source
7591 );
7592 let non_role = adapt_workflow_source(
7593 r#"workflow({
7594 "goal": "default non-role child",
7595 "nodes": [{ "agent": { "id": "audit", "prompt": "Audit only." } }]
7596 });"#,
7597 None,
7598 )
7599 .expect("lower non-role workflow");
7600 assert!(
7601 non_role.source.contains("type: \"review\""),
7602 "non-role read-only leaves retain the review default:\n{}",
7603 non_role.source
7604 );
7605 let explicit_type_source = r#"
7606 workflow({
7607 "goal": "preserve an authored role type",
7608 "nodes": [{
7609 "agent": {
7610 "id": "review-source",
7611 "prompt": "Review the source without editing.",
7612 "agent_type": "review",
7613 "role": "reviewer",
7614 "mode": "read_only"
7615 }
7616 }]
7617 });
7618 "#;
7619 let explicit_type = adapt_workflow_source(explicit_type_source, None)
7620 .expect("lower explicitly typed Fleet role");
7621 assert!(
7622 explicit_type.source.contains("type: \"review\""),
7623 "an authored non-General type must remain a validated override:\n{}",
7624 explicit_type.source
7625 );
7626
7627 let ctx = ToolContext::new(tmp.path().to_path_buf());
7628 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
7629 let (client, calls) = fake_chat_client("scout evidence").await;
7630 let runtime = SubAgentRuntime::new(
7631 client,
7632 "deepseek-v4-flash".to_string(),
7633 ctx.clone(),
7634 true,
7635 None,
7636 manager,
7637 );
7638 let tool = WorkflowTool::new(runtime.manager.clone(), runtime);
7639 let result = tool
7640 .execute(
7641 json!({
7642 "action": "run",
7643 "script": source,
7644 "fleet": "role-only-test"
7645 }),
7646 &ctx,
7647 )
7648 .await
7649 .expect("role-only workflow should resolve through its named Fleet");
7650 let payload: Value = serde_json::from_str(&result.content).expect("workflow JSON");
7651
7652 assert_eq!(payload["status"], "completed", "{payload}");
7653 assert_eq!(calls.load(Ordering::SeqCst), 1);
7654 let started = payload["events"]
7655 .as_array()
7656 .expect("typed events")
7657 .iter()
7658 .find(|event| event["type"] == "task_started")
7659 .expect("task_started receipt");
7660 assert_eq!(started["role"], "explore");
7661 assert_eq!(started["profile"], "scout");
7662 assert_eq!(started["resolved_profile"], "scout");
7663
7664 let explicit_result = tool
7665 .execute(
7666 json!({
7667 "action": "run",
7668 "script": explicit_type_source,
7669 "fleet": "role-only-test"
7670 }),
7671 &ctx,
7672 )
7673 .await
7674 .expect("matching explicit role type should remain valid");
7675 let explicit_payload: Value =
7676 serde_json::from_str(&explicit_result.content).expect("workflow JSON");
7677 assert_eq!(
7678 explicit_payload["status"], "completed",
7679 "{explicit_payload}"
7680 );
7681 assert_eq!(calls.load(Ordering::SeqCst), 2);
7682
7683 let conflicting_result = tool
7684 .execute(
7685 json!({
7686 "action": "run",
7687 "script": r#"workflow({
7688 "goal": "reject a conflicting authored type",
7689 "nodes": [{ "agent": {
7690 "id": "bad-scout",
7691 "prompt": "Review as a scout.",
7692 "agent_type": "review",
7693 "role": "scout",
7694 "mode": "read_only"
7695 } }]
7696 });"#,
7697 "fleet": "role-only-test"
7698 }),
7699 &ctx,
7700 )
7701 .await
7702 .expect("conflicting type returns a terminal workflow record");
7703 let conflicting_payload: Value =
7704 serde_json::from_str(&conflicting_result.content).expect("workflow JSON");
7705 assert_eq!(
7706 conflicting_payload["status"], "failed",
7707 "{conflicting_payload}"
7708 );
7709 assert!(
7710 conflicting_payload["error"]
7711 .as_str()
7712 .is_some_and(|error| error
7713 .contains("Fleet role conflicts with the explicit legacy agent type")),
7714 "{conflicting_payload}"
7715 );
7716 assert_eq!(
7717 calls.load(Ordering::SeqCst),
7718 2,
7719 "conflicting explicit type must fail before the provider"
7720 );
7721 }
7722
7723 #[test]
7724 fn parallel_write_children_default_to_worktree_isolation() {
7725 // #4120: write-capable parallel leaves get worktree: true by default.
7726 let source = r#"
7727 export default workflow({
7728 "goal": "parallel write isolation default",
7729 "nodes": [
7730 {
7731 "branch": {
7732 "id": "implement",
7733 "parallel": true,
7734 "children": [
7735 {
7736 "agent": {
7737 "id": "left",
7738 "prompt": "Patch left lane",
7739 "agent_type": "implementer",
7740 "mode": "read_write",
7741 "file_scope": ["src/left.rs"]
7742 }
7743 },
7744 {
7745 "agent": {
7746 "id": "right",
7747 "prompt": "Patch right lane",
7748 "agent_type": "implementer",
7749 "mode": "read_write",
7750 "file_scope": ["src/right.rs"]
7751 }
7752 }
7753 ]
7754 }
7755 }
7756 ]
7757 });
7758 "#;
7759 let adapted = adapt_workflow_source(source, None).expect("lower parallel write workflow");
7760 let spec = adapted.spec.expect("declarative spec");
7761 let WorkflowNode::BranchSet(branch) = &spec.nodes[0] else {
7762 panic!("expected branch_set");
7763 };
7764 assert!(branch.parallel);
7765 for child in &branch.children {
7766 let WorkflowNode::Leaf(leaf) = child else {
7767 panic!("expected leaf");
7768 };
7769 assert!(leaf_is_write_capable(leaf));
7770 assert!(
7771 leaf_wants_worktree(leaf, true),
7772 "parallel write leaf {} should default to worktree",
7773 leaf.id
7774 );
7775 assert_eq!(leaf.isolation, IsolationMode::Auto);
7776 }
7777 assert!(
7778 adapted.source.contains("worktree: true"),
7779 "lowered JS should request worktree isolation:\n{}",
7780 adapted.source
7781 );
7782 // Both parallel children should carry the worktree flag.
7783 assert_eq!(
7784 adapted.source.matches("worktree: true").count(),
7785 2,
7786 "each parallel write child should get worktree: true:\n{}",
7787 adapted.source
7788 );
7789 assert_eq!(
7790 adapted
7791 .source
7792 .matches("writeAuthority: \"worktree_write\"")
7793 .count(),
7794 2,
7795 "each isolated writer should carry enforced worktree authority:\n{}",
7796 adapted.source
7797 );
7798 assert!(adapted.source.contains("writeRoots: [\"src/left.rs\"]"));
7799 assert!(adapted.source.contains("writeRoots: [\"src/right.rs\"]"));
7800 }
7801
7802 #[test]
7803 fn parallel_write_same_worktree_requires_explicit_shared_isolation() {
7804 // #4120: isolation: shared is the approved same-worktree override.
7805 let source = r#"
7806 export default workflow({
7807 "goal": "parallel write same-worktree override",
7808 "nodes": [
7809 {
7810 "branch": {
7811 "id": "implement",
7812 "parallel": true,
7813 "children": [
7814 {
7815 "agent": {
7816 "id": "shared-writer",
7817 "prompt": "Patch in the parent checkout",
7818 "agent_type": "implementer",
7819 "mode": "read_write",
7820 "isolation": "shared",
7821 "file_scope": ["src/shared.rs"]
7822 }
7823 },
7824 {
7825 "agent": {
7826 "id": "isolated-writer",
7827 "prompt": "Patch in a worktree",
7828 "agent_type": "implementer",
7829 "mode": "read_write",
7830 "isolation": "worktree",
7831 "file_scope": ["src/isolated.rs"]
7832 }
7833 }
7834 ]
7835 }
7836 }
7837 ]
7838 });
7839 "#;
7840 let adapted =
7841 adapt_workflow_source(source, None).expect("lower same-worktree override workflow");
7842 let spec = adapted.spec.expect("declarative spec");
7843 let WorkflowNode::BranchSet(branch) = &spec.nodes[0] else {
7844 panic!("expected branch_set");
7845 };
7846 let leaves: Vec<&LeafSpec> = branch
7847 .children
7848 .iter()
7849 .map(|child| match child {
7850 WorkflowNode::Leaf(leaf) => leaf,
7851 _ => panic!("expected leaf"),
7852 })
7853 .collect();
7854 assert_eq!(leaves[0].isolation, IsolationMode::Shared);
7855 assert!(
7856 !leaf_wants_worktree(leaves[0], true),
7857 "explicit shared should keep same-worktree"
7858 );
7859 assert_eq!(leaves[1].isolation, IsolationMode::Worktree);
7860 assert!(leaf_wants_worktree(leaves[1], true));
7861
7862 // Only the explicit worktree child should emit worktree: true.
7863 assert_eq!(
7864 adapted.source.matches("worktree: true").count(),
7865 1,
7866 "same-worktree override must not force worktree on shared leaf:\n{}",
7867 adapted.source
7868 );
7869 assert!(
7870 adapted.source.contains("shared-writer") && adapted.source.contains("isolated-writer"),
7871 "both children should still be lowered:\n{}",
7872 adapted.source
7873 );
7874 assert!(
7875 adapted
7876 .source
7877 .contains("writeAuthority: \"workspace_write\"")
7878 );
7879 assert!(
7880 adapted
7881 .source
7882 .contains("writeAuthority: \"worktree_write\"")
7883 );
7884 }
7885
7886 #[test]
7887 fn parallel_read_only_children_do_not_default_to_worktree() {
7888 let source = r#"
7889 export default workflow({
7890 "goal": "parallel read-only stays shared",
7891 "nodes": [
7892 {
7893 "branch": {
7894 "id": "audit",
7895 "parallel": true,
7896 "children": [
7897 {
7898 "agent": {
7899 "id": "review-a",
7900 "prompt": "Review A",
7901 "agent_type": "review",
7902 "mode": "read_only"
7903 }
7904 },
7905 {
7906 "agent": {
7907 "id": "review-b",
7908 "prompt": "Review B",
7909 "agent_type": "verifier",
7910 "mode": "read_only"
7911 }
7912 }
7913 ]
7914 }
7915 }
7916 ]
7917 });
7918 "#;
7919 let adapted = adapt_workflow_source(source, None).expect("lower parallel read-only");
7920 assert!(
7921 !adapted.source.contains("worktree: true"),
7922 "read-only parallel children should not get worktree isolation:\n{}",
7923 adapted.source
7924 );
7925 assert_eq!(
7926 adapted
7927 .source
7928 .matches("writeAuthority: \"read_only\"")
7929 .count(),
7930 2,
7931 "read-only mode must reach the task authority contract:\n{}",
7932 adapted.source
7933 );
7934 }
7935
7936 #[test]
7937 fn sequential_write_children_do_not_default_to_worktree() {
7938 let source = r#"
7939 export default workflow({
7940 "goal": "sequential write stays shared by default",
7941 "nodes": [
7942 {
7943 "sequence": {
7944 "id": "implement",
7945 "children": [
7946 {
7947 "agent": {
7948 "id": "writer",
7949 "prompt": "Patch sequentially",
7950 "agent_type": "implementer",
7951 "mode": "read_write",
7952 "file_scope": ["src/main.rs"]
7953 }
7954 }
7955 ]
7956 }
7957 }
7958 ]
7959 });
7960 "#;
7961 let adapted = adapt_workflow_source(source, None).expect("lower sequential write");
7962 assert!(
7963 !adapted.source.contains("worktree: true"),
7964 "sequential writes should not default to worktree:\n{}",
7965 adapted.source
7966 );
7967 assert!(
7968 adapted
7969 .source
7970 .contains("writeAuthority: \"workspace_write\"")
7971 );
7972 assert!(adapted.source.contains("writeRoots: [\"src/main.rs\"]"));
7973 }
7974
7975 #[test]
7976 fn write_scope_suffix_globs_lower_to_enforceable_roots() {
7977 let source = r#"workflow({
7978 "goal": "bounded auth patch",
7979 "nodes": [{ "agent": {
7980 "id": "writer",
7981 "prompt": "Patch auth",
7982 "agent_type": "implementer",
7983 "mode": "read_write",
7984 "file_scope": ["./src/auth/**"]
7985 }}]
7986 });"#;
7987 let adapted = adapt_workflow_source(source, None).expect("lower trailing glob scope");
7988 assert!(
7989 adapted.source.contains("writeRoots: [\"src/auth\"]"),
7990 "runtime claim must contain src/auth/login.rs:\n{}",
7991 adapted.source
7992 );
7993
7994 let unsupported = r#"workflow({
7995 "goal": "reject ambiguous glob",
7996 "nodes": [{ "agent": {
7997 "id": "writer",
7998 "prompt": "Patch auth",
7999 "agent_type": "implementer",
8000 "mode": "read_write",
8001 "file_scope": ["src/*/auth"]
8002 }}]
8003 });"#;
8004 let error = match adapt_workflow_source(unsupported, None) {
8005 Ok(_) => panic!("internal globs cannot become literal runtime roots"),
8006 Err(error) => error.to_string(),
8007 };
8008 assert!(error.contains("unsupported file_scope"), "{error}");
8009 }
8010
8011 #[test]
8012 fn write_leaves_require_scope_and_reduce_stays_read_only() {
8013 let unscoped_writer = r#"workflow({
8014 "goal": "reject an unbounded writer",
8015 "nodes": [{ "agent": {
8016 "id": "writer",
8017 "prompt": "Patch it",
8018 "agent_type": "implementer",
8019 "mode": "read_write"
8020 }}]
8021 });"#;
8022 let error = match adapt_workflow_source(unscoped_writer, None) {
8023 Ok(_) => panic!("write-capable leaves must declare file_scope"),
8024 Err(error) => error.to_string(),
8025 };
8026 assert!(error.contains("declares no file_scope"), "{error}");
8027
8028 let reduce = r#"workflow({
8029 "goal": "reduce read-only evidence",
8030 "nodes": [{ "reduce": {
8031 "id": "summary",
8032 "inputs": [],
8033 "prompt": "Summarize the evidence"
8034 }}]
8035 });"#;
8036 let adapted = adapt_workflow_source(reduce, None).expect("lower reduce");
8037 assert!(adapted.source.contains("type: \"plan\""));
8038 assert!(adapted.source.contains("writeAuthority: \"read_only\""));
8039 }
8040
8041 #[test]
8042 fn inline_script_and_source_path_are_mutually_exclusive() {
8043 let ctx = ToolContext::new(".");
8044 let err = workflow_source(
8045 &json!({
8046 "script": "return 1;",
8047 "source_path": "workflow.js"
8048 }),
8049 &ctx,
8050 )
8051 .unwrap_err();
8052 assert!(
8053 err.to_string()
8054 .contains("exactly one of script, source_path, or plan"),
8055 "{err}"
8056 );
8057 }
8058
8059 #[test]
8060 fn structured_plan_lowers_to_parallel_not_promise_all() {
8061 // #4124: planner plan → JS with parallel() partial-success semantics.
8062 let ctx = ToolContext::new(".");
8063 let source = workflow_source(
8064 &json!({
8065 "plan": {
8066 "goal": "audit two independent scopes",
8067 "risk": "read_only",
8068 "max_children": 8,
8069 "token_budget": 120000,
8070 "phases": [{
8071 "id": "scout",
8072 "title": "Scout",
8073 "children": [
8074 {
8075 "id": "left",
8076 "label": "left-lane",
8077 "prompt": "Inspect crates/left",
8078 "type": "explore"
8079 },
8080 {
8081 "id": "right",
8082 "prompt": "Inspect crates/right",
8083 "type": "explore"
8084 }
8085 ]
8086 }]
8087 }
8088 }),
8089 &ctx,
8090 )
8091 .expect("structured plan should lower");
8092
8093 assert!(
8094 source.source.contains("await parallel(["),
8095 "lowered JS must use parallel():\n{}",
8096 source.source
8097 );
8098 assert!(
8099 !source.source.contains("Promise.all"),
8100 "lowered JS must not use raw Promise.all:\n{}",
8101 source.source
8102 );
8103 assert!(
8104 source.source.contains("() => task("),
8105 "parallel slots should be thunks:\n{}",
8106 source.source
8107 );
8108 let spec = source.spec.expect("plan should produce WorkflowSpec");
8109 assert_eq!(spec.goal, "audit two independent scopes");
8110 assert_eq!(spec.budget.max_tokens, Some(120000));
8111 assert_eq!(spec.nodes.len(), 1);
8112 let WorkflowNode::BranchSet(branch) = &spec.nodes[0] else {
8113 panic!("expected parallel branch for multi-child phase");
8114 };
8115 assert!(branch.parallel);
8116 assert_eq!(branch.children.len(), 2);
8117 }
8118
8119 #[test]
8120 fn structured_plan_child_cwd_lowers_to_task_cwd() {
8121 // #6232: plan children accept `cwd` (repository-relative, like
8122 // task({cwd})) so multi-repo workspaces can disambiguate the child
8123 // repository — including the worktree root the parallel-write default
8124 // (#4120) resolves. A blank value is refused, never lowered.
8125 let ctx = ToolContext::new(".");
8126 let source = workflow_source(
8127 &json!({
8128 "plan": {
8129 "goal": "patch two repos",
8130 "risk": "writes",
8131 "phases": [{
8132 "id": "build",
8133 "children": [
8134 {
8135 "id": "a",
8136 "prompt": "Patch repo A",
8137 "type": "implement",
8138 "file_scope": ["src/**"],
8139 "cwd": "repos/a"
8140 },
8141 {
8142 "id": "b",
8143 "prompt": "Patch repo B",
8144 "type": "implement",
8145 "file_scope": ["src/**"],
8146 "cwd": "repos/b"
8147 }
8148 ]
8149 }]
8150 }
8151 }),
8152 &ctx,
8153 )
8154 .expect("structured plan with child cwd should lower");
8155 assert!(
8156 source.source.contains(r#"cwd: "repos/a""#),
8157 "child cwd must reach the lowered task() call:\n{}",
8158 source.source
8159 );
8160 assert!(
8161 source.source.contains(r#"cwd: "repos/b""#),
8162 "child cwd must reach the lowered task() call:\n{}",
8163 source.source
8164 );
8165
8166 let blank = workflow_source(
8167 &json!({
8168 "plan": {
8169 "goal": "blank cwd",
8170 "risk": "read_only",
8171 "children": [
8172 {
8173 "id": "blank",
8174 "prompt": "Inspect",
8175 "type": "explore",
8176 "cwd": " "
8177 }
8178 ]
8179 }
8180 }),
8181 &ctx,
8182 );
8183 let err = blank
8184 .expect_err("blank child cwd must be refused")
8185 .to_string();
8186 assert!(err.contains("cwd"), "{err}");
8187 }
8188
8189 #[test]
8190 fn structured_plan_validation_errors_are_typed() {
8191 let ctx = ToolContext::new(".");
8192 let missing_goal = workflow_source(
8193 &json!({
8194 "plan": {
8195 "goal": " ",
8196 "children": [{ "prompt": "do work" }]
8197 }
8198 }),
8199 &ctx,
8200 )
8201 .unwrap_err();
8202 assert!(missing_goal.to_string().contains("goal"), "{missing_goal}");
8203
8204 let over_limit = workflow_source(
8205 &json!({
8206 "plan": {
8207 "goal": "too many children",
8208 "max_children": 1,
8209 "children": [
8210 { "id": "a", "prompt": "one" },
8211 { "id": "b", "prompt": "two" }
8212 ]
8213 }
8214 }),
8215 &ctx,
8216 )
8217 .unwrap_err();
8218 assert!(
8219 over_limit.to_string().contains("max_children"),
8220 "{over_limit}"
8221 );
8222
8223 let bad_type = workflow_source(
8224 &json!({
8225 "plan": {
8226 "goal": "bad type",
8227 "children": [{ "prompt": "x", "type": "wizard" }]
8228 }
8229 }),
8230 &ctx,
8231 )
8232 .unwrap_err();
8233 assert!(
8234 bad_type
8235 .to_string()
8236 .contains("Invalid sub-agent type 'wizard'"),
8237 "{bad_type}"
8238 );
8239
8240 let exclusive = workflow_source(
8241 &json!({
8242 "script": "return 1;",
8243 "plan": { "goal": "x", "children": [{ "prompt": "y" }] }
8244 }),
8245 &ctx,
8246 )
8247 .unwrap_err();
8248 assert!(
8249 exclusive
8250 .to_string()
8251 .contains("exactly one of script, source_path, or plan"),
8252 "{exclusive}"
8253 );
8254 }
8255
8256 #[test]
8257 fn plan_child_type_shares_the_agent_tool_option_vocabulary() {
8258 // #5035: type values accepted by direct Agent dispatch must not be
8259 // rejected by Workflow plan authoring; aliases normalize onto the IR.
8260 for (alias, expected) in [
8261 ("worker", AgentType::General),
8262 ("delegate", AgentType::General),
8263 ("scout", AgentType::Explore),
8264 ("Explorer", AgentType::Explore),
8265 ("planner", AgentType::Plan),
8266 ("awaiter", AgentType::Plan),
8267 ("reviewer", AgentType::Review),
8268 ("consultant", AgentType::Review),
8269 ("oracle", AgentType::Review),
8270 ("advisor", AgentType::Review),
8271 ("builder", AgentType::Implementer),
8272 ("verifier", AgentType::Verifier),
8273 ] {
8274 assert_eq!(
8275 parse_plan_agent_type(Some(alias))
8276 .unwrap_or_else(|err| panic!("{alias} rejected: {err}")),
8277 expected,
8278 "{alias}"
8279 );
8280 }
8281
8282 // Typos fail with the Agent tool's error contract and the full set.
8283 let typo = parse_plan_agent_type(Some("wizard"))
8284 .unwrap_err()
8285 .to_string();
8286 assert!(typo.contains("Invalid sub-agent type 'wizard'"), "{typo}");
8287 assert!(
8288 typo.contains("worker, scout, planner, reviewer, builder"),
8289 "{typo}"
8290 );
8291 assert!(
8292 typo.contains("consultant/oracle/advisor"),
8293 "accepted advisory aliases must remain visible in the guidance: {typo}"
8294 );
8295
8296 // `custom` is Agent-only; the rejection says why and what to use.
8297 let custom = parse_plan_agent_type(Some("custom"))
8298 .unwrap_err()
8299 .to_string();
8300 assert!(
8301 custom.contains("Invalid sub-agent type 'custom'"),
8302 "{custom}"
8303 );
8304 assert!(custom.contains("allowed_tools"), "{custom}");
8305 }
8306
8307 #[test]
8308 fn declarative_parallel_branch_uses_parallel_helper() {
8309 let source = r#"
8310 export default workflow({
8311 "goal": "partial success fan-out",
8312 "nodes": [
8313 {
8314 "branch": {
8315 "id": "fan",
8316 "parallel": true,
8317 "children": [
8318 { "agent": { "id": "a", "prompt": "A", "agent_type": "explore", "mode": "read_only" } },
8319 { "agent": { "id": "b", "prompt": "B", "agent_type": "explore", "mode": "read_only" } }
8320 ]
8321 }
8322 }
8323 ]
8324 });
8325 "#;
8326 let adapted = adapt_workflow_source(source, None).expect("lower declarative");
8327 assert!(
8328 adapted.source.contains("await parallel(["),
8329 "declarative parallel must lower via parallel():\n{}",
8330 adapted.source
8331 );
8332 assert!(
8333 !adapted.source.contains("Promise.all"),
8334 "must not emit raw Promise.all:\n{}",
8335 adapted.source
8336 );
8337 }
8338
8339 #[test]
8340 fn source_path_must_stay_inside_workspace_without_trust_mode() {
8341 let workspace = tempfile::tempdir().expect("workspace tempdir");
8342 let outside = tempfile::tempdir().expect("outside tempdir");
8343 let outside_path = outside.path().join("outside.workflow.js");
8344 std::fs::write(&outside_path, "return 1;").expect("outside workflow source");
8345 let ctx = ToolContext::new(workspace.path().to_path_buf());
8346
8347 let err = workflow_source(
8348 &json!({
8349 "source_path": outside_path
8350 }),
8351 &ctx,
8352 )
8353 .expect_err("outside source_path should be denied");
8354
8355 assert!(
8356 err.to_string().contains("must stay inside the workspace"),
8357 "{err}"
8358 );
8359 }
8360
8361 #[test]
8362 fn subagent_tool_surface_registers_workflow_and_agent() {
8363 let tmp = tempfile::tempdir().expect("tempdir");
8364 let ctx = ToolContext::new(tmp.path().to_path_buf());
8365 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
8366 let runtime = SubAgentRuntime::new(
8367 stub_client(),
8368 "deepseek-v4-flash".to_string(),
8369 ctx.clone(),
8370 true,
8371 None,
8372 manager.clone(),
8373 );
8374 let registry = ToolRegistryBuilder::new()
8375 .with_subagent_tools(manager, runtime)
8376 .build(ctx);
8377
8378 assert!(registry.contains("workflow"));
8379 assert!(registry.contains("agent"));
8380 assert!(registry.contains("agents/list"));
8381 assert!(registry.contains("agents/message"));
8382 assert!(registry.contains("agents/followup"));
8383 assert!(registry.contains("agents/interrupt"));
8384 assert!(registry.contains("agents/wait"));
8385 assert!(
8386 registry
8387 .to_api_tools()
8388 .iter()
8389 .any(|tool| tool.name == "workflow")
8390 );
8391 }
8392
8393 #[tokio::test]
8394 #[allow(clippy::await_holding_lock)]
8395 async fn workflow_run_dispatches_task_through_subagent_manager() {
8396 let _retry_guard = workflow_test_retry_guard();
8397 let tmp = tempfile::tempdir().expect("tempdir");
8398 let ctx = ToolContext::new(tmp.path().to_path_buf());
8399 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
8400 let (client, calls) = fake_chat_client("child done").await;
8401 let runtime = SubAgentRuntime::new(
8402 client,
8403 "deepseek-v4-flash".to_string(),
8404 ctx.clone(),
8405 true,
8406 None,
8407 manager.clone(),
8408 );
8409 let tool = WorkflowTool::new(manager.clone(), runtime);
8410
8411 let result = tool
8412 .execute(
8413 json!({
8414 "action": "run",
8415 "script": "phase('dispatch'); log('starting child'); const out = await task({ description: 'say done', allowedTools: [], label: 'inspect-child', model: 'deepseek-v4-flash', modelStrength: 'same', thinking: 'low' }); return { out };"
8416 }),
8417 &ctx,
8418 )
8419 .await
8420 .expect("workflow run should complete");
8421 let payload: Value = serde_json::from_str(&result.content).expect("json result");
8422
8423 assert_eq!(payload["status"], "completed", "{payload}");
8424 assert_eq!(payload["result"]["out"], "child done");
8425 assert_eq!(payload["child_ids"].as_array().unwrap().len(), 1);
8426 assert_eq!(calls.load(Ordering::SeqCst), 1);
8427
8428 let child_id = payload["child_ids"][0].as_str().unwrap();
8429 let events = payload["events"].as_array().expect("events array");
8430 assert!(
8431 events
8432 .iter()
8433 .any(|event| event["type"] == "phase_started" && event["title"] == "dispatch"),
8434 "{events:#?}"
8435 );
8436 assert!(
8437 events
8438 .iter()
8439 .any(|event| event["type"] == "log" && event["message"] == "starting child"),
8440 "{events:#?}"
8441 );
8442 assert!(
8443 events.iter().any(|event| event["type"] == "budget_updated"),
8444 "{events:#?}"
8445 );
8446 let task_started = events
8447 .iter()
8448 .find(|event| event["type"] == "task_started")
8449 .expect("task_started event");
8450 assert_eq!(task_started["task_id"], child_id);
8451 assert_eq!(task_started["label"], "inspect-child");
8452 assert!(task_started["profile"].is_null());
8453 assert_eq!(task_started["model"], "deepseek-v4-flash");
8454 assert_eq!(task_started["strength"], "same");
8455 assert_eq!(task_started["thinking"], "low");
8456 assert_eq!(task_started["requested_reasoning"], "low");
8457 assert!(
8458 task_started["effective_reasoning"]
8459 .as_str()
8460 .is_some_and(|value| !value.is_empty()),
8461 "{task_started}"
8462 );
8463 assert_eq!(task_started["resolved_provider"], "deepseek");
8464 assert_eq!(task_started["resolved_model"], "deepseek-v4-flash");
8465 assert_eq!(task_started["route_source"], "task.model");
8466 assert_eq!(task_started["worktree"], false);
8467 assert!(task_started["parent_task_id"].is_null());
8468 assert_eq!(task_started["depth"], 1);
8469 // #4119: workflow identity on spawn / task_started metadata.
8470 assert_eq!(
8471 task_started["workflow_run_id"].as_str(),
8472 payload["run_id"].as_str()
8473 );
8474 assert_eq!(task_started["workflow_phase_id"], "dispatch");
8475 assert_eq!(task_started["workflow_task_label"], "inspect-child");
8476 assert_eq!(task_started["workflow_child_index"], 0);
8477 assert!(
8478 events.iter().any(|event| event["type"] == "task_completed"
8479 && event["task_id"] == child_id
8480 && event["status"] == "succeeded"),
8481 "{events:#?}"
8482 );
8483 let child = manager
8484 .read()
8485 .await
8486 .get_result(child_id)
8487 .expect("child result");
8488 assert_eq!(child.status, SubAgentStatus::Completed);
8489 assert_eq!(child.result.as_deref(), Some("child done"));
8490
8491 // Full receipt chain: the spawn-minted event survives the JSONL
8492 // journal reload, hydrates the live projection, and round-trips into
8493 // history without following a later route or inventing missing usage.
8494 let reloaded = WorkflowWorkspaceState::open(tmp.path());
8495 let persisted = reloaded
8496 .runs
8497 .lock()
8498 .expect("reloaded workflow runs")
8499 .get(payload["run_id"].as_str().expect("run id"))
8500 .cloned()
8501 .expect("persisted run");
8502 let persisted_json = serde_json::to_value(&persisted).expect("persisted run JSON");
8503 let persisted_started = persisted_json["events"]
8504 .as_array()
8505 .and_then(|events| events.iter().find(|event| event["type"] == "task_started"))
8506 .expect("persisted task_started receipt");
8507 assert_eq!(persisted_started["requested_reasoning"], "low");
8508 assert_eq!(
8509 persisted_started["effective_reasoning"],
8510 task_started["effective_reasoning"]
8511 );
8512 assert_eq!(persisted_started["resolved_provider"], "deepseek");
8513 assert_eq!(persisted_started["resolved_model"], "deepseek-v4-flash");
8514 assert_eq!(persisted_started["route_source"], "task.model");
8515
8516 let mut panel =
8517 crate::tui::widgets::workflow_panel::WorkflowPanel::from_run_json(&persisted_json)
8518 .expect("journal should hydrate workflow panel");
8519 let original_receipt = panel
8520 .phases
8521 .iter()
8522 .flat_map(|phase| phase.rows.iter())
8523 .find(|row| row.task_id == child_id)
8524 .map(crate::tui::widgets::workflow_panel::row_receipt_text)
8525 .expect("spawned child receipt");
8526 assert!(original_receipt.contains("deepseek/deepseek-v4-flash"));
8527 assert!(original_receipt.contains("reasoning low→"));
8528 assert!(original_receipt.contains("via task.model"));
8529
8530 panel.apply_json_event(&json!({
8531 "type": "task_started",
8532 "at_ms": 9_000,
8533 "task_id": "later-route",
8534 "label": "later-route",
8535 "resolved_role": "consultant",
8536 "resolved_provider": "moonshot",
8537 "resolved_model": "kimi-k3",
8538 "requested_reasoning": "auto",
8539 "effective_reasoning": "medium",
8540 "route_source": "agent_profile.model",
8541 "worktree": false,
8542 }));
8543 panel.apply_json_event(&json!({
8544 "type": "task_completed",
8545 "at_ms": 9_100,
8546 "task_id": "later-route",
8547 "status": "succeeded"
8548 }));
8549 let unchanged = panel
8550 .phases
8551 .iter()
8552 .flat_map(|phase| phase.rows.iter())
8553 .find(|row| row.task_id == child_id)
8554 .map(crate::tui::widgets::workflow_panel::row_receipt_text)
8555 .expect("original child after later route");
8556 assert_eq!(unchanged, original_receipt);
8557
8558 let history =
8559 crate::tui::widgets::workflow_panel::WorkflowPanel::from_run_json(&panel.to_run_json())
8560 .expect("history receipt round trip");
8561 let later_receipt = history
8562 .phases
8563 .iter()
8564 .flat_map(|phase| phase.rows.iter())
8565 .find(|row| row.task_id == "later-route")
8566 .map(crate::tui::widgets::workflow_panel::row_receipt_text)
8567 .expect("later route history receipt");
8568 assert!(later_receipt.contains("moonshot/kimi-k3"));
8569 assert!(later_receipt.contains("reasoning auto→medium"));
8570 assert!(later_receipt.contains("tokens unknown"));
8571 assert!(!later_receipt.contains("tokens 0"));
8572 }
8573
8574 #[tokio::test]
8575 #[allow(clippy::await_holding_lock)]
8576 async fn named_fleet_run_emits_role_resolved_receipt_and_rejects_unknown_before_provider() {
8577 let _retry_guard = workflow_test_retry_guard();
8578 let _env_lock = crate::test_support::lock_test_env();
8579 let tmp = tempfile::tempdir().expect("tempdir");
8580 let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", tmp.path());
8581 std::fs::create_dir_all(tmp.path().join("fleets")).expect("fleets dir");
8582 std::fs::write(
8583 tmp.path().join("fleets/offline.toml"),
8584 r#"
8585 name = "offline"
8586 [roles]
8587 reviewer = "reviewer"
8588 "#,
8589 )
8590 .expect("named fleet fixture");
8591
8592 let ctx = ToolContext::new(tmp.path().to_path_buf());
8593 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
8594 let (client, calls) = fake_chat_client("role-resolved child").await;
8595 let runtime = SubAgentRuntime::new(
8596 client,
8597 "deepseek-v4-flash".to_string(),
8598 ctx.clone(),
8599 true,
8600 None,
8601 manager.clone(),
8602 );
8603 let tool = WorkflowTool::new(manager, runtime);
8604
8605 let completed = tool
8606 .execute(
8607 json!({
8608 "action": "run",
8609 "fleet": "offline",
8610 "script": "return await task({ description: 'review it', type: 'review', role: 'reviewer', label: 'offline-review' });"
8611 }),
8612 &ctx,
8613 )
8614 .await
8615 .expect("named fleet workflow");
8616 let payload: Value = serde_json::from_str(&completed.content).expect("workflow JSON");
8617 assert_eq!(payload["status"], "completed", "{payload}");
8618 assert_eq!(payload["result"], "role-resolved child");
8619 assert_eq!(calls.load(Ordering::SeqCst), 1);
8620 let started = payload["events"]
8621 .as_array()
8622 .and_then(|events| events.iter().find(|event| event["type"] == "task_started"))
8623 .expect("task_started receipt");
8624 assert_eq!(started["role"], "reviewer");
8625 assert_eq!(started["profile"], "reviewer");
8626 assert_eq!(started["resolved_role"], "reviewer");
8627 assert_eq!(started["resolved_profile"], "reviewer");
8628 assert_eq!(started["resolved_provider"], "deepseek");
8629 assert_eq!(started["resolved_model"], "deepseek-v4-flash");
8630 assert_eq!(started["route_source"], "run.model");
8631 assert!(
8632 payload["events"]
8633 .as_array()
8634 .is_some_and(|events| events.iter().any(|event| event["type"] == "task_completed"))
8635 );
8636
8637 let rejected = tool
8638 .execute(
8639 json!({
8640 "action": "run",
8641 "fleet": "offline",
8642 "script": "return await task({ description: 'must not launch', type: 'review', role: 'wizard' });"
8643 }),
8644 &ctx,
8645 )
8646 .await
8647 .expect("rejected workflow still returns its terminal record");
8648 let rejected: Value = serde_json::from_str(&rejected.content).expect("rejected JSON");
8649 assert_eq!(rejected["status"], "failed", "{rejected}");
8650 assert!(
8651 rejected["error"]
8652 .as_str()
8653 .is_some_and(|error| error.contains("unknown fleet role `wizard`")),
8654 "{rejected}"
8655 );
8656 assert_eq!(
8657 calls.load(Ordering::SeqCst),
8658 1,
8659 "unknown role must fail before a second provider call"
8660 );
8661 }
8662
8663 #[tokio::test]
8664 #[allow(clippy::await_holding_lock)]
8665 async fn workflow_spawn_records_carry_child_index_and_phase_metadata() {
8666 // #4119: sequential children get monotonic workflow_child_index and
8667 // inherit the active phase when task options omit `phase`.
8668 let _retry_guard = workflow_test_retry_guard();
8669 let tmp = tempfile::tempdir().expect("tempdir");
8670 let ctx = ToolContext::new(tmp.path().to_path_buf());
8671 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 4);
8672 let (client, calls) = fake_chat_client("ok").await;
8673 let runtime = SubAgentRuntime::new(
8674 client,
8675 "deepseek-v4-flash".to_string(),
8676 ctx.clone(),
8677 true,
8678 None,
8679 manager.clone(),
8680 );
8681 let tool = WorkflowTool::new(manager.clone(), runtime);
8682
8683 let result = tool
8684 .execute(
8685 json!({
8686 "action": "run",
8687 "script": "phase('alpha'); await task({ description: 'first', type: 'explore', allowedTools: [], label: 'one' }); phase('beta'); await task({ description: 'second', type: 'explore', allowedTools: [], label: 'two', phase: 'beta-explicit' }); return { ok: true };"
8688 }),
8689 &ctx,
8690 )
8691 .await
8692 .expect("workflow run should complete");
8693 let payload: Value = serde_json::from_str(&result.content).expect("json result");
8694 assert_eq!(payload["status"], "completed", "{payload}");
8695 assert_eq!(payload["child_ids"].as_array().unwrap().len(), 2);
8696 assert_eq!(calls.load(Ordering::SeqCst), 2);
8697
8698 let mut started: Vec<&Value> = payload["events"]
8699 .as_array()
8700 .expect("events")
8701 .iter()
8702 .filter(|event| event["type"] == "task_started")
8703 .collect();
8704 started.sort_by_key(|event| event["workflow_child_index"].as_u64().unwrap_or(u64::MAX));
8705 assert_eq!(started.len(), 2, "{started:#?}");
8706
8707 assert_eq!(started[0]["workflow_run_id"], payload["run_id"]);
8708 assert_eq!(started[0]["workflow_phase_id"], "alpha");
8709 assert_eq!(started[0]["workflow_task_label"], "one");
8710 assert_eq!(started[0]["workflow_child_index"], 0);
8711 assert_eq!(started[0]["label"], "one");
8712
8713 assert_eq!(started[1]["workflow_run_id"], payload["run_id"]);
8714 // Explicit task phase wins over the driver's current phase.
8715 assert_eq!(started[1]["workflow_phase_id"], "beta-explicit");
8716 assert_eq!(started[1]["workflow_task_label"], "two");
8717 assert_eq!(started[1]["workflow_child_index"], 1);
8718 assert_eq!(started[1]["label"], "two");
8719 }
8720
8721 #[tokio::test]
8722 #[allow(clippy::await_holding_lock)]
8723 async fn declarative_parallel_spawn_failure_nulls_slot_and_continues() {
8724 // #4124: parallel() is all-settled — a rejected spawn becomes a null slot
8725 // (with a breadcrumb) instead of aborting the rest of the script the way
8726 // raw Promise.all would. Downstream reduce still runs on partial results.
8727 let _retry_guard = workflow_test_retry_guard();
8728 let tmp = tempfile::tempdir().expect("tempdir");
8729 let ctx = ToolContext::new(tmp.path().to_path_buf());
8730 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
8731 let (client, calls) = fake_chat_client("reduce-with-partial").await;
8732 let runtime = SubAgentRuntime::new(
8733 client,
8734 "deepseek-v4-flash".to_string(),
8735 ctx.clone(),
8736 true,
8737 None,
8738 manager,
8739 );
8740 let tool = WorkflowTool::new(runtime.manager.clone(), runtime);
8741
8742 let result = tool
8743 .execute(
8744 json!({
8745 "action": "run",
8746 "script": r#"export default workflow({
8747 "goal": "partial success fan-out",
8748 "nodes": [
8749 {
8750 "branch": {
8751 "id": "parallel",
8752 "parallel": true,
8753 "children": [
8754 {
8755 "agent": {
8756 "id": "bad-profile",
8757 "prompt": "This child should be rejected before model execution.",
8758 "profile": "missing-profile"
8759 }
8760 }
8761 ]
8762 }
8763 },
8764 {
8765 "reduce": {
8766 "id": "summary",
8767 "inputs": ["bad-profile"],
8768 "prompt": "Summarize whatever survived the parallel fan-out."
8769 }
8770 }
8771 ]
8772 });"#
8773 }),
8774 &ctx,
8775 )
8776 .await
8777 .expect("partial-success workflow still returns run record");
8778 let payload: Value = serde_json::from_str(&result.content).expect("json result");
8779
8780 // Receipt honesty (morning-report issue #2): the run keeps its output
8781 // and the reduce still runs, but a dropped slot means the status is
8782 // degraded, never a plain completed.
8783 assert_eq!(payload["status"], "degraded", "{payload}");
8784 let degradation = payload["error"].as_str().expect("degradation surfaced");
8785 assert!(
8786 degradation.contains("result may be partial"),
8787 "{degradation}"
8788 );
8789 assert!(
8790 payload["result"]["bad-profile"].is_null(),
8791 "failed parallel slot should be null: {}",
8792 payload["result"]
8793 );
8794 assert_eq!(payload["result"]["summary"], "reduce-with-partial");
8795 let progress = payload["progress"]
8796 .as_array()
8797 .expect("progress array")
8798 .iter()
8799 .filter_map(|v| v.as_str())
8800 .collect::<Vec<_>>()
8801 .join("\n");
8802 assert!(
8803 progress.contains("missing-profile") && progress.contains("dropped a failed slot"),
8804 "breadcrumb should surface the spawn rejection:\n{progress}"
8805 );
8806 // #5035: partial success is explicit — the rejected slot lands in the
8807 // run record as a structured dispatch failure, not only a log line.
8808 let failures = payload["dispatch_failures"]
8809 .as_array()
8810 .expect("dispatch failures surfaced on the run record");
8811 assert_eq!(failures.len(), 1, "{payload}");
8812 assert!(
8813 failures[0]["message"]
8814 .as_str()
8815 .unwrap_or_default()
8816 .contains("missing-profile"),
8817 "{failures:?}"
8818 );
8819 assert_eq!(
8820 result.metadata.as_ref().expect("metadata")["dispatch_failure_count"],
8821 1
8822 );
8823 assert!(
8824 calls.load(Ordering::SeqCst) >= 1,
8825 "reduce should still run after a null parallel slot"
8826 );
8827 }
8828
8829 #[tokio::test]
8830 #[allow(clippy::await_holding_lock)]
8831 async fn parallel_slots_rejected_by_the_vm_cannot_report_plain_success() {
8832 // Morning-report issue #2: options that fail VM validation throw
8833 // before the driver ever sees a dispatch, and parallel() collapses
8834 // those throws into null slots. The run must classify against the
8835 // slot ledger instead of reporting completed with [null, ...].
8836 let _retry_guard = workflow_test_retry_guard();
8837 let tmp = tempfile::tempdir().expect("tempdir");
8838 let ctx = ToolContext::new(tmp.path().to_path_buf());
8839 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
8840 let (client, calls) = fake_chat_client("unused").await;
8841 let runtime = SubAgentRuntime::new(
8842 client,
8843 "deepseek-v4-flash".to_string(),
8844 ctx.clone(),
8845 true,
8846 None,
8847 manager,
8848 );
8849 let tool = WorkflowTool::new(runtime.manager.clone(), runtime);
8850
8851 let result = tool
8852 .execute(
8853 json!({
8854 "action": "run",
8855 "script": "return await parallel([() => task({ description: 'bad a', type: 'explore', allowedTools: [], cwd: '/absolute/a' }), () => task({ description: 'bad b', type: 'explore', allowedTools: [], cwd: '/absolute/b' })]);"
8856 }),
8857 &ctx,
8858 )
8859 .await
8860 .expect("vm-rejected fan-out still returns the run record");
8861 let payload: Value = serde_json::from_str(&result.content).expect("json result");
8862
8863 assert_eq!(payload["status"], "failed", "{payload}");
8864 let error = payload["error"].as_str().expect("error surfaced");
8865 assert!(
8866 error.contains("all 2 task dispatch(es) were rejected"),
8867 "error should name the total rejection: {error}"
8868 );
8869 let failures = payload["dispatch_failures"]
8870 .as_array()
8871 .expect("collected dispatch failures");
8872 assert_eq!(failures.len(), 2, "{payload}");
8873 for failure in failures {
8874 let message = failure["message"].as_str().expect("failure message");
8875 assert!(message.contains("bounded repo-relative paths"), "{message}");
8876 }
8877 assert_eq!(
8878 calls.load(Ordering::SeqCst),
8879 0,
8880 "no provider call should be spent on vm-rejected slots"
8881 );
8882 }
8883
8884 #[tokio::test]
8885 #[allow(clippy::await_holding_lock)]
8886 async fn partially_dropped_parallel_slots_degrade_the_run() {
8887 // One slot completes, one is rejected before dispatch: the run keeps
8888 // its output but must report degraded, not completed.
8889 let _retry_guard = workflow_test_retry_guard();
8890 let tmp = tempfile::tempdir().expect("tempdir");
8891 let ctx = ToolContext::new(tmp.path().to_path_buf());
8892 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
8893 let (client, calls) = fake_chat_client("child done").await;
8894 let runtime = SubAgentRuntime::new(
8895 client,
8896 "deepseek-v4-flash".to_string(),
8897 ctx.clone(),
8898 true,
8899 None,
8900 manager,
8901 );
8902 let tool = WorkflowTool::new(runtime.manager.clone(), runtime);
8903
8904 let result = tool
8905 .execute(
8906 json!({
8907 "action": "run",
8908 "script": "return await parallel([() => task({ description: 'say done', type: 'explore', allowedTools: [] }), () => task({ description: 'bad slot', type: 'explore', allowedTools: [], cwd: '/absolute/path' })]);"
8909 }),
8910 &ctx,
8911 )
8912 .await
8913 .expect("partially dropped fan-out still returns the run record");
8914 let payload: Value = serde_json::from_str(&result.content).expect("json result");
8915
8916 assert_eq!(payload["status"], "degraded", "{payload}");
8917 let error = payload["error"].as_str().expect("degradation surfaced");
8918 assert!(
8919 error.contains("1 dispatch(es) were rejected"),
8920 "error should count the dropped slot: {error}"
8921 );
8922 assert!(
8923 error.contains("result may be partial"),
8924 "error should warn the result is partial: {error}"
8925 );
8926 let results = payload["result"].as_array().expect("run kept its output");
8927 assert_eq!(results.len(), 2, "{payload}");
8928 assert!(results[1].is_null(), "the rejected slot stays null");
8929 assert!(
8930 calls.load(Ordering::SeqCst) >= 1,
8931 "the healthy slot still ran"
8932 );
8933 }
8934
8935 #[tokio::test]
8936 #[allow(clippy::await_holding_lock)]
8937 async fn parallel_fan_out_with_every_dispatch_rejected_fails_the_run() {
8938 // #5035: when every parallel slot is rejected before dispatch, the run
8939 // must not report overall success that ran nothing — the collected
8940 // per-slot failures surface and the run fails loudly.
8941 let _retry_guard = workflow_test_retry_guard();
8942 let tmp = tempfile::tempdir().expect("tempdir");
8943 let ctx = ToolContext::new(tmp.path().to_path_buf());
8944 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
8945 let (client, calls) = fake_chat_client("unused").await;
8946 let runtime = SubAgentRuntime::new(
8947 client,
8948 "deepseek-v4-flash".to_string(),
8949 ctx.clone(),
8950 true,
8951 None,
8952 manager,
8953 );
8954 let tool = WorkflowTool::new(runtime.manager.clone(), runtime);
8955
8956 let result = tool
8957 .execute(
8958 json!({
8959 "action": "run",
8960 "script": r#"export default workflow({
8961 "goal": "total dispatch failure fan-out",
8962 "nodes": [
8963 {
8964 "branch": {
8965 "id": "parallel",
8966 "parallel": true,
8967 "children": [
8968 {
8969 "agent": {
8970 "id": "bad-one",
8971 "prompt": "Rejected before model execution.",
8972 "profile": "missing-profile"
8973 }
8974 },
8975 {
8976 "agent": {
8977 "id": "bad-two",
8978 "prompt": "Also rejected before model execution.",
8979 "profile": "missing-profile"
8980 }
8981 }
8982 ]
8983 }
8984 }
8985 ]
8986 });"#
8987 }),
8988 &ctx,
8989 )
8990 .await
8991 .expect("total dispatch failure still returns the run record");
8992 let payload: Value = serde_json::from_str(&result.content).expect("json result");
8993
8994 assert_eq!(payload["status"], "failed", "{payload}");
8995 let error = payload["error"].as_str().expect("error surfaced");
8996 assert!(
8997 error.contains("all 2 task dispatch(es) were rejected"),
8998 "error should name the total dispatch failure: {error}"
8999 );
9000 let failures = payload["dispatch_failures"]
9001 .as_array()
9002 .expect("collected dispatch failures");
9003 assert_eq!(failures.len(), 2, "{payload}");
9004 for failure in failures {
9005 let message = failure["message"].as_str().expect("failure message");
9006 assert!(message.contains("missing-profile"), "{message}");
9007 }
9008 assert_eq!(payload["child_ids"].as_array().unwrap().len(), 0);
9009 assert_eq!(
9010 result.metadata.as_ref().expect("metadata")["dispatch_failure_count"],
9011 2
9012 );
9013 assert_eq!(
9014 calls.load(Ordering::SeqCst),
9015 0,
9016 "no provider call should be spent on an all-rejected fan-out"
9017 );
9018 }
9019
9020 #[tokio::test]
9021 #[allow(clippy::await_holding_lock)]
9022 async fn a_token_budget_never_kills_a_child_nor_fails_the_run() {
9023 // #6189 removed token-budget enforcement: usage is tracked, never
9024 // enforced, and no token budget may kill a worker. This drives the
9025 // same end-to-end path the former budget-death test used — a 1-token
9026 // `tokenBudget` on both children, with the fake provider reporting
9027 // more tokens than that cap — and asserts the run completes with both
9028 // results. A regression back to enforcement (children dying of the
9029 // token ceiling, the run failing) fails here.
9030 let _retry_guard = workflow_test_retry_guard();
9031 let tmp = tempfile::tempdir().expect("tempdir");
9032 let ctx = ToolContext::new(tmp.path().to_path_buf());
9033 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
9034 let (client, calls) = fake_chat_client("child done").await;
9035 let runtime = SubAgentRuntime::new(
9036 client,
9037 "deepseek-v4-flash".to_string(),
9038 ctx.clone(),
9039 true,
9040 None,
9041 manager,
9042 );
9043 let tool = WorkflowTool::new(runtime.manager.clone(), runtime);
9044
9045 let result = tool
9046 .execute(
9047 json!({
9048 "action": "run",
9049 "script": "return await parallel([() => task({ description: 'spendy a', type: 'explore', allowedTools: [], tokenBudget: 1 }), () => task({ description: 'spendy b', type: 'explore', allowedTools: [], tokenBudget: 1 })]);"
9050 }),
9051 &ctx,
9052 )
9053 .await
9054 .expect("budgeted fan-out still returns the run record");
9055 let payload: Value = serde_json::from_str(&result.content).expect("json result");
9056
9057 assert_eq!(payload["status"], "completed", "{payload}");
9058 let slots = payload["result"].as_array().expect("run kept its output");
9059 assert_eq!(slots.len(), 2, "{payload}");
9060 assert!(
9061 slots.iter().all(|slot| slot == "child done"),
9062 "a token budget must not stop a child from reporting: {slots:?}"
9063 );
9064 assert_eq!(payload["child_ids"].as_array().unwrap().len(), 2);
9065 assert!(
9066 calls.load(Ordering::SeqCst) >= 2,
9067 "both children should have run and reported usage"
9068 );
9069 }
9070
9071 #[tokio::test]
9072 #[allow(clippy::await_holding_lock)]
9073 async fn a_fan_out_of_script_throws_cannot_report_completed() {
9074 // R9 blocker: thunks that throw without calling `task()` produce
9075 // kind `script`, drop to null, and previously left no host-visible
9076 // trace beyond a log line — no task records, no dispatch failures,
9077 // classify() = None, run recorded Completed. The structured
9078 // every-slot-failed signal must carry the dead fan-out to the host.
9079 let _retry_guard = workflow_test_retry_guard();
9080 let tmp = tempfile::tempdir().expect("tempdir");
9081 let ctx = ToolContext::new(tmp.path().to_path_buf());
9082 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
9083 let (client, calls) = fake_chat_client("unused").await;
9084 let runtime = SubAgentRuntime::new(
9085 client,
9086 "deepseek-v4-flash".to_string(),
9087 ctx.clone(),
9088 true,
9089 None,
9090 manager,
9091 );
9092 let tool = WorkflowTool::new(runtime.manager.clone(), runtime);
9093
9094 let result = tool
9095 .execute(
9096 json!({
9097 "action": "run",
9098 "script": r#"
9099 const results = await parallel([
9100 () => { throw new Error("script slot a died"); },
9101 () => { throw new Error("script slot b died"); },
9102 ]);
9103 return { slots: results };
9104 "#
9105 }),
9106 &ctx,
9107 )
9108 .await
9109 .expect("all-script-throw fan-out still returns the run record");
9110 let payload: Value = serde_json::from_str(&result.content).expect("json result");
9111
9112 assert_eq!(payload["status"], "failed", "{payload}");
9113 let error = payload["error"].as_str().expect("error surfaced");
9114 assert!(
9115 error.contains("no work survived") && error.contains("1 fan-out(s) lost every slot"),
9116 "error should name the dead fan-out: {error}"
9117 );
9118 // The null slots survive on the record; the run refuses the success.
9119 assert_eq!(
9120 payload["result"]["slots"],
9121 json!([null, null]),
9122 "the settled array is preserved verbatim: {}",
9123 payload["result"]
9124 );
9125 let progress = payload["progress"]
9126 .as_array()
9127 .expect("progress array")
9128 .iter()
9129 .filter_map(|value| value.as_str())
9130 .collect::<Vec<_>>()
9131 .join("\n");
9132 assert!(
9133 progress.contains("every slot failed (2 of 2)")
9134 && progress.contains("no work survived this fan-out"),
9135 "the operator breadcrumb must stay in the run log:\n{progress}"
9136 );
9137 assert_eq!(
9138 calls.load(Ordering::SeqCst),
9139 0,
9140 "no provider call should be spent on thunks that never call task()"
9141 );
9142 }
9143
9144 #[tokio::test]
9145 #[allow(clippy::await_holding_lock)]
9146 async fn a_mixed_fan_out_with_surviving_work_stays_degraded_not_failed() {
9147 // One slot runs a real child and survives; one thunk throws. Some
9148 // work survived, so the run degrades with its null preserved — it is
9149 // not a dead fan-out (1 of 2 failed) and must not record failed.
9150 let _retry_guard = workflow_test_retry_guard();
9151 let tmp = tempfile::tempdir().expect("tempdir");
9152 let ctx = ToolContext::new(tmp.path().to_path_buf());
9153 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
9154 let (client, calls) = fake_chat_client("child done").await;
9155 let runtime = SubAgentRuntime::new(
9156 client,
9157 "deepseek-v4-flash".to_string(),
9158 ctx.clone(),
9159 true,
9160 None,
9161 manager,
9162 );
9163 let tool = WorkflowTool::new(runtime.manager.clone(), runtime);
9164
9165 let result = tool
9166 .execute(
9167 json!({
9168 "action": "run",
9169 "script": "return await parallel([() => task({ description: 'healthy slot', type: 'explore', allowedTools: [] }), () => { throw new Error('script slot died'); }]);"
9170 }),
9171 &ctx,
9172 )
9173 .await
9174 .expect("mixed fan-out still returns the run record");
9175 let payload: Value = serde_json::from_str(&result.content).expect("json result");
9176
9177 assert_eq!(payload["status"], "degraded", "{payload}");
9178 let error = payload["error"].as_str().expect("degradation surfaced");
9179 assert!(
9180 error.contains("result may be partial"),
9181 "error should keep the partial-output caveat: {error}"
9182 );
9183 let slots = payload["result"].as_array().expect("run kept its output");
9184 assert_eq!(slots.len(), 2, "{payload}");
9185 assert!(slots[0].is_string() && slots[1].is_null(), "{slots:?}");
9186 assert!(
9187 calls.load(Ordering::SeqCst) >= 1,
9188 "the healthy slot still ran"
9189 );
9190 }
9191
9192 #[tokio::test]
9193 #[allow(clippy::await_holding_lock)]
9194 async fn structured_plan_phases_forward_results_and_stop_on_missing_dependencies() {
9195 let _retry_guard = workflow_test_retry_guard();
9196 for fail_upstream in [false, true] {
9197 let tmp = tempfile::tempdir().expect("tempdir");
9198 let ctx = ToolContext::new(tmp.path().to_path_buf());
9199 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
9200 let (client, calls, bodies) = fake_chat_client_capturing("upstream-output").await;
9201 let runtime = SubAgentRuntime::new(
9202 client,
9203 "deepseek-v4-flash".to_string(),
9204 ctx.clone(),
9205 true,
9206 None,
9207 manager,
9208 );
9209 let tool = WorkflowTool::new(runtime.manager.clone(), runtime);
9210 let mut upstream = vec![json!({
9211 "id": "__proto__", "prompt": "Produce the upstream finding.", "role": "reviewer"
9212 })];
9213 if fail_upstream {
9214 upstream.push(json!({
9215 "id": "missing", "prompt": "This profile does not exist.",
9216 "profile": "required-profile-does-not-exist"
9217 }));
9218 }
9219 let result = tool.execute(json!({
9220 "action": "run",
9221 "plan": {
9222 "goal": "Carry evidence through native workflow phases",
9223 "risk": "read_only",
9224 "token_budget": 1_000,
9225 "phases": [
9226 {"id": "inspect", "children": upstream},
9227 {"id": "finish", "children": [{
9228 "id": "second", "prompt": "Use the upstream finding.", "role": "reviewer"
9229 }]}
9230 ]
9231 }
9232 }), &ctx).await.expect("workflow returns its run receipt");
9233 let payload: Value = serde_json::from_str(&result.content).expect("json result");
9234 assert_eq!(payload["token_budget"], 1_000, "{payload}");
9235 if fail_upstream {
9236 assert_ne!(payload["status"], "completed", "{payload}");
9237 assert_eq!(
9238 calls.load(Ordering::SeqCst),
9239 1,
9240 "dependent child must not run: {payload}"
9241 );
9242 assert!(
9243 payload
9244 .to_string()
9245 .contains("Required Workflow result unavailable: missing"),
9246 "{payload}"
9247 );
9248 } else {
9249 assert_eq!(payload["status"], "completed", "{payload}");
9250 assert_eq!(calls.load(Ordering::SeqCst), 2);
9251 let bodies = bodies.lock().expect("captured bodies");
9252 let downstream = bodies.get(1).expect("dependent provider call").to_string();
9253 assert!(downstream.contains("--- __proto__ ---"), "{downstream}");
9254 assert!(downstream.contains("upstream-output"), "{downstream}");
9255 let phases = payload["events"]
9256 .as_array()
9257 .expect("events")
9258 .iter()
9259 .filter(|event| event["type"] == "phase_started")
9260 .filter_map(|event| event["title"].as_str())
9261 .collect::<Vec<_>>();
9262 assert!(
9263 phases.contains(&"inspect") && phases.contains(&"finish"),
9264 "{phases:?}"
9265 );
9266 let task_phases = payload["events"]
9267 .as_array()
9268 .expect("events")
9269 .iter()
9270 .filter(|event| event["type"] == "task_started")
9271 .filter_map(|event| event["workflow_phase_id"].as_str())
9272 .collect::<Vec<_>>();
9273 assert_eq!(task_phases, ["inspect", "finish"]);
9274 }
9275 }
9276 }
9277
9278 #[tokio::test]
9279 #[allow(clippy::await_holding_lock)]
9280 async fn declarative_dependency_results_are_forwarded_to_downstream_prompt() {
9281 let _retry_guard = workflow_test_retry_guard();
9282 let tmp = tempfile::tempdir().expect("tempdir");
9283 let ctx = ToolContext::new(tmp.path().to_path_buf());
9284 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
9285 let (client, calls, bodies) = fake_chat_client_capturing("upstream-output").await;
9286 let runtime = SubAgentRuntime::new(
9287 client,
9288 "deepseek-v4-flash".to_string(),
9289 ctx.clone(),
9290 true,
9291 None,
9292 manager,
9293 );
9294 let tool = WorkflowTool::new(runtime.manager.clone(), runtime);
9295
9296 let result = tool
9297 .execute(
9298 json!({
9299 "action": "run",
9300 "script": r#"export default workflow({
9301 "goal": "dependency forwarding",
9302 "nodes": [
9303 {
9304 "agent": {
9305 "id": "first",
9306 "prompt": "Produce the upstream finding.",
9307 "agent_type": "review"
9308 }
9309 },
9310 {
9311 "agent": {
9312 "id": "second",
9313 "prompt": "Use the upstream finding.",
9314 "agent_type": "review",
9315 "depends_on_results": ["first"]
9316 }
9317 }
9318 ]
9319 });"#
9320 }),
9321 &ctx,
9322 )
9323 .await
9324 .expect("dependency workflow should complete");
9325 let payload: Value = serde_json::from_str(&result.content).expect("json result");
9326
9327 assert_eq!(payload["status"], "completed", "{payload}");
9328 assert_eq!(payload["execution"]["status"], "succeeded");
9329 assert_eq!(
9330 payload["execution"]["leaf_results"][0]["output"],
9331 "upstream-output"
9332 );
9333 assert_eq!(
9334 payload["execution"]["leaf_results"][1]["output"],
9335 "upstream-output"
9336 );
9337 assert_eq!(calls.load(Ordering::SeqCst), 2);
9338 let bodies = bodies.lock().expect("captured bodies");
9339 let second_body = bodies.get(1).expect("second provider call").to_string();
9340 assert!(second_body.contains("--- first ---"), "{second_body}");
9341 assert!(second_body.contains("upstream-output"), "{second_body}");
9342 }
9343
9344 #[tokio::test]
9345 #[allow(clippy::await_holding_lock)]
9346 async fn workflow_runtime_gates_promote_handoff_and_block_downstream_role() {
9347 let tmp = tempfile::tempdir().expect("tempdir");
9348 let ctx = ToolContext::new(tmp.path().to_path_buf());
9349 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
9350 let runtime = SubAgentRuntime::new(
9351 stub_client(),
9352 "deepseek-v4-flash".to_string(),
9353 ctx.clone(),
9354 true,
9355 None,
9356 manager.clone(),
9357 );
9358 let state = WorkflowWorkspaceState::open(tmp.path());
9359 let run_id = "workflow_gate".to_string();
9360 let gates = vec![GateSpec {
9361 id: "explore-findings".to_string(),
9362 role: "explore".to_string(),
9363 on: GateOn::RoleComplete,
9364 gate: GateKind::Approve,
9365 on_fail: codewhale_workflow::GateOnFail::Block,
9366 blocks_role: Some("implement".to_string()),
9367 max_retries: 0,
9368 artifact_kind: Some("findings".to_string()),
9369 require_explicit_verdict: false,
9370 }];
9371 let spec = WorkflowSpec {
9372 id: Some("gate-fixture".to_string()),
9373 goal: "gate fixture".to_string(),
9374 description: None,
9375 budget: BudgetSpec::default(),
9376 permissions: Default::default(),
9377 model_policy: Default::default(),
9378 promotion_policy: Default::default(),
9379 gates: gates.clone(),
9380 nodes: Vec::new(),
9381 };
9382 state.runs.lock().expect("runs").insert(
9383 run_id.clone(),
9384 WorkflowRunRecord::new(
9385 run_id.clone(),
9386 Some("session-test".to_string()),
9387 None,
9388 None,
9389 Some(&spec),
9390 ),
9391 );
9392 let driver = SubAgentWorkflowDriver::new(
9393 run_id.clone(),
9394 "session-test".to_string(),
9395 manager,
9396 runtime,
9397 state.clone(),
9398 None,
9399 WorkflowFleetBinding::None,
9400 gates,
9401 tmp.path().to_path_buf(),
9402 );
9403
9404 driver.evaluate_gates_for_completed_role(&RuntimeTaskRecord {
9405 agent_id: "explore-agent".to_string(),
9406 label: Some("explore".to_string()),
9407 role: Some("explore".to_string()),
9408 status: IrWorkflowRunStatus::Succeeded,
9409 output: Some("findings: inspect tui exit path".to_string()),
9410 schema_error: None,
9411 usage: None,
9412 });
9413
9414 let mut implementer = TaskRequest {
9415 description: "Use the findings.".to_string(),
9416 subagent_type: Some("implement".to_string()),
9417 role: Some("implement".to_string()),
9418 profile: None,
9419 model: None,
9420 model_strength: None,
9421 thinking: None,
9422 cwd: None,
9423 worktree: false,
9424 write_authority: Some("workspace_write".to_string()),
9425 write_roots: vec!["src".to_string()],
9426 exact_files: Vec::new(),
9427 coordination_contracts: vec!["test-contract".to_string()],
9428 dependencies: Vec::new(),
9429 acceptance: Vec::new(),
9430 allowed_tools: Some(Vec::new()),
9431 disallowed_tools: Vec::new(),
9432 max_depth: None,
9433 token_budget: None,
9434 max_steps: None,
9435 wall_time_secs: None,
9436 response_schema: None,
9437 schema_repair_attempts: None,
9438 label: Some("fix".to_string()),
9439 phase: None,
9440 };
9441 let handoffs = driver
9442 .prepare_request_for_gates(&mut implementer)
9443 .expect("passed gate should admit implementer");
9444 assert_eq!(handoffs.len(), 1, "{handoffs:?}");
9445 assert_eq!(handoffs[0].kind, "findings");
9446 assert_eq!(handoffs[0].from_role, "explore");
9447 assert_eq!(handoffs[0].to_role, "implement");
9448 assert!(
9449 implementer
9450 .description
9451 .contains("Workflow handoff artifacts available"),
9452 "{}",
9453 implementer.description
9454 );
9455 assert!(
9456 implementer.description.contains("inspect tui exit path"),
9457 "{}",
9458 implementer.description
9459 );
9460
9461 driver.evaluate_gates_for_completed_role(&RuntimeTaskRecord {
9462 agent_id: "explore-agent-2".to_string(),
9463 label: Some("explore".to_string()),
9464 role: Some("explore".to_string()),
9465 status: IrWorkflowRunStatus::Failed,
9466 output: Some("explore incomplete".to_string()),
9467 schema_error: None,
9468 usage: None,
9469 });
9470 let mut blocked = TaskRequest {
9471 description: "Try after block.".to_string(),
9472 role: Some("implement".to_string()),
9473 ..implementer.clone()
9474 };
9475 let err = driver
9476 .prepare_request_for_gates(&mut blocked)
9477 .expect_err("blocked gate should reject downstream role");
9478 assert!(err.to_string().contains("explore incomplete"), "{err}");
9479
9480 let run = state
9481 .runs
9482 .lock()
9483 .expect("runs")
9484 .get(&run_id)
9485 .cloned()
9486 .expect("run");
9487 assert!(
9488 run.gate_status
9489 .iter()
9490 .any(|line| line.gate_id == "explore-findings"
9491 && line.state == "blocked"
9492 && line.blocked_reason.as_deref() == Some("explore incomplete")),
9493 "{:?}",
9494 run.gate_status
9495 );
9496 assert!(
9497 run.events
9498 .iter()
9499 .any(|event| event.event_type() == "gate_updated"),
9500 "{:?}",
9501 run.events
9502 );
9503 assert_eq!(
9504 run.events
9505 .iter()
9506 .filter(|event| event.event_type() == "handoff_promoted")
9507 .count(),
9508 1,
9509 "a later blocked gate must not publish another handoff: {:?}",
9510 run.events
9511 );
9512 assert!(
9513 run.events
9514 .iter()
9515 .all(|event| event.event_type() != "handoff_consumed"),
9516 "request preparation alone must not claim consumption: {:?}",
9517 run.events
9518 );
9519 }
9520
9521 #[tokio::test]
9522 async fn workflow_handoff_is_delivered_to_exactly_one_task() {
9523 // LaneGateBoard.artifacts used to be append-only: every same-role
9524 // task re-received up to 4 prior handoff payloads while
9525 // HandoffConsumed receipts fired as if they were spent.
9526 let tmp = tempfile::tempdir().expect("tempdir");
9527 let ctx = ToolContext::new(tmp.path().to_path_buf());
9528 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
9529 let runtime = SubAgentRuntime::new(
9530 stub_client(),
9531 "deepseek-v4-flash".to_string(),
9532 ctx.clone(),
9533 true,
9534 None,
9535 manager.clone(),
9536 );
9537 let state = WorkflowWorkspaceState::open(tmp.path());
9538 let run_id = "workflow_handoff_once".to_string();
9539 let gates = vec![GateSpec {
9540 id: "scout-findings".to_string(),
9541 role: "scout".to_string(),
9542 on: GateOn::RoleComplete,
9543 gate: GateKind::Approve,
9544 on_fail: codewhale_workflow::GateOnFail::Block,
9545 blocks_role: Some("implementer".to_string()),
9546 max_retries: 0,
9547 artifact_kind: Some("findings".to_string()),
9548 require_explicit_verdict: false,
9549 }];
9550 let spec = WorkflowSpec {
9551 id: Some("handoff-once-fixture".to_string()),
9552 goal: "handoff delivered once".to_string(),
9553 description: None,
9554 budget: BudgetSpec::default(),
9555 permissions: Default::default(),
9556 model_policy: Default::default(),
9557 promotion_policy: Default::default(),
9558 gates: gates.clone(),
9559 nodes: Vec::new(),
9560 };
9561 state.runs.lock().expect("runs").insert(
9562 run_id.clone(),
9563 WorkflowRunRecord::new(
9564 run_id.clone(),
9565 Some("session-test".to_string()),
9566 None,
9567 None,
9568 Some(&spec),
9569 ),
9570 );
9571 let driver = SubAgentWorkflowDriver::new(
9572 run_id.clone(),
9573 "session-test".to_string(),
9574 manager,
9575 runtime,
9576 state.clone(),
9577 None,
9578 WorkflowFleetBinding::None,
9579 gates,
9580 tmp.path().to_path_buf(),
9581 );
9582
9583 driver.evaluate_gates_for_completed_role(&RuntimeTaskRecord {
9584 agent_id: "scout-agent".to_string(),
9585 label: Some("scout".to_string()),
9586 role: Some("scout".to_string()),
9587 status: IrWorkflowRunStatus::Succeeded,
9588 output: Some("findings: exactly once".to_string()),
9589 schema_error: None,
9590 usage: None,
9591 });
9592
9593 let implementer = TaskRequest {
9594 description: "Use the findings.".to_string(),
9595 subagent_type: Some("implementer".to_string()),
9596 role: Some("implementer".to_string()),
9597 profile: None,
9598 model: None,
9599 model_strength: None,
9600 thinking: None,
9601 cwd: None,
9602 worktree: false,
9603 write_authority: Some("workspace_write".to_string()),
9604 write_roots: vec!["src".to_string()],
9605 exact_files: Vec::new(),
9606 coordination_contracts: Vec::new(),
9607 dependencies: Vec::new(),
9608 acceptance: Vec::new(),
9609 allowed_tools: Some(Vec::new()),
9610 disallowed_tools: Vec::new(),
9611 max_depth: None,
9612 token_budget: None,
9613 max_steps: None,
9614 wall_time_secs: None,
9615 response_schema: None,
9616 schema_repair_attempts: None,
9617 label: Some("fix".to_string()),
9618 phase: None,
9619 };
9620
9621 let mut first = implementer.clone();
9622 let handoffs = driver
9623 .prepare_request_for_gates(&mut first)
9624 .expect("passed gate should admit first implementer");
9625 assert_eq!(handoffs.len(), 1, "{handoffs:?}");
9626 assert!(first.description.contains("exactly once"));
9627
9628 // A second same-role task must not re-receive the spent handoff.
9629 let mut second = TaskRequest {
9630 description: "Second implementer task.".to_string(),
9631 ..implementer
9632 };
9633 let handoffs = driver
9634 .prepare_request_for_gates(&mut second)
9635 .expect("passed gate should admit second implementer");
9636 assert!(
9637 handoffs.is_empty(),
9638 "handoff already consumed must not re-deliver: {handoffs:?}"
9639 );
9640 assert!(
9641 !second
9642 .description
9643 .contains("Workflow handoff artifacts available"),
9644 "{}",
9645 second.description
9646 );
9647 }
9648
9649 #[tokio::test]
9650 async fn workflow_gate_evaluation_error_persists_blocked_and_denies_target_role() {
9651 let tmp = tempfile::tempdir().expect("tempdir");
9652 let ctx = ToolContext::new(tmp.path().to_path_buf());
9653 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
9654 let runtime = SubAgentRuntime::new(
9655 stub_client(),
9656 "deepseek-v4-flash".to_string(),
9657 ctx,
9658 true,
9659 None,
9660 manager.clone(),
9661 );
9662 let state = WorkflowWorkspaceState::open(tmp.path());
9663 let run_id = "workflow_malformed_gate".to_string();
9664 let gates = vec![GateSpec {
9665 id: String::new(),
9666 role: "scout".to_string(),
9667 on: GateOn::RoleComplete,
9668 gate: GateKind::Approve,
9669 on_fail: codewhale_workflow::GateOnFail::Block,
9670 blocks_role: Some("implementer".to_string()),
9671 max_retries: 0,
9672 artifact_kind: Some("findings".to_string()),
9673 require_explicit_verdict: false,
9674 }];
9675 let spec = WorkflowSpec {
9676 id: Some("malformed-gate-fixture".to_string()),
9677 goal: "malformed gate must fail closed".to_string(),
9678 description: None,
9679 budget: BudgetSpec::default(),
9680 permissions: Default::default(),
9681 model_policy: Default::default(),
9682 promotion_policy: Default::default(),
9683 gates: gates.clone(),
9684 nodes: Vec::new(),
9685 };
9686 state.runs.lock().expect("runs").insert(
9687 run_id.clone(),
9688 WorkflowRunRecord::new(
9689 run_id.clone(),
9690 Some("session-test".to_string()),
9691 None,
9692 None,
9693 Some(&spec),
9694 ),
9695 );
9696 let driver = SubAgentWorkflowDriver::new(
9697 run_id.clone(),
9698 "session-test".to_string(),
9699 manager,
9700 runtime,
9701 state.clone(),
9702 None,
9703 WorkflowFleetBinding::None,
9704 gates,
9705 tmp.path().to_path_buf(),
9706 );
9707
9708 driver.evaluate_gates_for_completed_role(&RuntimeTaskRecord {
9709 agent_id: "scout-agent".to_string(),
9710 label: Some("scout".to_string()),
9711 role: Some("scout".to_string()),
9712 status: IrWorkflowRunStatus::Succeeded,
9713 output: Some("findings".to_string()),
9714 schema_error: None,
9715 usage: None,
9716 });
9717
9718 let mut request = TaskRequest {
9719 description: "Must not be admitted.".to_string(),
9720 subagent_type: Some("implementer".to_string()),
9721 role: Some("implementer".to_string()),
9722 profile: None,
9723 model: None,
9724 model_strength: None,
9725 thinking: None,
9726 cwd: None,
9727 worktree: false,
9728 write_authority: Some("workspace_write".to_string()),
9729 write_roots: vec!["src".to_string()],
9730 exact_files: Vec::new(),
9731 coordination_contracts: vec!["test-contract".to_string()],
9732 dependencies: Vec::new(),
9733 acceptance: Vec::new(),
9734 allowed_tools: Some(Vec::new()),
9735 disallowed_tools: Vec::new(),
9736 max_depth: None,
9737 token_budget: None,
9738 max_steps: None,
9739 wall_time_secs: None,
9740 response_schema: None,
9741 schema_repair_attempts: None,
9742 label: Some("blocked".to_string()),
9743 phase: None,
9744 };
9745 let error = driver
9746 .prepare_request_for_gates(&mut request)
9747 .expect_err("malformed gate must deny its target role");
9748 assert!(
9749 error.to_string().contains("gate id must not be empty"),
9750 "{error}"
9751 );
9752 let board = driver.gate_board.lock().expect("gate board");
9753 assert!(matches!(
9754 board.gates.get(""),
9755 Some(GateState::Blocked { reason }) if reason.contains("gate id must not be empty")
9756 ));
9757 assert!(board.artifacts.is_empty(), "{:?}", board.artifacts);
9758 drop(board);
9759 let run = state
9760 .runs
9761 .lock()
9762 .expect("runs")
9763 .get(&run_id)
9764 .cloned()
9765 .expect("run");
9766 assert!(run.gate_status.iter().any(|line| {
9767 line.gate_id.is_empty()
9768 && line.state == "blocked"
9769 && line
9770 .blocked_reason
9771 .as_deref()
9772 .is_some_and(|reason| reason.contains("gate id must not be empty"))
9773 }));
9774 assert!(
9775 run.events
9776 .iter()
9777 .all(|event| event.event_type() != "handoff_promoted"),
9778 "{:?}",
9779 run.events
9780 );
9781 }
9782
9783 #[tokio::test]
9784 async fn workflow_handoff_record_error_changes_pass_to_blocked() {
9785 let tmp = tempfile::tempdir().expect("tempdir");
9786 let ctx = ToolContext::new(tmp.path().to_path_buf());
9787 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
9788 let runtime = SubAgentRuntime::new(
9789 stub_client(),
9790 "deepseek-v4-flash".to_string(),
9791 ctx,
9792 true,
9793 None,
9794 manager.clone(),
9795 );
9796 let state = WorkflowWorkspaceState::open(tmp.path());
9797 let run_id = "workflow_handoff_record_error".to_string();
9798 let gates = vec![GateSpec {
9799 id: "scout-findings".to_string(),
9800 role: "scout".to_string(),
9801 on: GateOn::RoleComplete,
9802 gate: GateKind::Approve,
9803 on_fail: codewhale_workflow::GateOnFail::Block,
9804 blocks_role: Some("implementer".to_string()),
9805 max_retries: 0,
9806 artifact_kind: Some("findings".to_string()),
9807 require_explicit_verdict: false,
9808 }];
9809 let spec = WorkflowSpec {
9810 id: Some("handoff-record-error-fixture".to_string()),
9811 goal: "failed handoff recording must fail closed".to_string(),
9812 description: None,
9813 budget: BudgetSpec::default(),
9814 permissions: Default::default(),
9815 model_policy: Default::default(),
9816 promotion_policy: Default::default(),
9817 gates: gates.clone(),
9818 nodes: Vec::new(),
9819 };
9820 state.runs.lock().expect("runs").insert(
9821 run_id.clone(),
9822 WorkflowRunRecord::new(
9823 run_id.clone(),
9824 Some("session-test".to_string()),
9825 None,
9826 None,
9827 Some(&spec),
9828 ),
9829 );
9830 let driver = SubAgentWorkflowDriver::new(
9831 run_id.clone(),
9832 "session-test".to_string(),
9833 manager,
9834 runtime,
9835 state.clone(),
9836 None,
9837 WorkflowFleetBinding::None,
9838 gates,
9839 tmp.path().to_path_buf(),
9840 );
9841 driver.gate_board.lock().expect("gate board").lane_id = "wrong-lane".to_string();
9842
9843 driver.evaluate_gates_for_completed_role(&RuntimeTaskRecord {
9844 agent_id: "scout-agent".to_string(),
9845 label: Some("scout".to_string()),
9846 role: Some("scout".to_string()),
9847 status: IrWorkflowRunStatus::Succeeded,
9848 output: Some("findings".to_string()),
9849 schema_error: None,
9850 usage: None,
9851 });
9852
9853 let mut request = TaskRequest {
9854 description: "Must not be admitted.".to_string(),
9855 subagent_type: Some("implementer".to_string()),
9856 role: Some("implementer".to_string()),
9857 profile: None,
9858 model: None,
9859 model_strength: None,
9860 thinking: None,
9861 cwd: None,
9862 worktree: false,
9863 write_authority: Some("workspace_write".to_string()),
9864 write_roots: vec!["src".to_string()],
9865 exact_files: Vec::new(),
9866 coordination_contracts: vec!["test-contract".to_string()],
9867 dependencies: Vec::new(),
9868 acceptance: Vec::new(),
9869 allowed_tools: Some(Vec::new()),
9870 disallowed_tools: Vec::new(),
9871 max_depth: None,
9872 token_budget: None,
9873 max_steps: None,
9874 wall_time_secs: None,
9875 response_schema: None,
9876 schema_repair_attempts: None,
9877 label: Some("blocked".to_string()),
9878 phase: None,
9879 };
9880 let error = driver
9881 .prepare_request_for_gates(&mut request)
9882 .expect_err("unrecorded handoff must deny its target role");
9883 assert!(
9884 error.to_string().contains("handoff could not be recorded"),
9885 "{error}"
9886 );
9887 let board = driver.gate_board.lock().expect("gate board");
9888 assert!(matches!(
9889 board.gates.get("scout-findings"),
9890 Some(GateState::Blocked { reason }) if reason.contains("does not match board lane")
9891 ));
9892 assert!(board.artifacts.is_empty(), "{:?}", board.artifacts);
9893 drop(board);
9894 let run = state
9895 .runs
9896 .lock()
9897 .expect("runs")
9898 .get(&run_id)
9899 .cloned()
9900 .expect("run");
9901 assert!(run.events.iter().any(|event| {
9902 matches!(
9903 &event.kind,
9904 WorkflowUiEventKind::GateUpdated {
9905 state,
9906 blocked_reason: Some(reason),
9907 ..
9908 } if state == "blocked" && reason.contains("handoff could not be recorded")
9909 )
9910 }));
9911 assert!(
9912 run.events
9913 .iter()
9914 .all(|event| event.event_type() != "handoff_promoted"),
9915 "{:?}",
9916 run.events
9917 );
9918 }
9919
9920 #[test]
9921 fn explicit_gate_verdict_only_reads_first_standalone_token() {
9922 assert_eq!(
9923 explicit_gate_verdict(Some("\n APPROVE \nreview complete")),
9924 Some(ExplicitGateVerdict::Approve)
9925 );
9926 assert_eq!(
9927 explicit_gate_verdict(Some("PASS\nverification complete")),
9928 Some(ExplicitGateVerdict::Approve)
9929 );
9930 assert_eq!(
9931 explicit_gate_verdict(Some("BLOCK\nmissing receipt")),
9932 Some(ExplicitGateVerdict::Reject)
9933 );
9934 assert_eq!(
9935 explicit_gate_verdict(Some("\nFAIL\nmissing receipt")),
9936 Some(ExplicitGateVerdict::Reject)
9937 );
9938 assert_eq!(
9939 explicit_gate_verdict(Some("Review result: BLOCK")),
9940 None,
9941 "prose remains backward-compatible success output"
9942 );
9943 assert_eq!(
9944 explicit_gate_verdict(Some("review notes\nBLOCK")),
9945 None,
9946 "later verdict words must not override the first meaningful line"
9947 );
9948 }
9949
9950 #[test]
9951 fn required_explicit_gate_verdict_fails_closed_when_missing_or_malformed() {
9952 let mut record = RuntimeTaskRecord {
9953 agent_id: "reviewer-malformed".to_string(),
9954 label: Some("reviewer".to_string()),
9955 role: Some("reviewer".to_string()),
9956 status: IrWorkflowRunStatus::Succeeded,
9957 output: Some("Review result: BLOCK".to_string()),
9958 schema_error: None,
9959 usage: None,
9960 };
9961
9962 match gate_outcome_for_completed_role(&record, true, None) {
9963 GateOutcome::Fail { reason } => {
9964 assert!(
9965 reason.contains("required first-line gate verdict"),
9966 "{reason}"
9967 );
9968 }
9969 outcome => panic!("required malformed verdict must fail closed: {outcome:?}"),
9970 }
9971 assert_eq!(
9972 gate_outcome_for_completed_role(&record, false, None),
9973 GateOutcome::Pass,
9974 "legacy gates retain pass-on-success behavior"
9975 );
9976
9977 record.output = None;
9978 assert!(matches!(
9979 gate_outcome_for_completed_role(&record, true, None),
9980 GateOutcome::Fail { .. }
9981 ));
9982 }
9983
9984 #[test]
9985 fn required_gate_artifact_rejects_bare_or_placeholder_approval() {
9986 let mut record = RuntimeTaskRecord {
9987 agent_id: "implementer-bare".to_string(),
9988 label: Some("implementer".to_string()),
9989 role: Some("implementer".to_string()),
9990 status: IrWorkflowRunStatus::Succeeded,
9991 output: Some("APPROVE".to_string()),
9992 schema_error: None,
9993 usage: None,
9994 };
9995
9996 match gate_outcome_for_completed_role(&record, true, Some("verification_plan")) {
9997 GateOutcome::Fail { reason } => {
9998 assert!(
9999 reason.contains("verification_plan artifact body"),
10000 "{reason}"
10001 );
10002 }
10003 outcome => panic!("bare approval must not promote an empty artifact: {outcome:?}"),
10004 }
10005
10006 record.output = Some("APPROVE\nacceptance evidence".to_string());
10007 match gate_outcome_for_completed_role(&record, true, Some("verification_plan")) {
10008 GateOutcome::Fail { reason } => {
10009 assert!(
10010 reason.contains("verification_plan artifact body"),
10011 "{reason}"
10012 );
10013 }
10014 outcome => {
10015 panic!("one placeholder line must not count as an artifact: {outcome:?}");
10016 }
10017 }
10018
10019 record.output = Some("APPROVE\nPLAN\n- verify the typed receipt".to_string());
10020 assert_eq!(
10021 gate_outcome_for_completed_role(&record, true, Some("verification_plan")),
10022 GateOutcome::Pass
10023 );
10024 }
10025
10026 #[tokio::test]
10027 #[allow(clippy::await_holding_lock)]
10028 async fn terminal_blocked_gate_fails_workflow_finalization() {
10029 let _retry_guard = workflow_test_retry_guard();
10030 let tmp = tempfile::tempdir().expect("tempdir");
10031 let ctx = ToolContext::new(tmp.path().to_path_buf());
10032 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
10033 let (client, calls) =
10034 fake_chat_client("BLOCK\nFINAL RECEIPT\n- missing terminal evidence").await;
10035 let runtime = SubAgentRuntime::new(
10036 client,
10037 "deepseek-v4-flash".to_string(),
10038 ctx.clone(),
10039 true,
10040 None,
10041 manager,
10042 );
10043 let tool = WorkflowTool::new(runtime.manager.clone(), runtime);
10044
10045 let result = tool
10046 .execute(
10047 json!({
10048 "action": "run",
10049 "script": r#"export default workflow({
10050 "goal": "fail closed on the terminal release verdict",
10051 "gates": [
10052 {
10053 "id": "terminal-release",
10054 "role": "reviewer",
10055 "on": "role_complete",
10056 "gate": "approve",
10057 "on_fail": "block",
10058 "max_retries": 0,
10059 "artifact_kind": "final_receipt",
10060 "require_explicit_verdict": true
10061 }
10062 ],
10063 "nodes": [
10064 {
10065 "agent": {
10066 "id": "release-receipt",
10067 "prompt": "Return the terminal verdict and receipt.",
10068 "agent_type": "general",
10069 "role": "reviewer",
10070 "mode": "read_only",
10071 "permissions": { "deny_all_tools": true },
10072 "budget": { "max_steps": 1 }
10073 }
10074 }
10075 ]
10076 });"#
10077 }),
10078 &ctx,
10079 )
10080 .await
10081 .expect("blocked terminal gate should return its failed run record");
10082 let payload: Value = serde_json::from_str(&result.content).expect("workflow JSON");
10083
10084 assert_eq!(calls.load(Ordering::SeqCst), 1, "{payload}");
10085 assert_eq!(payload["status"], "failed", "{payload}");
10086 assert_eq!(payload["execution"]["status"], "failed", "{payload}");
10087 assert!(
10088 payload["error"]
10089 .as_str()
10090 .is_some_and(|error| error.contains("terminal-release")
10091 && error.contains("ended blocked")
10092 && error.contains("missing terminal evidence")),
10093 "{payload}"
10094 );
10095 assert!(payload["gate_status"].as_array().is_some_and(|gates| {
10096 gates
10097 .iter()
10098 .any(|gate| gate["gate_id"] == "terminal-release" && gate["state"] == "blocked")
10099 }));
10100 assert!(payload["events"].as_array().is_some_and(|events| {
10101 events
10102 .iter()
10103 .any(|event| event["type"] == "run_completed" && event["status"] == "failed")
10104 }));
10105 }
10106
10107 #[tokio::test]
10108 async fn workflow_runtime_gate_honors_explicit_reviewer_verdicts() {
10109 let tmp = tempfile::tempdir().expect("tempdir");
10110 let ctx = ToolContext::new(tmp.path().to_path_buf());
10111 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
10112 let runtime = SubAgentRuntime::new(
10113 stub_client(),
10114 "deepseek-v4-flash".to_string(),
10115 ctx,
10116 true,
10117 None,
10118 manager.clone(),
10119 );
10120 let state = WorkflowWorkspaceState::open(tmp.path());
10121 let run_id = "workflow_explicit_verdict".to_string();
10122 let gates = vec![GateSpec {
10123 id: "review-findings".to_string(),
10124 role: "reviewer".to_string(),
10125 on: GateOn::RoleComplete,
10126 gate: GateKind::Review,
10127 on_fail: codewhale_workflow::GateOnFail::Block,
10128 blocks_role: Some("verifier".to_string()),
10129 max_retries: 0,
10130 artifact_kind: Some("review_report".to_string()),
10131 require_explicit_verdict: true,
10132 }];
10133 let spec = WorkflowSpec {
10134 id: Some("explicit-verdict-fixture".to_string()),
10135 goal: "honor reviewer verdict".to_string(),
10136 description: None,
10137 budget: BudgetSpec::default(),
10138 permissions: Default::default(),
10139 model_policy: Default::default(),
10140 promotion_policy: Default::default(),
10141 gates: gates.clone(),
10142 nodes: Vec::new(),
10143 };
10144 state.runs.lock().expect("runs").insert(
10145 run_id.clone(),
10146 WorkflowRunRecord::new(
10147 run_id.clone(),
10148 Some("session-test".to_string()),
10149 None,
10150 None,
10151 Some(&spec),
10152 ),
10153 );
10154 let driver = SubAgentWorkflowDriver::new(
10155 run_id,
10156 "session-test".to_string(),
10157 manager,
10158 runtime,
10159 state,
10160 None,
10161 WorkflowFleetBinding::None,
10162 gates,
10163 tmp.path().to_path_buf(),
10164 );
10165
10166 driver.evaluate_gates_for_completed_role(&RuntimeTaskRecord {
10167 agent_id: "reviewer-block".to_string(),
10168 label: Some("reviewer".to_string()),
10169 role: Some("reviewer".to_string()),
10170 status: IrWorkflowRunStatus::Succeeded,
10171 output: Some("\nBLOCK\nmissing terminal receipt".to_string()),
10172 schema_error: None,
10173 usage: None,
10174 });
10175
10176 let verifier_request = || TaskRequest {
10177 description: "Verify the accepted review.".to_string(),
10178 subagent_type: Some("verifier".to_string()),
10179 role: Some("verifier".to_string()),
10180 profile: None,
10181 model: None,
10182 model_strength: None,
10183 thinking: None,
10184 cwd: None,
10185 worktree: false,
10186 write_authority: Some("read_only".to_string()),
10187 write_roots: Vec::new(),
10188 exact_files: Vec::new(),
10189 coordination_contracts: Vec::new(),
10190 dependencies: Vec::new(),
10191 acceptance: Vec::new(),
10192 allowed_tools: Some(Vec::new()),
10193 disallowed_tools: Vec::new(),
10194 max_depth: None,
10195 token_budget: None,
10196 max_steps: None,
10197 wall_time_secs: None,
10198 response_schema: None,
10199 schema_repair_attempts: None,
10200 label: Some("verify".to_string()),
10201 phase: None,
10202 };
10203 let mut blocked_verifier = verifier_request();
10204 let error = driver
10205 .prepare_request_for_gates(&mut blocked_verifier)
10206 .expect_err("successful reviewer BLOCK must not admit verifier");
10207 assert!(error.to_string().contains("BLOCK"), "{error}");
10208 {
10209 let board = driver.gate_board.lock().expect("gate board");
10210 assert!(
10211 board.artifacts.is_empty(),
10212 "rejected output must not produce a handoff: {:?}",
10213 board.artifacts
10214 );
10215 assert!(matches!(
10216 board.gates.get("review-findings"),
10217 Some(GateState::Blocked { .. })
10218 ));
10219 }
10220
10221 driver.evaluate_gates_for_completed_role(&RuntimeTaskRecord {
10222 agent_id: "reviewer-approve".to_string(),
10223 label: Some("reviewer".to_string()),
10224 role: Some("reviewer".to_string()),
10225 status: IrWorkflowRunStatus::Succeeded,
10226 output: Some("APPROVE\nEVIDENCE REVIEW\n- all receipt owners confirmed".to_string()),
10227 schema_error: None,
10228 usage: None,
10229 });
10230
10231 let mut admitted_verifier = verifier_request();
10232 driver
10233 .prepare_request_for_gates(&mut admitted_verifier)
10234 .expect("explicit reviewer APPROVE should admit verifier");
10235 assert!(
10236 admitted_verifier
10237 .description
10238 .contains("all receipt owners confirmed"),
10239 "{}",
10240 admitted_verifier.description
10241 );
10242 let board = driver.gate_board.lock().expect("gate board");
10243 assert!(
10244 board.artifacts.is_empty(),
10245 "the admitted verifier consumed the handoff; spent artifacts leave the board: {:?}",
10246 board.artifacts
10247 );
10248 assert!(matches!(
10249 board.gates.get("review-findings"),
10250 Some(GateState::Passed)
10251 ));
10252 }
10253
10254 #[tokio::test]
10255 #[allow(clippy::await_holding_lock)]
10256 async fn workflow_status_lists_compact_typed_receipts() {
10257 let _retry_guard = workflow_test_retry_guard();
10258 let tmp = tempfile::tempdir().expect("tempdir");
10259 let ctx = ToolContext::new(tmp.path().to_path_buf());
10260 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
10261 let (client, _calls) = fake_chat_client("status-output").await;
10262 let runtime = SubAgentRuntime::new(
10263 client,
10264 "deepseek-v4-flash".to_string(),
10265 ctx.clone(),
10266 true,
10267 None,
10268 manager,
10269 );
10270 let tool = WorkflowTool::new(runtime.manager.clone(), runtime);
10271
10272 let run = tool
10273 .execute(
10274 json!({
10275 "action": "run",
10276 "script": r#"export default workflow({
10277 "id": "status-fixture",
10278 "goal": "status summary",
10279 "nodes": [
10280 {
10281 "agent": {
10282 "id": "inspect",
10283 "prompt": "Inspect the code.",
10284 "agent_type": "review"
10285 }
10286 }
10287 ]
10288 });"#
10289 }),
10290 &ctx,
10291 )
10292 .await
10293 .expect("workflow run");
10294 let run_payload: Value = serde_json::from_str(&run.content).expect("run json");
10295
10296 let status = tool
10297 .execute(json!({"action": "status"}), &ctx)
10298 .await
10299 .expect("workflow status");
10300 let status_payload: Value = serde_json::from_str(&status.content).expect("status json");
10301 let summary = &status_payload["runs"][0];
10302
10303 assert_eq!(status_payload["count"], 1);
10304 assert_eq!(summary["run_id"], run_payload["run_id"]);
10305 assert_eq!(summary["workflow_id"], "status-fixture");
10306 assert_eq!(summary["workflow_goal"], "status summary");
10307 assert_eq!(summary["status"], "completed");
10308 assert_eq!(summary["execution_status"], "succeeded");
10309 assert_eq!(summary["child_count"], 1);
10310 assert_eq!(summary["leaf_count"], 1);
10311 assert_eq!(summary["branch_count"], 0);
10312 assert_eq!(summary["control_count"], 0);
10313 assert!(summary["event_count"].as_u64().unwrap_or_default() >= 3);
10314 assert_eq!(summary["last_event_type"], "run_completed");
10315 assert!(summary.get("result").is_none());
10316 assert!(summary.get("execution").is_none());
10317 }
10318
10319 #[tokio::test]
10320 #[allow(clippy::await_holding_lock)]
10321 async fn workflow_status_survives_tool_rebuild_via_journal() {
10322 let _retry_guard = workflow_test_retry_guard();
10323 let tmp = tempfile::tempdir().expect("tempdir");
10324 let ctx = ToolContext::new(tmp.path().to_path_buf());
10325 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
10326 let (client, _calls) = fake_chat_client("journal-output").await;
10327 let runtime = SubAgentRuntime::new(
10328 client,
10329 "deepseek-v4-flash".to_string(),
10330 ctx.clone(),
10331 true,
10332 None,
10333 manager.clone(),
10334 );
10335 let tool = WorkflowTool::new(manager.clone(), runtime.clone());
10336
10337 let run = tool
10338 .execute(
10339 json!({
10340 "action": "run",
10341 "script": "return { ok: true };"
10342 }),
10343 &ctx,
10344 )
10345 .await
10346 .expect("workflow run");
10347 let run_payload: Value = serde_json::from_str(&run.content).expect("run json");
10348 let run_id = run_payload["run_id"].as_str().expect("run id");
10349
10350 let journal_path = tmp.path().join(".codewhale/workflow-runs.jsonl");
10351 assert!(
10352 journal_path.exists(),
10353 "journal should be created under workspace"
10354 );
10355
10356 let rebuilt = WorkflowTool::new(
10357 manager.clone(),
10358 SubAgentRuntime::new(
10359 stub_client(),
10360 "deepseek-v4-flash".to_string(),
10361 ctx.clone(),
10362 true,
10363 None,
10364 manager,
10365 ),
10366 );
10367 let status = rebuilt
10368 .execute(json!({"action": "status", "run_id": run_id}), &ctx)
10369 .await
10370 .expect("workflow status after rebuild");
10371 let status_payload: Value = serde_json::from_str(&status.content).expect("status json");
10372 assert_eq!(status_payload["run_id"], run_id);
10373 assert_eq!(status_payload["status"], "completed");
10374 }
10375
10376 #[tokio::test]
10377 #[allow(clippy::await_holding_lock)]
10378 async fn workflow_status_surfaces_schema_failure_instead_of_null_success() {
10379 let _retry_guard = workflow_test_retry_guard();
10380 let tmp = tempfile::tempdir().expect("tempdir");
10381 let ctx = ToolContext::new(tmp.path().to_path_buf());
10382 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
10383 let (client, _calls) = fake_chat_client(r#"{"refuted":"yes"}"#).await;
10384 let runtime = SubAgentRuntime::new(
10385 client,
10386 "deepseek-v4-flash".to_string(),
10387 ctx.clone(),
10388 true,
10389 None,
10390 manager.clone(),
10391 );
10392 let tool = WorkflowTool::new(manager, runtime);
10393
10394 let run = tool
10395 .execute(
10396 json!({
10397 "action": "run",
10398 "script": r#"
10399 return await parallel([
10400 () => task({
10401 description: "Return the schema fixture.",
10402 responseSchema: {
10403 type: "object",
10404 properties: { refuted: { type: "boolean" } },
10405 required: ["refuted"],
10406 },
10407 }),
10408 ]);
10409 "#
10410 }),
10411 &ctx,
10412 )
10413 .await
10414 .expect("workflow run returns a record");
10415 let run_payload: Value = serde_json::from_str(&run.content).expect("run json");
10416
10417 assert_eq!(run_payload["status"], "failed");
10418 assert!(run_payload["result"].is_null());
10419 assert!(
10420 run_payload["error"]
10421 .as_str()
10422 .unwrap()
10423 .contains("responseSchema validation")
10424 );
10425 assert!(
10426 run_payload["progress"]
10427 .as_array()
10428 .unwrap()
10429 .iter()
10430 .any(|message| message
10431 .as_str()
10432 .is_some_and(|message| message.contains("schema validation failed"))),
10433 "schema validation error should be visible in the run receipt: {run_payload}"
10434 );
10435 assert!(
10436 run_payload["events"]
10437 .as_array()
10438 .unwrap()
10439 .iter()
10440 .any(|event| event["type"] == "task_schema_validation_failed"
10441 && event["message"]
10442 .as_str()
10443 .is_some_and(|message| message.contains("responseSchema validation"))),
10444 "schema validation event should be visible in the typed receipt: {run_payload}"
10445 );
10446 }
10447
10448 #[tokio::test]
10449 #[allow(clippy::await_holding_lock)]
10450 async fn schema_failure_repairs_and_the_receipt_survives_journal_reload() {
10451 let _retry_guard = workflow_test_retry_guard();
10452 let tmp = tempfile::tempdir().expect("tempdir");
10453 let ctx = ToolContext::new(tmp.path().to_path_buf());
10454 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
10455 // The #5583 report shape: attempt 1 wraps valid JSON in prose (a
10456 // parse failure), the bounded repair answers with bare corrected
10457 // JSON. Two model calls, no more — the fake server panics on extras.
10458 let (client, calls) = fake_chat_client_responses(&[
10459 "Sure — here it is:\n```json\n{\"refuted\": true}\n```\nDone!",
10460 "{\"refuted\": true}",
10461 ])
10462 .await;
10463 let runtime = SubAgentRuntime::new(
10464 client,
10465 "deepseek-v4-flash".to_string(),
10466 ctx.clone(),
10467 true,
10468 None,
10469 manager.clone(),
10470 );
10471 let tool = WorkflowTool::new(manager, runtime);
10472
10473 let run = tool
10474 .execute(
10475 json!({
10476 "action": "run",
10477 "script": r#"
10478 return await task({
10479 description: "Return the schema fixture.",
10480 responseSchema: {
10481 type: "object",
10482 properties: { refuted: { type: "boolean" } },
10483 required: ["refuted"],
10484 },
10485 });
10486 "#
10487 }),
10488 &ctx,
10489 )
10490 .await
10491 .expect("workflow run returns a record");
10492 let run_payload: Value = serde_json::from_str(&run.content).expect("run json");
10493
10494 assert_eq!(run_payload["status"], "completed", "{run_payload}");
10495 assert_eq!(run_payload["result"]["refuted"], json!(true));
10496 assert_eq!(calls.load(Ordering::SeqCst), 2, "one repair re-ask");
10497 assert_eq!(
10498 run.metadata
10499 .as_ref()
10500 .and_then(|metadata| metadata.get("schema_repair_count"))
10501 .and_then(Value::as_u64),
10502 Some(1),
10503 "the repair should be counted in the run metadata"
10504 );
10505 assert!(
10506 run_payload["progress"]
10507 .as_array()
10508 .unwrap()
10509 .iter()
10510 .any(|message| message
10511 .as_str()
10512 .is_some_and(|message| message.contains("dispatching a bounded repair"))),
10513 "the repair should be visible in the run receipt: {run_payload}"
10514 );
10515
10516 // The durable receipt (kind, attempt, bounded raw preview) survives a
10517 // journal reload — a restarted session still sees what was repaired.
10518 let state = WorkflowWorkspaceState::open(tmp.path());
10519 let record = state
10520 .runs
10521 .lock()
10522 .expect("runs")
10523 .get(run_payload["run_id"].as_str().expect("run id"))
10524 .cloned()
10525 .expect("record survives reload");
10526 assert_eq!(record.schema_repairs.len(), 1);
10527 let repair = &record.schema_repairs[0];
10528 assert_eq!(repair.kind, "json_parse");
10529 assert_eq!(repair.attempt, 1);
10530 assert!(repair.raw_preview.contains("Sure — here it is"));
10531 assert!(!repair.raw_truncated);
10532 assert!(repair.artifact.is_none(), "a short reply needs no artifact");
10533 assert!(
10534 record.schema_errors.is_empty(),
10535 "a successful repair leaves no terminal schema error"
10536 );
10537 }
10538
10539 #[test]
10540 fn bounded_raw_preview_truncates_on_a_char_boundary() {
10541 let short = "plain reply";
10542 assert_eq!(bounded_raw_preview(short), short);
10543 let long = "é".repeat(SCHEMA_RAW_PREVIEW_CHARS + 5);
10544 let preview = bounded_raw_preview(&long);
10545 assert!(preview.contains("preview truncated"));
10546 let kept = preview.lines().next().expect("kept line");
10547 assert_eq!(kept.chars().count(), SCHEMA_RAW_PREVIEW_CHARS);
10548 }
10549
10550 #[test]
10551 fn schema_raw_artifacts_land_beside_the_report_and_refuse_bad_ids() {
10552 let tmp = tempfile::tempdir().expect("tempdir");
10553 let path = write_schema_raw_artifact(
10554 tmp.path(),
10555 "run-2049",
10556 "agent_0007",
10557 2,
10558 "the full raw reply",
10559 )
10560 .expect("artifact path");
10561 let written = std::fs::read_to_string(&path).expect("artifact written");
10562 assert_eq!(written, "the full raw reply");
10563 assert!(
10564 path.ends_with("run-2049.schema.agent_0007.attempt2.txt"),
10565 "{path}"
10566 );
10567 // The slug filter sanitizes hostile ids into safe file names rather
10568 // than refusing: path traversal cannot survive it.
10569 let sanitized = write_schema_raw_artifact(tmp.path(), "../../etc", "agent_0007", 1, "x")
10570 .expect("sanitized path");
10571 assert!(
10572 sanitized.contains("etc.schema.agent_0007.attempt1.txt"),
10573 "{sanitized}"
10574 );
10575 assert!(
10576 !sanitized.contains(".."),
10577 "no parent traversal may survive: {sanitized}"
10578 );
10579 assert!(
10580 write_schema_raw_artifact(tmp.path(), "///", "agent_0007", 1, "x").is_none(),
10581 "an id with no slug characters must refuse rather than write"
10582 );
10583 }
10584
10585 #[tokio::test]
10586 #[allow(clippy::await_holding_lock)]
10587 async fn declarative_issue_audit_fixture_runs_through_subagent_driver() {
10588 let _retry_guard = workflow_test_retry_guard();
10589 let tmp = tempfile::tempdir().expect("tempdir");
10590 let workflow_dir = tmp.path().join("workflows");
10591 std::fs::create_dir_all(&workflow_dir).expect("workflow dir");
10592 let fixture = std::fs::read_to_string(
10593 std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
10594 .join("../../workflows/issue_audit.workflow.js"),
10595 )
10596 .expect("issue audit fixture");
10597 std::fs::write(workflow_dir.join("issue_audit.workflow.js"), fixture)
10598 .expect("write fixture into workspace");
10599
10600 let mut ctx = ToolContext::new(tmp.path().to_path_buf());
10601 ctx.runtime.work = Some(crate::work_graph::new_shared_work_runtime(
10602 crate::tools::todo::new_shared_todo_list(),
10603 crate::tools::plan::new_shared_plan_state(),
10604 ));
10605 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 4);
10606 let (client, calls) = fake_chat_client("audited").await;
10607 let runtime = SubAgentRuntime::new(
10608 client,
10609 "deepseek-v4-flash".to_string(),
10610 ctx.clone(),
10611 true,
10612 None,
10613 manager,
10614 );
10615 let tool = WorkflowTool::new(runtime.manager.clone(), runtime);
10616
10617 let result = tool
10618 .execute(
10619 json!({
10620 "action": "run",
10621 "source_path": "workflows/issue_audit.workflow.js"
10622 }),
10623 &ctx,
10624 )
10625 .await
10626 .expect("declarative workflow should complete");
10627 let payload: Value = serde_json::from_str(&result.content).expect("json result");
10628
10629 assert_eq!(payload["status"], "completed", "{payload}");
10630 assert_eq!(payload["result"]["code-audit"], "audited");
10631 assert_eq!(payload["result"]["test-audit"], "audited");
10632 assert_eq!(payload["result"]["docs-audit"], "audited");
10633 assert_eq!(payload["result"]["synthesize-release-risk"], "audited");
10634 assert_eq!(payload["execution"]["status"], "succeeded");
10635 assert_eq!(
10636 payload["execution"]["leaf_results"]
10637 .as_array()
10638 .expect("leaf results")
10639 .len(),
10640 3
10641 );
10642 assert_eq!(
10643 payload["execution"]["branch_results"][0]["branch_id"],
10644 "parallel-audit"
10645 );
10646 assert!(
10647 payload["execution"]["control_node_results"]
10648 .as_array()
10649 .expect("control results")
10650 .iter()
10651 .any(|result| result["node_id"] == "synthesize-release-risk"
10652 && result["kind"] == "reduce"
10653 && result["status"] == "succeeded")
10654 );
10655 assert_eq!(payload["child_ids"].as_array().unwrap().len(), 4);
10656 assert_eq!(calls.load(Ordering::SeqCst), 4);
10657 assert!(
10658 payload["progress"]
10659 .as_array()
10660 .unwrap()
10661 .iter()
10662 .any(|message| message == "phase: parallel-audit")
10663 );
10664
10665 // Operate projects Workflow fan-out/fan-in and its children through
10666 // the same canonical Work Graph. The Workflow remains the accountable
10667 // operation identity while the worker bindings stay inspectable; no
10668 // second plan/strategy lifecycle is created for the reduce step.
10669 let work = ctx.runtime.work.as_ref().expect("work runtime");
10670 let graph = work
10671 .capture(Some(&ctx.state_namespace))
10672 .expect("capture workflow work")
10673 .expect("workflow graph")
10674 .graph;
10675 let workflow_external = format!(
10676 "workflow:{}",
10677 payload["run_id"].as_str().expect("workflow run id")
10678 );
10679 let workflow_operations = graph
10680 .nodes
10681 .iter()
10682 .filter(|node| {
10683 node.binding
10684 .as_ref()
10685 .is_some_and(|binding| binding.external == workflow_external)
10686 })
10687 .collect::<Vec<_>>();
10688 assert_eq!(
10689 workflow_operations.len(),
10690 1,
10691 "one accountable Workflow operation: {graph:#?}"
10692 );
10693 assert_eq!(
10694 workflow_operations[0].state,
10695 crate::work_graph::NodeState::Completed
10696 );
10697 for child_id in payload["child_ids"].as_array().expect("workflow child ids") {
10698 let worker_external = format!(
10699 "worker:{}",
10700 child_id.as_str().expect("workflow child id string")
10701 );
10702 assert!(
10703 graph.nodes.iter().any(|node| {
10704 node.binding
10705 .as_ref()
10706 .is_some_and(|binding| binding.external == worker_external)
10707 }),
10708 "Workflow worker {worker_external} must remain inspectable in the same graph: {graph:#?}"
10709 );
10710 }
10711 }
10712
10713 #[tokio::test]
10714 #[allow(clippy::await_holding_lock)]
10715 async fn stopship_acceptance_fixture_emits_role_gate_and_terminal_receipts() {
10716 let _retry_guard = workflow_test_retry_guard();
10717 let _env_lock = crate::test_support::lock_test_env();
10718 let tmp = tempfile::tempdir().expect("tempdir");
10719 let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", tmp.path());
10720 let workflow_dir = tmp.path().join("workflows");
10721 let fleet_dir = tmp.path().join("fleets");
10722 std::fs::create_dir_all(&workflow_dir).expect("workflow dir");
10723 std::fs::create_dir_all(&fleet_dir).expect("fleet dir");
10724 let repo_root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");
10725 std::fs::copy(
10726 repo_root.join("workflows/stopship.workflow.js"),
10727 workflow_dir.join("stopship.workflow.js"),
10728 )
10729 .expect("copy stopship acceptance fixture");
10730 std::fs::copy(
10731 repo_root.join("fleets/stopship.toml"),
10732 fleet_dir.join("stopship.toml"),
10733 )
10734 .expect("copy stopship fleet");
10735
10736 let source = std::fs::read_to_string(workflow_dir.join("stopship.workflow.js"))
10737 .expect("read stopship acceptance fixture");
10738 let compiled =
10739 codewhale_workflow::compile_javascript_workflow("stopship.workflow.js", &source)
10740 .expect("compile stopship acceptance fixture");
10741 let codewhale_workflow::WorkflowNode::Sequence(sequence) = &compiled.nodes[0] else {
10742 panic!("stopship fixture should be one ordered role chain");
10743 };
10744 for (index, node) in sequence.children.iter().enumerate() {
10745 let codewhale_workflow::WorkflowNode::Leaf(leaf) = node else {
10746 panic!("stopship role chain must contain only leaves");
10747 };
10748 let tools = leaf_allowed_tools(leaf).expect("lower stopship child tools");
10749 if index == 0 {
10750 assert!(tools.as_ref().is_some_and(|tools| !tools.is_empty()));
10751 } else {
10752 assert_eq!(
10753 tools,
10754 Some(Vec::<String>::new()),
10755 "downstream handoff consumer {} must receive no tools",
10756 leaf.id
10757 );
10758 }
10759 }
10760
10761 let ctx = ToolContext::new(tmp.path().to_path_buf());
10762 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 8);
10763 let responses = [
10764 r#"APPROVE
10765 SOURCE EVIDENCE
10766 - crates/cli/src/lib.rs: load_named_fleet
10767 - crates/workflow/src/role_resolve.rs: resolve_workflow_agent
10768 - crates/cli/src/lib.rs: start_lane
10769 - crates/tui/src/tools/workflow/mod.rs: record_task_started
10770 - crates/tui/src/tools/workflow/mod.rs: WorkflowUiEventKind::GateUpdated
10771 - crates/tui/src/tools/workflow/mod.rs: WorkflowUiEventKind::RunCompleted -> terminal_completed_receipt
10772 - crates/lane/src/runtime.rs: process_exit_receipt -> lane_reconciled"#,
10773 r#"APPROVE
10774 PLAN
10775 - fleets/stopship.toml: name = "stopship" -> named Fleet loading
10776 - crates/workflow/src/role_resolve.rs: resolve_workflow_agent -> role resolution
10777 - crates/cli/src/lib.rs: start_lane -> tmux Lane launch
10778 - crates/tui/src/tools/workflow/mod.rs: record_task_started -> typed task_started
10779 - crates/tui/src/tools/workflow/mod.rs: WorkflowUiEventKind::GateUpdated -> gate promotion
10780 - crates/tui/src/tools/workflow/mod.rs: WorkflowUiEventKind::RunCompleted -> terminal_completed_receipt
10781 - crates/lane/src/runtime.rs: process_exit_receipt -> tmux Lane reconciliation"#,
10782 r#"APPROVE
10783 EVIDENCE REVIEW
10784 - fleets/stopship.toml: name = "stopship"
10785 - crates/workflow/src/role_resolve.rs: resolve_workflow_agent
10786 - crates/cli/src/lib.rs: start_lane
10787 - crates/tui/src/tools/workflow/mod.rs: record_task_started
10788 - crates/tui/src/tools/workflow/mod.rs: WorkflowUiEventKind::GateUpdated
10789 - crates/tui/src/tools/workflow/mod.rs: WorkflowUiEventKind::RunCompleted -> terminal_completed_receipt
10790 - crates/lane/src/runtime.rs: process_exit_receipt -> lane_reconciled"#,
10791 r#"APPROVE
10792 EVIDENCE MATRIX
10793 - fleet_load: fleets/stopship.toml: name = "stopship"
10794 - role_resolution: crates/workflow/src/role_resolve.rs: resolve_workflow_agent
10795 - lane_launch: crates/cli/src/lib.rs: start_lane
10796 - task_started: crates/tui/src/tools/workflow/mod.rs: record_task_started
10797 - gate_updated: crates/tui/src/tools/workflow/mod.rs: WorkflowUiEventKind::GateUpdated
10798 - run_completed: crates/tui/src/tools/workflow/mod.rs: WorkflowUiEventKind::RunCompleted -> terminal_completed_receipt
10799 - lane_exit: crates/lane/src/runtime.rs: process_exit_receipt -> lane_reconciled"#,
10800 r#"APPROVE
10801 FINAL RECEIPT
10802 - fleet_load: fleets/stopship.toml: name = "stopship"
10803 - role_resolution: crates/workflow/src/role_resolve.rs: resolve_workflow_agent
10804 - lane_launch: crates/cli/src/lib.rs: start_lane
10805 - task_started: crates/tui/src/tools/workflow/mod.rs: record_task_started
10806 - gate_updated: crates/tui/src/tools/workflow/mod.rs: WorkflowUiEventKind::GateUpdated
10807 - run_completed: crates/tui/src/tools/workflow/mod.rs: WorkflowUiEventKind::RunCompleted -> terminal_completed_receipt
10808 - lane_exit: crates/lane/src/runtime.rs: process_exit_receipt -> lane_reconciled"#,
10809 ];
10810 let (client, calls) = fake_chat_client_responses(&responses).await;
10811 let runtime = SubAgentRuntime::new(
10812 client,
10813 "deepseek-v4-flash".to_string(),
10814 ctx.clone(),
10815 true,
10816 None,
10817 manager,
10818 );
10819 let tool = WorkflowTool::new(runtime.manager.clone(), runtime);
10820
10821 let result = tool
10822 .execute(
10823 json!({
10824 "action": "run",
10825 "source_path": "workflows/stopship.workflow.js",
10826 "fleet": "stopship",
10827 "token_budget": 60_000
10828 }),
10829 &ctx,
10830 )
10831 .await
10832 .expect("stopship acceptance workflow returns a terminal record");
10833 let payload: Value = serde_json::from_str(&result.content).expect("workflow JSON");
10834
10835 assert_eq!(payload["status"], "completed", "{payload}");
10836 assert_eq!(payload["execution"]["status"], "succeeded", "{payload}");
10837 assert_eq!(calls.load(Ordering::SeqCst), 5, "one child per Fleet role");
10838 let approval = &payload["plan_approval"];
10839 assert_eq!(approval["decision"], "auto_read_only", "{approval}");
10840 assert_eq!(approval["token_budget"], 60_000, "{approval}");
10841 assert_eq!(approval["writes"], false, "{approval}");
10842 assert_eq!(approval["shell"], false, "{approval}");
10843 assert_eq!(approval["network"], false, "{approval}");
10844 assert_eq!(approval["high_budget"], false, "{approval}");
10845 assert_eq!(approval["elevated"], false, "{approval}");
10846 assert!(
10847 approval["reasons"].as_array().is_some_and(Vec::is_empty),
10848 "{approval}"
10849 );
10850
10851 let events = payload["events"].as_array().expect("typed events");
10852 let started = events
10853 .iter()
10854 .filter(|event| event["type"] == "task_started")
10855 .collect::<Vec<_>>();
10856 let expected_roles = [
10857 ("explore", "scout"),
10858 ("implement", "builder"),
10859 ("reviewer", "reviewer"),
10860 ("test", "verifier"),
10861 ("release_lead", "advisor"),
10862 ];
10863 assert_eq!(started.len(), expected_roles.len(), "{started:#?}");
10864 for (event, (role, profile)) in started.iter().zip(expected_roles) {
10865 assert_eq!(event["role"], role);
10866 assert_eq!(event["profile"], profile);
10867 assert_eq!(event["resolved_profile"], profile);
10868 assert_eq!(event["workflow_run_id"], payload["run_id"]);
10869 }
10870
10871 let gates = events
10872 .iter()
10873 .filter(|event| event["type"] == "gate_updated")
10874 .collect::<Vec<_>>();
10875 assert_eq!(gates.len(), 5, "{gates:#?}");
10876 assert!(gates.iter().all(|event| event["state"] == "passed"));
10877 assert_eq!(gates[0]["role"], "explore");
10878 assert_eq!(gates[0]["blocked_role"], "implement");
10879 assert_eq!(gates[3]["role"], "test");
10880 assert_eq!(gates[3]["blocked_role"], "release_lead");
10881 assert_eq!(gates[4]["role"], "release_lead");
10882 assert!(gates[4]["blocked_role"].is_null());
10883
10884 let promoted = events
10885 .iter()
10886 .filter(|event| event["type"] == "handoff_promoted")
10887 .collect::<Vec<_>>();
10888 let consumed = events
10889 .iter()
10890 .filter(|event| event["type"] == "handoff_consumed")
10891 .collect::<Vec<_>>();
10892 let expected_handoffs = [
10893 ("explore", "implement", "source_evidence"),
10894 ("implement", "reviewer", "verification_plan"),
10895 ("reviewer", "test", "review_report"),
10896 ("test", "release_lead", "verification_report"),
10897 ];
10898 assert_eq!(promoted.len(), expected_handoffs.len(), "{promoted:#?}");
10899 assert_eq!(consumed.len(), expected_handoffs.len(), "{consumed:#?}");
10900 let artifact_ids = promoted
10901 .iter()
10902 .map(|event| {
10903 event["artifact_id"]
10904 .as_str()
10905 .filter(|id| id.starts_with("handoff_") && id.len() > "handoff_".len())
10906 .expect("opaque non-empty handoff artifact id")
10907 })
10908 .collect::<std::collections::HashSet<_>>();
10909 assert_eq!(
10910 artifact_ids.len(),
10911 promoted.len(),
10912 "every promotion must have a unique artifact id: {promoted:#?}"
10913 );
10914 for (index, (from_role, to_role, kind)) in expected_handoffs.into_iter().enumerate() {
10915 assert_eq!(promoted[index]["from_role"], from_role);
10916 assert_eq!(promoted[index]["to_role"], to_role);
10917 assert_eq!(promoted[index]["kind"], kind);
10918 assert_eq!(promoted[index]["gate_id"], gates[index]["gate_id"]);
10919 assert_eq!(
10920 promoted[index]["producer_task_id"],
10921 started[index]["task_id"]
10922 );
10923 assert!(
10924 promoted[index].get("payload").is_none(),
10925 "{:#?}",
10926 promoted[index]
10927 );
10928
10929 assert_eq!(
10930 consumed[index]["artifact_id"],
10931 promoted[index]["artifact_id"]
10932 );
10933 assert_eq!(consumed[index]["from_role"], from_role);
10934 assert_eq!(consumed[index]["to_role"], to_role);
10935 assert_eq!(consumed[index]["kind"], kind);
10936 assert_eq!(
10937 consumed[index]["consumer_task_id"],
10938 started[index + 1]["task_id"]
10939 );
10940 assert!(
10941 consumed[index].get("payload").is_none(),
10942 "{:#?}",
10943 consumed[index]
10944 );
10945
10946 let producer_task_id = promoted[index]["producer_task_id"]
10947 .as_str()
10948 .expect("producer task id");
10949 let consumer_task_id = consumed[index]["consumer_task_id"]
10950 .as_str()
10951 .expect("consumer task id");
10952 let gate_id = promoted[index]["gate_id"].as_str().expect("gate id");
10953 let artifact_id = promoted[index]["artifact_id"]
10954 .as_str()
10955 .expect("artifact id");
10956 let task_completed_index = events
10957 .iter()
10958 .position(|event| {
10959 event["type"] == "task_completed" && event["task_id"] == producer_task_id
10960 })
10961 .expect("producer completion receipt");
10962 let gate_updated_index = events
10963 .iter()
10964 .position(|event| event["type"] == "gate_updated" && event["gate_id"] == gate_id)
10965 .expect("gate update receipt");
10966 let promoted_index = events
10967 .iter()
10968 .position(|event| {
10969 event["type"] == "handoff_promoted" && event["artifact_id"] == artifact_id
10970 })
10971 .expect("handoff promotion receipt");
10972 let consumer_started_index = events
10973 .iter()
10974 .position(|event| {
10975 event["type"] == "task_started" && event["task_id"] == consumer_task_id
10976 })
10977 .expect("consumer start receipt");
10978 let consumed_index = events
10979 .iter()
10980 .position(|event| {
10981 event["type"] == "handoff_consumed" && event["artifact_id"] == artifact_id
10982 })
10983 .expect("handoff consumption receipt");
10984 assert!(
10985 task_completed_index < gate_updated_index
10986 && gate_updated_index < promoted_index
10987 && promoted_index < consumer_started_index
10988 && consumer_started_index < consumed_index,
10989 "causal receipt order must be task_completed -> gate_updated -> handoff_promoted -> task_started -> handoff_consumed: {events:#?}"
10990 );
10991 }
10992 let terminal_completed_receipt = events
10993 .iter()
10994 .any(|event| event["type"] == "run_completed" && event["status"] == "completed");
10995 assert!(terminal_completed_receipt, "{events:#?}");
10996 }
10997
10998 /// The deadline is the point of this test: an id the manager never records
10999 /// must fail closed on the first read. With the old retry loop the body
11000 /// slept 50 x 20ms before answering, and this timeout fires. A plain
11001 /// `#[tokio::test]` is deliberate — `start_paused = true` auto-advances
11002 /// time and would let the polling version pass (#6211).
11003 #[tokio::test]
11004 async fn completion_from_manager_fails_closed_without_polling() {
11005 let tmp = tempfile::tempdir().expect("tempdir");
11006 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
11007
11008 let (completion, usage) = tokio::time::timeout(
11009 std::time::Duration::from_millis(100),
11010 completion_from_manager(manager, "missing_agent", "fallback".to_string()),
11011 )
11012 .await
11013 .expect("completion_from_manager must answer from one read, not by polling");
11014 assert!(usage.is_none(), "fail-closed path carries no telemetry");
11015 match completion {
11016 TaskCompletion::Failed { message } => {
11017 assert!(message.contains("no terminal manager record"), "{message}");
11018 }
11019 other => panic!("expected a fail-closed failure, got {other:?}"),
11020 }
11021 }
11022
11023 #[tokio::test]
11024 #[allow(clippy::await_holding_lock)]
11025 async fn task_completed_and_run_completed_carry_usage_telemetry() {
11026 let _retry_guard = workflow_test_retry_guard();
11027 let tmp = tempfile::tempdir().expect("tempdir");
11028 let ctx = ToolContext::new(tmp.path().to_path_buf());
11029 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
11030 let (client, calls) = fake_chat_client("telemetry-output").await;
11031 let runtime = SubAgentRuntime::new(
11032 client,
11033 "deepseek-v4-flash".to_string(),
11034 ctx.clone(),
11035 true,
11036 None,
11037 manager,
11038 );
11039 let tool = WorkflowTool::new(runtime.manager.clone(), runtime);
11040
11041 let run = tool
11042 .execute(
11043 json!({
11044 "action": "run",
11045 "script": r#"export default workflow({
11046 "id": "telemetry-fixture",
11047 "goal": "usage telemetry",
11048 "nodes": [
11049 {
11050 "agent": {
11051 "id": "inspect",
11052 "prompt": "Inspect the code.",
11053 "agent_type": "review"
11054 }
11055 }
11056 ]
11057 });"#
11058 }),
11059 &ctx,
11060 )
11061 .await
11062 .expect("workflow run");
11063 let payload: Value = serde_json::from_str(&run.content).expect("run json");
11064 assert_eq!(payload["status"], "completed", "{payload}");
11065 assert_eq!(calls.load(Ordering::SeqCst), 1, "{payload}");
11066
11067 let events = payload["events"].as_array().expect("typed events");
11068 let task_completed = events
11069 .iter()
11070 .find(|event| event["type"] == "task_completed")
11071 .expect("task_completed event");
11072 let usage = &task_completed["usage"];
11073 let input = usage["input_tokens"].as_u64().expect("worker input tokens");
11074 let output = usage["output_tokens"]
11075 .as_u64()
11076 .expect("worker output tokens");
11077 let total = usage["total_tokens"].as_u64().expect("worker total tokens");
11078 assert!(total >= input + output, "{task_completed}");
11079 assert!(
11080 usage["tool_calls"].as_u64().unwrap_or_default() >= 1,
11081 "{task_completed}"
11082 );
11083 assert!(usage["duration_ms"].is_u64(), "{task_completed}");
11084 // #4039: a row may only label tokens as provider-reported when the
11085 // worker ledger actually received them.
11086 assert_eq!(
11087 usage["token_source"], "provider_reported",
11088 "{task_completed}"
11089 );
11090 assert!(
11091 usage["result_ref"]
11092 .as_str()
11093 .is_some_and(|target| target.starts_with("agent:")),
11094 "{task_completed}"
11095 );
11096
11097 // Run totals reconcile exactly with the per-task telemetry.
11098 let run_completed = events
11099 .iter()
11100 .find(|event| event["type"] == "run_completed")
11101 .expect("run_completed event");
11102 let run_usage = &run_completed["usage"];
11103 assert_eq!(run_usage["total_tokens"], total, "{run_completed}");
11104 assert_eq!(run_usage["input_tokens"], input, "{run_completed}");
11105 assert_eq!(run_usage["output_tokens"], output, "{run_completed}");
11106 assert_eq!(
11107 run_usage["tool_calls"], usage["tool_calls"],
11108 "{run_completed}"
11109 );
11110 assert_eq!(run_usage["tasks_reported"], 1, "{run_completed}");
11111
11112 // Totals also land on the persisted record and execution receipt.
11113 assert_eq!(payload["usage"]["total_tokens"], total, "{payload}");
11114 assert_eq!(
11115 payload["execution"]["usage"]["input_tokens"], input,
11116 "{payload}"
11117 );
11118 assert_eq!(
11119 payload["execution"]["leaf_results"][0]["usage"]["input_tokens"], input,
11120 "{payload}"
11121 );
11122 }
11123
11124 #[test]
11125 fn provider_usage_presence_preserves_a_real_zero_receipt() {
11126 assert!(!provider_usage_was_reported(None, None, None));
11127 assert!(provider_usage_was_reported(Some(0), Some(0), Some(0)));
11128 assert!(provider_usage_was_reported(None, Some(0), None));
11129 }
11130
11131 #[test]
11132 fn run_usage_totals_reconcile_task_telemetry() {
11133 let task_usage = |total: u64, calls: u32| WorkflowTaskUsage {
11134 input_tokens: Some(total / 2),
11135 output_tokens: Some(total - total / 2),
11136 total_tokens: Some(total),
11137 cost_microusd: Some(total),
11138 tool_calls: Some(calls),
11139 duration_ms: Some(7),
11140 result_ref: None,
11141 token_source: Some(WorkflowTokenSource::ProviderReported),
11142 };
11143 let record = |agent_id: &str, usage: Option<WorkflowTaskUsage>| RuntimeTaskRecord {
11144 agent_id: agent_id.to_string(),
11145 label: None,
11146 role: None,
11147 status: IrWorkflowRunStatus::Succeeded,
11148 output: None,
11149 schema_error: None,
11150 usage,
11151 };
11152 let records = vec![
11153 record("a", Some(task_usage(100, 2))),
11154 record("b", Some(task_usage(60, 1))),
11155 record("c", None),
11156 ];
11157 let totals = run_usage_totals(&records).expect("totals");
11158 assert_eq!(totals.total_tokens, Some(160));
11159 assert_eq!(totals.cost_microusd, Some(160));
11160 assert_eq!(totals.input_tokens, Some(80));
11161 assert_eq!(totals.output_tokens, Some(80));
11162 assert_eq!(totals.tool_calls, Some(3));
11163 assert_eq!(totals.tasks_reported, 2);
11164
11165 assert!(run_usage_totals(&[]).is_none());
11166 assert!(run_usage_totals(&[record("d", None)]).is_none());
11167 }
11168
11169 #[test]
11170 fn run_and_ir_usage_keep_unknown_distinct_from_reported_zero() {
11171 let record = |agent_id: &str, usage: WorkflowTaskUsage| RuntimeTaskRecord {
11172 agent_id: agent_id.to_string(),
11173 label: Some(agent_id.to_string()),
11174 role: None,
11175 status: IrWorkflowRunStatus::Succeeded,
11176 output: None,
11177 schema_error: None,
11178 usage: Some(usage),
11179 };
11180 let unknown = WorkflowTaskUsage {
11181 tool_calls: Some(1),
11182 duration_ms: Some(4),
11183 ..WorkflowTaskUsage::default()
11184 };
11185 let reported_zero = WorkflowTaskUsage {
11186 input_tokens: Some(0),
11187 output_tokens: Some(0),
11188 total_tokens: Some(0),
11189 cost_microusd: Some(0),
11190 tool_calls: Some(0),
11191 duration_ms: Some(0),
11192 token_source: Some(WorkflowTokenSource::ProviderReported),
11193 ..WorkflowTaskUsage::default()
11194 };
11195
11196 let unknown_totals = run_usage_totals(&[record("unknown", unknown.clone())])
11197 .expect("tool/duration receipt still creates run usage");
11198 assert_eq!(unknown_totals.input_tokens, None);
11199 assert_eq!(unknown_totals.output_tokens, None);
11200 assert_eq!(unknown_totals.total_tokens, None);
11201 assert_eq!(unknown_totals.tool_calls, Some(1));
11202
11203 let zero_totals = run_usage_totals(&[record("zero", reported_zero.clone())])
11204 .expect("reported zero receipt");
11205 assert_eq!(zero_totals.input_tokens, Some(0));
11206 assert_eq!(zero_totals.output_tokens, Some(0));
11207 assert_eq!(zero_totals.total_tokens, Some(0));
11208 assert_eq!(zero_totals.cost_microusd, Some(0));
11209 assert_eq!(zero_totals.tool_calls, Some(0));
11210
11211 let unknown_ir = workflow_usage_from_task(&unknown);
11212 assert_eq!(unknown_ir.input_tokens, None);
11213 assert_eq!(unknown_ir.output_tokens, None);
11214 assert_eq!(unknown_ir.cost_microusd, None);
11215 let zero_ir = workflow_usage_from_task(&reported_zero);
11216 assert_eq!(zero_ir.input_tokens, Some(0));
11217 assert_eq!(zero_ir.output_tokens, Some(0));
11218 assert_eq!(zero_ir.cost_microusd, Some(0));
11219
11220 let mixed = run_usage_totals(&[record("zero", reported_zero), record("unknown", unknown)])
11221 .expect("mixed receipts");
11222 assert_eq!(
11223 mixed.total_tokens,
11224 Some(0),
11225 "a missing contributor keeps the observed subtotal"
11226 );
11227 assert_eq!(mixed.input_tokens, Some(0));
11228 assert_eq!(mixed.output_tokens, Some(0));
11229 assert_eq!(mixed.cost_microusd, Some(0));
11230 assert_eq!(mixed.tool_calls, Some(1));
11231 }
11232
11233 #[test]
11234 fn workflow_ui_event_usage_telemetry_serde_round_trip() {
11235 let event = WorkflowUiEvent::at(
11236 7,
11237 "session-test",
11238 WorkflowUiEventKind::TaskCompleted {
11239 task_id: "child-1".to_string(),
11240 status: IrWorkflowRunStatus::Succeeded,
11241 usage: Some(WorkflowTaskUsage {
11242 input_tokens: Some(128),
11243 output_tokens: Some(32),
11244 total_tokens: Some(160),
11245 cost_microusd: Some(42),
11246 tool_calls: Some(2),
11247 duration_ms: Some(42),
11248 result_ref: Some("agent:child-1".to_string()),
11249 token_source: Some(WorkflowTokenSource::ProviderReported),
11250 }),
11251 },
11252 );
11253 let json = serde_json::to_value(&event).expect("serialize");
11254 assert_eq!(json["usage"]["total_tokens"], 160);
11255 let parsed: WorkflowUiEvent = serde_json::from_value(json).expect("deserialize round trip");
11256 match parsed.kind {
11257 WorkflowUiEventKind::TaskCompleted {
11258 usage: Some(usage), ..
11259 } => {
11260 assert_eq!(usage.total_tokens, Some(160));
11261 assert_eq!(usage.cost_microusd, Some(42));
11262 assert_eq!(usage.tool_calls, Some(2));
11263 assert_eq!(usage.duration_ms, Some(42));
11264 }
11265 other => panic!("expected task_completed with usage, got {other:?}"),
11266 }
11267
11268 // Journals written before #2974 carry no usage fields; they must
11269 // still parse with `usage == None`.
11270 let legacy_task: WorkflowUiEvent = serde_json::from_str(
11271 r#"{"at_ms":5,"type":"task_completed","task_id":"child-1","status":"succeeded"}"#,
11272 )
11273 .expect("legacy task_completed parses");
11274 match legacy_task.kind {
11275 WorkflowUiEventKind::TaskCompleted { usage: None, .. } => {}
11276 other => panic!("expected legacy task_completed without usage, got {other:?}"),
11277 }
11278 let legacy_run: WorkflowUiEvent = serde_json::from_str(
11279 r#"{"at_ms":6,"type":"run_completed","status":"completed","error":null}"#,
11280 )
11281 .expect("legacy run_completed parses");
11282 match legacy_run.kind {
11283 WorkflowUiEventKind::RunCompleted { usage: None, .. } => {}
11284 other => panic!("expected legacy run_completed without usage, got {other:?}"),
11285 }
11286
11287 // A telemetry-less event serializes without a `usage` key, so old
11288 // consumers see a byte-compatible shape.
11289 let plain = serde_json::to_value(WorkflowUiEvent::at(
11290 8,
11291 "session-test",
11292 WorkflowUiEventKind::RunCompleted {
11293 status: WorkflowRunStatus::Completed,
11294 error: None,
11295 usage: None,
11296 },
11297 ))
11298 .expect("serialize plain run_completed");
11299 assert!(plain.get("usage").is_none(), "{plain}");
11300 }
11301
11302 #[test]
11303 fn run_record_event_retention_is_bounded() {
11304 let mut record = WorkflowRunRecord::new(
11305 "workflow_tail".to_string(),
11306 Some("session-test".to_string()),
11307 None,
11308 None,
11309 None,
11310 );
11311 for index in 0..(WORKFLOW_RUN_EVENTS_MAX_RETAINED + 5) {
11312 record.push_event(WorkflowUiEvent::at(
11313 index as u64,
11314 "session-test",
11315 WorkflowUiEventKind::Log {
11316 message: format!("log {index}"),
11317 },
11318 ));
11319 }
11320 assert_eq!(record.events.len(), WORKFLOW_RUN_EVENTS_MAX_RETAINED);
11321 assert_eq!(record.events_dropped, 5);
11322 assert_eq!(
11323 record.events_total,
11324 (WORKFLOW_RUN_EVENTS_MAX_RETAINED + 5) as u64
11325 );
11326 // The retained window is the newest tail.
11327 let first = &record.events[0];
11328 assert_eq!(first.at_ms, 5);
11329 // Summaries report the truthful total, not the retained tail length.
11330 let summary = record.summary();
11331 assert_eq!(summary.event_count, WORKFLOW_RUN_EVENTS_MAX_RETAINED + 5);
11332 assert_eq!(summary.events_dropped, 5);
11333 assert_eq!(summary.last_event_type.as_deref(), Some("log"));
11334 }
11335
11336 #[test]
11337 fn run_record_progress_and_rejection_ledgers_are_bounded_with_exact_counts() {
11338 let mut record = WorkflowRunRecord::new(
11339 "workflow_rejection_loop".to_string(),
11340 Some("session-test".to_string()),
11341 None,
11342 None,
11343 None,
11344 );
11345 let progress_total = WORKFLOW_RUN_PROGRESS_MAX_RETAINED + 7;
11346 for index in 0..progress_total {
11347 record.push_progress(format!("progress {index}"));
11348 }
11349 let rejection_total = WORKFLOW_RUN_DISPATCH_FAILURES_MAX_RETAINED + 7;
11350 for index in 0..rejection_total {
11351 record.push_dispatch_failure(WorkflowDispatchFailure {
11352 at_ms: index as u64,
11353 label: Some(format!("rejected-{index}")),
11354 phase: Some("fan-out".to_string()),
11355 message: "invalid task options".to_string(),
11356 });
11357 }
11358
11359 assert_eq!(record.progress.len(), WORKFLOW_RUN_PROGRESS_MAX_RETAINED);
11360 assert_eq!(record.progress_count, progress_total as u64);
11361 assert_eq!(
11362 record.progress.first().map(String::as_str),
11363 Some("progress 7")
11364 );
11365 assert_eq!(
11366 record.dispatch_failures.len(),
11367 WORKFLOW_RUN_DISPATCH_FAILURES_MAX_RETAINED
11368 );
11369 assert_eq!(record.dispatch_failure_count, rejection_total as u64);
11370 assert_eq!(record.dispatch_failures[0].at_ms, 7);
11371
11372 let summary = record.summary();
11373 assert_eq!(summary.progress_count, progress_total as u64);
11374 assert_eq!(summary.dispatch_failure_count, rejection_total as u64);
11375
11376 let (payload, bounds) = bounded_run_record_value(&record, Path::new("workflow-runs.jsonl"));
11377 assert_eq!(payload["progress_count"], progress_total as u64);
11378 assert_eq!(payload["dispatch_failure_count"], rejection_total as u64);
11379 assert_eq!(
11380 bounds.dispatch_failures_omitted,
11381 (rejection_total - WORKFLOW_RESULT_DISPATCH_FAILURES_TAIL) as u64
11382 );
11383
11384 // Imported counters can already be saturated. Another rejection must
11385 // retain its newest detail without wrapping the authoritative total.
11386 record.dispatch_failure_count = u64::MAX;
11387 record.push_dispatch_failure(WorkflowDispatchFailure {
11388 at_ms: u64::MAX,
11389 label: None,
11390 phase: None,
11391 message: "malformed rejection loop".to_string(),
11392 });
11393 assert_eq!(record.dispatch_failure_count, u64::MAX);
11394 assert_eq!(
11395 record.dispatch_failures.len(),
11396 WORKFLOW_RUN_DISPATCH_FAILURES_MAX_RETAINED
11397 );
11398 assert_eq!(
11399 record.dispatch_failures.last().map(|failure| failure.at_ms),
11400 Some(u64::MAX)
11401 );
11402 }
11403
11404 #[test]
11405 fn workflow_status_payload_bounds_oversized_run_records() {
11406 let tmp = tempfile::tempdir().expect("tempdir");
11407 let state = WorkflowWorkspaceState::open(tmp.path());
11408 let run_id = "workflow_big_run".to_string();
11409 let mut record = WorkflowRunRecord::new(
11410 run_id.clone(),
11411 Some("session-test".to_string()),
11412 None,
11413 None,
11414 None,
11415 );
11416 record.status = WorkflowRunStatus::Completed;
11417 // A high-fan-out run: far more events/progress than the model needs.
11418 for index in 0..200u64 {
11419 record.push_event(WorkflowUiEvent::at(
11420 index,
11421 "session-test",
11422 WorkflowUiEventKind::Log {
11423 message: format!("fan-out event {index}"),
11424 },
11425 ));
11426 }
11427 for index in 0..60 {
11428 record.push_progress(format!("progress line {index}"));
11429 }
11430 for index in 0..40u64 {
11431 record.push_dispatch_failure(WorkflowDispatchFailure {
11432 at_ms: index,
11433 label: Some(format!("rejected-{index}")),
11434 phase: Some("fan-out".to_string()),
11435 message: format!("dispatch rejected {index}: {}", "x".repeat(2_000)),
11436 });
11437 }
11438 record.result = Some(json!({ "blob": "r".repeat(10_000) }));
11439 record.execution = Some(IrWorkflowExecution {
11440 status: IrWorkflowRunStatus::Succeeded,
11441 usage: WorkflowUsage::default(),
11442 memo_usage: WorkflowMemoUsage::default(),
11443 leaf_results: vec![LeafResult {
11444 leaf_id: "inspect".to_string(),
11445 task_id: "agent_1".to_string(),
11446 role: None,
11447 profile: None,
11448 status: IrWorkflowRunStatus::Succeeded,
11449 usage: WorkflowUsage::default(),
11450 memo_usage: WorkflowMemoUsage::default(),
11451 output: Some("o".repeat(5_000)),
11452 artifacts: Vec::new(),
11453 schema_error: None,
11454 }],
11455 branch_results: Vec::new(),
11456 control_node_results: Vec::new(),
11457 });
11458 state
11459 .runs
11460 .lock()
11461 .expect("runs")
11462 .insert(run_id.clone(), record);
11463
11464 let result = workflow_result_for(&run_id, state, "session-test").expect("status result");
11465 assert!(
11466 result.content.len() < WORKFLOW_RESULT_MAX_CHARS,
11467 "bounded payload must stay under {WORKFLOW_RESULT_MAX_CHARS} chars, got {}",
11468 result.content.len()
11469 );
11470 let payload: Value = serde_json::from_str(&result.content).expect("payload json");
11471
11472 let events = payload["events"].as_array().expect("events");
11473 assert_eq!(events.len(), WORKFLOW_RESULT_EVENTS_TAIL, "{payload}");
11474 assert!(
11475 payload["events_note"]
11476 .as_str()
11477 .is_some_and(|note| note.contains("workflow-runs.jsonl")),
11478 "{payload}"
11479 );
11480 // The retained window is the newest events.
11481 assert_eq!(events[0]["at_ms"], 150);
11482
11483 let progress = payload["progress"].as_array().expect("progress");
11484 assert_eq!(progress.len(), WORKFLOW_RESULT_PROGRESS_TAIL, "{payload}");
11485 assert!(payload.get("progress_note").is_some(), "{payload}");
11486
11487 let dispatch_failures = payload["dispatch_failures"]
11488 .as_array()
11489 .expect("dispatch failures");
11490 assert_eq!(
11491 dispatch_failures.len(),
11492 WORKFLOW_RESULT_DISPATCH_FAILURES_TAIL,
11493 "{payload}"
11494 );
11495 assert_eq!(dispatch_failures[0]["at_ms"], 28);
11496 assert_eq!(
11497 payload["dispatch_failure_count"], 40,
11498 "exact count must survive the bounded failure tail"
11499 );
11500 assert!(
11501 dispatch_failures.iter().all(|failure| failure["message"]
11502 .as_str()
11503 .is_some_and(|message| message.chars().count()
11504 <= WORKFLOW_RESULT_DISPATCH_FAILURE_FIELD_MAX_CHARS)),
11505 "{dispatch_failures:?}"
11506 );
11507 assert!(
11508 payload["dispatch_failures_note"]
11509 .as_str()
11510 .is_some_and(|note| note.contains("workflow-runs.jsonl")),
11511 "{payload}"
11512 );
11513
11514 // Oversized VM result collapses to a preview with a journal pointer.
11515 assert_eq!(payload["result"]["truncated"], true, "{payload}");
11516 assert!(
11517 payload["result"]["preview"]
11518 .as_str()
11519 .is_some_and(|preview| preview.chars().count() <= WORKFLOW_RESULT_VALUE_MAX_CHARS),
11520 "{payload}"
11521 );
11522 assert!(
11523 payload["result"]["full_detail"]
11524 .as_str()
11525 .is_some_and(|path| path.contains("workflow-runs.jsonl")),
11526 "{payload}"
11527 );
11528
11529 // Leaf outputs carry a bounded preview instead of full child text.
11530 let leaf_output = payload["execution"]["leaf_results"][0]["output"]
11531 .as_str()
11532 .expect("leaf output");
11533 assert!(
11534 leaf_output.contains("leaf output truncated"),
11535 "{leaf_output}"
11536 );
11537 assert!(leaf_output.len() < 1_000, "{leaf_output}");
11538
11539 let metadata = result.metadata.expect("metadata");
11540 assert_eq!(metadata["events_returned"], WORKFLOW_RESULT_EVENTS_TAIL);
11541 assert_eq!(metadata["events_omitted"], 150);
11542 assert_eq!(metadata["event_count"], 200);
11543 assert_eq!(
11544 metadata["dispatch_failures_returned"],
11545 WORKFLOW_RESULT_DISPATCH_FAILURES_TAIL
11546 );
11547 assert_eq!(metadata["dispatch_failures_omitted"], 28);
11548 assert_eq!(metadata["truncated"], true);
11549 assert!(
11550 metadata["journal_path"]
11551 .as_str()
11552 .is_some_and(|path| path.contains("workflow-runs.jsonl")),
11553 "{metadata}"
11554 );
11555 }
11556
11557 #[test]
11558 fn workflow_status_payload_keeps_small_records_intact() {
11559 let tmp = tempfile::tempdir().expect("tempdir");
11560 let state = WorkflowWorkspaceState::open(tmp.path());
11561 let run_id = "workflow_small_run".to_string();
11562 let mut record = WorkflowRunRecord::new(
11563 run_id.clone(),
11564 Some("session-test".to_string()),
11565 None,
11566 None,
11567 None,
11568 );
11569 record.status = WorkflowRunStatus::Completed;
11570 record.push_event(WorkflowUiEvent::at(
11571 1,
11572 "session-test",
11573 WorkflowUiEventKind::PhaseStarted {
11574 title: "scan".to_string(),
11575 },
11576 ));
11577 record.result = Some(json!({ "ok": true }));
11578 state
11579 .runs
11580 .lock()
11581 .expect("runs")
11582 .insert(run_id.clone(), record);
11583
11584 let result = workflow_result_for(&run_id, state, "session-test").expect("status result");
11585 let payload: Value = serde_json::from_str(&result.content).expect("payload json");
11586 assert_eq!(payload["events"].as_array().map(Vec::len), Some(1));
11587 assert_eq!(payload["result"], json!({ "ok": true }));
11588 assert!(payload.get("events_note").is_none(), "{payload}");
11589 assert!(payload.get("progress_note").is_none(), "{payload}");
11590 let metadata = result.metadata.expect("metadata");
11591 assert_eq!(metadata["truncated"], false);
11592 assert_eq!(metadata["events_omitted"], 0);
11593 assert_eq!(metadata["events_returned"], 1);
11594 }
11595 #[tokio::test]
11596 #[allow(clippy::await_holding_lock)]
11597 async fn workflow_cancel_interrupts_vm_and_blocks_further_spawns() {
11598 let _retry_guard = workflow_test_retry_guard();
11599 let tmp = tempfile::tempdir().expect("tempdir");
11600 let ctx = ToolContext::new(tmp.path().to_path_buf());
11601 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 4);
11602 let (client, calls) = fake_chat_client("child done").await;
11603 let (event_tx, mut event_rx) = tokio::sync::mpsc::channel(256);
11604 let runtime = SubAgentRuntime::new(
11605 client,
11606 "deepseek-v4-flash".to_string(),
11607 ctx.clone(),
11608 true,
11609 Some(event_tx),
11610 manager.clone(),
11611 );
11612 let tool = WorkflowTool::new(manager.clone(), runtime);
11613
11614 let started = tool
11615 .execute(
11616 json!({
11617 "action": "start",
11618 "script": r#"
11619 let n = 0;
11620 while (n < 20) {
11621 await task({ description: `task ${n}`, type: 'explore', allowedTools: [] });
11622 n++;
11623 }
11624 return n;
11625 "#
11626 }),
11627 &ctx,
11628 )
11629 .await
11630 .expect("workflow start");
11631 let run_id = started
11632 .metadata
11633 .as_ref()
11634 .and_then(|metadata| metadata.get("run_id"))
11635 .and_then(Value::as_str)
11636 .expect("run_id metadata");
11637
11638 tokio::time::timeout(std::time::Duration::from_secs(15), async {
11639 while calls.load(Ordering::SeqCst) == 0 {
11640 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
11641 }
11642 })
11643 .await
11644 .expect("workflow should spawn at least one child before cancel");
11645 let calls_before_cancel = calls.load(Ordering::SeqCst);
11646 assert!(calls_before_cancel >= 1);
11647
11648 let cancelled = tool
11649 .execute(json!({"action": "cancel", "run_id": run_id}), &ctx)
11650 .await
11651 .expect("workflow cancel");
11652 let cancelled_payload: Value =
11653 serde_json::from_str(&cancelled.content).expect("cancel json");
11654 assert_eq!(cancelled_payload["status"], "cancelled");
11655 assert!(
11656 cancelled_payload["events"]
11657 .as_array()
11658 .is_some_and(|events| events.iter().any(|event| event["type"] == "run_cancelled")),
11659 "cancel receipt must include the authoritative terminal event: {cancelled_payload}"
11660 );
11661 let mut streamed_cancel = false;
11662 while let Ok(event) = event_rx.try_recv() {
11663 if let Event::WorkflowUi { event, .. } = event
11664 && event["type"] == "run_cancelled"
11665 {
11666 streamed_cancel = true;
11667 }
11668 }
11669 assert!(
11670 streamed_cancel,
11671 "cancel must stream a terminal UI event after any racing completion"
11672 );
11673 let first_event_count = cancelled_payload["events"]
11674 .as_array()
11675 .expect("events")
11676 .len();
11677 let first_completed_at = cancelled_payload["completed_at_ms"].clone();
11678 let cancelled_again = tool
11679 .execute(json!({"action": "cancel", "run_id": run_id}), &ctx)
11680 .await
11681 .expect("second workflow cancel is a no-op");
11682 let cancelled_again_payload: Value =
11683 serde_json::from_str(&cancelled_again.content).expect("second cancel json");
11684 assert_eq!(cancelled_again_payload["status"], "cancelled");
11685 assert_eq!(
11686 cancelled_again_payload["events"]
11687 .as_array()
11688 .expect("events")
11689 .len(),
11690 first_event_count,
11691 "second cancel must not append a duplicate terminal event"
11692 );
11693 assert_eq!(
11694 cancelled_again_payload["completed_at_ms"], first_completed_at,
11695 "second cancel must preserve the original completion time"
11696 );
11697
11698 tokio::time::sleep(std::time::Duration::from_millis(700)).await;
11699 let calls_after_cancel = calls.load(Ordering::SeqCst);
11700 assert!(
11701 calls_after_cancel <= calls_before_cancel + 1,
11702 "cancelled workflow kept spawning children: before={calls_before_cancel} after={calls_after_cancel}"
11703 );
11704 }
11705
11706 #[tokio::test]
11707 #[allow(clippy::await_holding_lock)]
11708 async fn workflow_token_budget_is_reported_never_enforced() {
11709 // #6189: a declared workflow `token_budget` survives as a reported
11710 // ceiling in the run snapshot; it no longer seeds a manager budget
11711 // scope, clamps a child's provider request, or stops a run.
11712 let _retry_guard = workflow_test_retry_guard();
11713 let tmp = tempfile::tempdir().expect("tempdir");
11714 let ctx = ToolContext::new(tmp.path().to_path_buf());
11715 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
11716 let (client, calls, bodies) = fake_chat_client_capturing("budgeted").await;
11717 let runtime = SubAgentRuntime::new(
11718 client,
11719 "deepseek-v4-flash".to_string(),
11720 ctx.clone(),
11721 true,
11722 None,
11723 manager.clone(),
11724 );
11725 let tool = WorkflowTool::new(manager.clone(), runtime);
11726
11727 let result = tool
11728 .execute(
11729 json!({
11730 "action": "run",
11731 "token_budget": 1000,
11732 "script": r#"
11733 await task({ description: 'budgeted work', type: 'explore', allowedTools: [] });
11734 return { spent: budget.spent(), total: budget.total, remaining: budget.remaining() };
11735 "#
11736 }),
11737 &ctx,
11738 )
11739 .await
11740 .expect("budget workflow should complete");
11741 let payload: Value = serde_json::from_str(&result.content).expect("json result");
11742
11743 assert_eq!(payload["status"], "completed", "{payload}");
11744 // #6189 removed per-scope token tracking: `spent` is deliberately 0
11745 // and the declared ceiling survives only as a reported total.
11746 assert_eq!(payload["result"]["spent"], 0, "{payload}");
11747 assert_eq!(payload["result"]["total"], 1000);
11748 assert_eq!(payload["result"]["remaining"], 1000);
11749 assert_eq!(calls.load(Ordering::SeqCst), 1);
11750 let bodies = bodies.lock().expect("captured first request");
11751 let max_tokens = bodies[0]
11752 .get("max_tokens")
11753 .or_else(|| bodies[0].get("max_completion_tokens"))
11754 .and_then(Value::as_u64);
11755 assert!(
11756 max_tokens.is_some_and(|tokens| tokens > 1000),
11757 "a declared token budget must not clamp the child's provider request (#6189): {max_tokens:?}"
11758 );
11759 }
11760
11761 #[tokio::test]
11762 async fn identical_budget_snapshots_emit_a_live_heartbeat_without_journal_duplication() {
11763 let tmp = tempfile::tempdir().expect("tempdir");
11764 let ctx = ToolContext::new(tmp.path().to_path_buf());
11765 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
11766 let (event_tx, mut event_rx) = tokio::sync::mpsc::channel(8);
11767 let runtime = SubAgentRuntime::new(
11768 stub_client(),
11769 "deepseek-v4-flash".to_string(),
11770 ctx,
11771 true,
11772 Some(event_tx),
11773 manager.clone(),
11774 );
11775 let state = WorkflowWorkspaceState::open(tmp.path());
11776 let run_id = "workflow_budget_heartbeat".to_string();
11777 state.runs.lock().expect("runs").insert(
11778 run_id.clone(),
11779 WorkflowRunRecord::new(
11780 run_id.clone(),
11781 Some("session-test".to_string()),
11782 None,
11783 None,
11784 None,
11785 ),
11786 );
11787 let driver = SubAgentWorkflowDriver::new(
11788 run_id.clone(),
11789 "session-test".to_string(),
11790 manager,
11791 runtime,
11792 state.clone(),
11793 Some(1_000),
11794 WorkflowFleetBinding::None,
11795 Vec::new(),
11796 tmp.path().to_path_buf(),
11797 );
11798 let snapshot = BudgetSnapshot {
11799 total: Some(1_000),
11800 spent: 0,
11801 };
11802
11803 driver.record_budget_snapshot(snapshot);
11804 driver.record_budget_snapshot(snapshot);
11805
11806 let mut streamed = 0;
11807 while let Ok(Event::WorkflowUi { event, .. }) = event_rx.try_recv() {
11808 if event["type"] == "budget_updated" {
11809 streamed += 1;
11810 }
11811 }
11812 assert_eq!(
11813 streamed, 2,
11814 "unchanged budget still refreshes the live panel"
11815 );
11816 let recorded = state
11817 .runs
11818 .lock()
11819 .expect("runs")
11820 .get(&run_id)
11821 .expect("run")
11822 .events
11823 .iter()
11824 .filter(|event| event.event_type() == "budget_updated")
11825 .count();
11826 assert_eq!(recorded, 1, "heartbeat must not grow the durable journal");
11827 }
11828
11829 fn native_lifecycle_tool(
11830 workspace: &Path,
11831 ) -> (
11832 WorkflowTool,
11833 ToolContext,
11834 mpsc::Receiver<SubAgentCompletion>,
11835 ) {
11836 let context = ToolContext::new(workspace.to_path_buf());
11837 let manager = new_shared_subagent_manager(workspace.to_path_buf(), 2);
11838 let (tx, rx) = mpsc::channel(16);
11839 let runtime = SubAgentRuntime::new(
11840 stub_client(),
11841 "deepseek-v4-flash".to_string(),
11842 context.clone(),
11843 true,
11844 None,
11845 manager.clone(),
11846 )
11847 .with_parent_completion_tx(tx);
11848 (WorkflowTool::new(manager, runtime), context, rx)
11849 }
11850
11851 fn native_lifecycle_driver(
11852 workspace: &Path,
11853 gates: Vec<GateSpec>,
11854 ) -> Arc<SubAgentWorkflowDriver> {
11855 let (tool, context, _rx) = native_lifecycle_tool(workspace);
11856 let state = shared_workflow_state(workspace);
11857 let run_id = "workflow_native_fixture".to_string();
11858 state.runs.lock().expect("runs").insert(
11859 run_id.clone(),
11860 WorkflowRunRecord::new(
11861 run_id.clone(),
11862 Some(context.state_namespace.clone()),
11863 None,
11864 None,
11865 None,
11866 ),
11867 );
11868 SubAgentWorkflowDriver::new(
11869 run_id,
11870 context.state_namespace.clone(),
11871 tool.manager,
11872 tool.runtime,
11873 state,
11874 None,
11875 WorkflowFleetBinding::None,
11876 gates,
11877 workspace.to_path_buf(),
11878 )
11879 }
11880
11881 fn terminal_workflow_receipt(payload: &str) -> Value {
11882 let (_, receipt) = payload
11883 .rsplit_once("<codewhale:subagent.done>")
11884 .expect("terminal envelope");
11885 serde_json::from_str(
11886 receipt
11887 .strip_suffix("</codewhale:subagent.done>")
11888 .expect("closing envelope"),
11889 )
11890 .expect("terminal metadata")
11891 }
11892
11893 #[tokio::test]
11894 async fn native_detached_workflow_wakes_once_and_foreground_does_not() {
11895 for (action, script, event) in [
11896 ("start", "return {ok: true};", "workflow.completed"),
11897 (
11898 "start",
11899 "throw new Error('failed native step');",
11900 "workflow.failed",
11901 ),
11902 ("run", "return {ok: true};", "workflow.completed"),
11903 ] {
11904 let tmp = tempfile::tempdir().expect("tempdir");
11905 let (tool, context, mut rx) = native_lifecycle_tool(tmp.path());
11906 let result = tool
11907 .execute(json!({"action": action, "script": script}), &context)
11908 .await
11909 .expect("run receipt");
11910 let result: Value = serde_json::from_str(&result.content).expect("run json");
11911 let run_id = result["run_id"].as_str().expect("run id");
11912 if action == "start" {
11913 let completion = tokio::time::timeout(std::time::Duration::from_secs(5), rx.recv())
11914 .await
11915 .expect("detached completion")
11916 .expect("parent inbox open");
11917 assert_eq!(completion.owner_session_id, context.state_namespace);
11918 assert_eq!(completion.agent_id, run_id);
11919 assert_eq!(
11920 terminal_workflow_receipt(&completion.payload)["event"],
11921 event
11922 );
11923 assert!(completion.payload.len() <= WORKFLOW_COMPLETION_MAX_BYTES);
11924 assert_eq!(live_workflow_count(tmp.path(), &context.state_namespace), 0);
11925 tool.execute(json!({"action":"status", "run_id":run_id}), &context)
11926 .await
11927 .expect("status remains addressable");
11928 tool.execute(json!({"action":"cancel", "run_id":run_id}), &context)
11929 .await
11930 .expect("terminal cancel is idempotent");
11931 } else {
11932 assert_eq!(result["status"], "completed");
11933 }
11934 assert!(rx.try_recv().is_err(), "no duplicate or foreground fan-in");
11935 }
11936 }
11937
11938 #[tokio::test]
11939 #[allow(clippy::await_holding_lock)]
11940 async fn native_terminal_journal_failure_agrees_across_status_event_and_parent_receipt() {
11941 let _retry_guard = workflow_test_retry_guard();
11942 for cancelled in [false, true] {
11943 let tmp = tempfile::tempdir().expect("tempdir");
11944 let requested = Arc::new(tokio::sync::Notify::new());
11945 let release = Arc::new(tokio::sync::Notify::new());
11946 let calls = Arc::new(AtomicUsize::new(0));
11947 let app = Router::new().route(
11948 "/{*path}",
11949 post({
11950 let requested = Arc::clone(&requested);
11951 let release = Arc::clone(&release);
11952 let calls = Arc::clone(&calls);
11953 move |Json(_body): Json<Value>| {
11954 let requested = Arc::clone(&requested);
11955 let release = Arc::clone(&release);
11956 let calls = Arc::clone(&calls);
11957 async move {
11958 calls.fetch_add(1, Ordering::SeqCst);
11959 requested.notify_one();
11960 release.notified().await;
11961 Json(json!({
11962 "id": "chatcmpl-journal-probe",
11963 "model": "deepseek-v4-flash",
11964 "choices": [{
11965 "index": 0,
11966 "message": {
11967 "role": "assistant",
11968 "content": "JOURNAL_CHILD_COMPLETED"
11969 },
11970 "finish_reason": "stop"
11971 }],
11972 "usage": {
11973 "prompt_tokens": 1,
11974 "completion_tokens": 1,
11975 "total_tokens": 2
11976 }
11977 }))
11978 }
11979 }
11980 }),
11981 );
11982 let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
11983 .await
11984 .expect("bind journal probe");
11985 let address = listener.local_addr().expect("journal probe address");
11986 let server = tokio::spawn(async move {
11987 let _ = axum::serve(listener, app).await;
11988 });
11989 let config = crate::config::Config {
11990 api_key: Some("test-key".to_string()),
11991 base_url: Some(format!("http://{address}/v1")),
11992 ..crate::config::Config::default()
11993 };
11994 let context = ToolContext::new(tmp.path().to_path_buf());
11995 let manager = new_shared_subagent_manager(tmp.path().to_path_buf(), 2);
11996 let (completion_tx, mut completion_rx) = mpsc::channel(16);
11997 let (event_tx, mut event_rx) = mpsc::channel(128);
11998 let runtime = SubAgentRuntime::new(
11999 CodewhaleClient::new(&config).expect("journal probe client"),
12000 "deepseek-v4-flash".to_string(),
12001 context.clone(),
12002 true,
12003 Some(event_tx),
12004 manager.clone(),
12005 )
12006 .with_parent_completion_tx(completion_tx);
12007 let parent_cancel = runtime.cancel_token.clone();
12008 let tool = WorkflowTool::new(manager, runtime);
12009 let started = tool
12010 .execute(
12011 json!({
12012 "action": "start",
12013 "plan": {
12014 "goal": "Retain truthful terminal evidence when the journal fails",
12015 "phases": [{
12016 "id": "journal-phase",
12017 "title": "Journal probe",
12018 "children": [{
12019 "id": "journal-child",
12020 "prompt": "Return the read-only probe result.",
12021 "type": "explore",
12022 "mode": "read_only"
12023 }]
12024 }]
12025 }
12026 }),
12027 &context,
12028 )
12029 .await
12030 .expect("launch succeeds while the journal is writable");
12031 let started: Value = serde_json::from_str(&started.content).expect("start json");
12032 let run_id = started["run_id"].as_str().expect("run id");
12033 tokio::time::timeout(std::time::Duration::from_secs(10), requested.notified())
12034 .await
12035 .expect("real workflow child reaches the loopback provider");
12036 assert_eq!(live_workflow_count(tmp.path(), &context.state_namespace), 1);
12037 let state = shared_workflow_state(tmp.path());
12038 let journal = state.journal_path();
12039 assert!(
12040 std::fs::read_to_string(journal)
12041 .expect("launch persisted")
12042 .contains(run_id)
12043 );
12044 // A directory at the append path fails even when tests run with
12045 // permissions that would bypass a read-only file's mode bits.
12046 std::fs::rename(journal, journal.with_extension("before-failure.jsonl"))
12047 .expect("preserve the successful launch journal");
12048 std::fs::create_dir(journal).expect("inject append failure after launch");
12049 if cancelled {
12050 parent_cancel.cancel();
12051 }
12052 release.notify_one();
12053
12054 let completion =
12055 tokio::time::timeout(std::time::Duration::from_secs(10), completion_rx.recv())
12056 .await
12057 .expect("workflow settles despite the failed append")
12058 .expect("parent receipt");
12059 server.abort();
12060 let expected_status = if cancelled { "cancelled" } else { "failed" };
12061 let receipt = terminal_workflow_receipt(&completion.payload);
12062 assert_eq!(receipt["status"], expected_status);
12063 assert_eq!(receipt["event"], "workflow.failed");
12064 assert!(completion.payload.contains("could not be persisted"));
12065 assert!(completion.payload.len() <= WORKFLOW_COMPLETION_MAX_BYTES);
12066 assert_eq!(live_workflow_count(tmp.path(), &context.state_namespace), 0);
12067 assert_eq!(calls.load(Ordering::SeqCst), 1);
12068 assert!(completion_rx.try_recv().is_err(), "only one parent receipt");
12069
12070 // Tool rebuilds share the same authoritative run cache.
12071 let rebuilt = WorkflowTool::new(tool.manager.clone(), tool.runtime.clone());
12072 let status = rebuilt
12073 .execute(json!({"action":"status", "run_id":run_id}), &context)
12074 .await
12075 .expect("status remains addressable after the failed append");
12076 let status: Value = serde_json::from_str(&status.content).expect("status json");
12077 assert_eq!(status["status"], expected_status);
12078 assert_eq!(status["execution"]["status"], expected_status);
12079 assert!(
12080 status["error"]
12081 .as_str()
12082 .unwrap()
12083 .contains("could not be persisted")
12084 );
12085 let expected_event = if cancelled {
12086 "run_cancelled"
12087 } else {
12088 "run_completed"
12089 };
12090 let mut final_event = None;
12091 while let Ok(event) = event_rx.try_recv() {
12092 if let Event::WorkflowUi { event, .. } = event
12093 && matches!(
12094 event["type"].as_str(),
12095 Some("run_completed" | "run_cancelled")
12096 )
12097 {
12098 final_event = Some(event);
12099 }
12100 }
12101 let final_event = final_event.expect("terminal UI event");
12102 assert_eq!(final_event["type"], expected_event);
12103 assert!(final_event.to_string().contains("could not be persisted"));
12104 let recorded_terminal = status["events"]
12105 .as_array()
12106 .unwrap()
12107 .iter()
12108 .rev()
12109 .find(|event| {
12110 matches!(
12111 event["type"].as_str(),
12112 Some("run_completed" | "run_cancelled")
12113 )
12114 })
12115 .expect("recorded terminal event");
12116 assert_eq!(recorded_terminal["type"], expected_event);
12117 assert!(
12118 recorded_terminal
12119 .to_string()
12120 .contains("could not be persisted")
12121 );
12122 let report = std::fs::read_to_string(
12123 tmp.path()
12124 .join(".codewhale/reports")
12125 .join(format!("{run_id}.md")),
12126 )
12127 .expect("best-effort report remains writable");
12128 assert!(report.contains(if cancelled {
12129 "- status: Cancelled"
12130 } else {
12131 "- status: Failed"
12132 }));
12133 assert!(report.contains("could not be persisted"));
12134 if cancelled {
12135 assert!(
12136 status["error"]
12137 .as_str()
12138 .unwrap()
12139 .contains("cancelled by its parent")
12140 );
12141 } else {
12142 assert_eq!(status["result"]["journal-child"], "JOURNAL_CHILD_COMPLETED");
12143 assert!(completion.payload.contains("JOURNAL_CHILD_COMPLETED"));
12144 assert!(report.contains("JOURNAL_CHILD_COMPLETED"));
12145 assert_eq!(final_event["status"], "failed");
12146 assert_eq!(
12147 status["execution"]["leaf_results"][0]["status"],
12148 "succeeded"
12149 );
12150 }
12151 assert!(
12152 journal.is_dir(),
12153 "no terminal durability is claimed after the failed append"
12154 );
12155 }
12156 }
12157
12158 #[tokio::test]
12159 async fn native_completion_is_enqueued_before_live_count_zero_and_bounded() {
12160 let tmp = tempfile::tempdir().expect("tempdir");
12161 let (tool, context, mut rx) = native_lifecycle_tool(tmp.path());
12162 let state = shared_workflow_state(tmp.path());
12163 let run_id = "workflow_bounded".to_string();
12164 let driver = SubAgentWorkflowDriver::new(
12165 run_id.clone(),
12166 context.state_namespace.clone(),
12167 tool.manager,
12168 tool.runtime,
12169 state.clone(),
12170 None,
12171 WorkflowFleetBinding::None,
12172 Vec::new(),
12173 tmp.path().to_path_buf(),
12174 );
12175 state.controllers.lock().expect("controllers").insert(
12176 run_id.clone(),
12177 Arc::new(
12178 WorkflowRunController::new(driver, WorkflowRunCancel::new())
12179 .with_parent_completion(true),
12180 ),
12181 );
12182 let mut record = WorkflowRunRecord::new(
12183 run_id,
12184 Some(context.state_namespace.clone()),
12185 None,
12186 None,
12187 None,
12188 );
12189 record.status = WorkflowRunStatus::Failed;
12190 record.workflow_goal = Some("🦀\u{0001}".repeat(10_000));
12191 record.error = Some("🦀\u{0001}".repeat(10_000));
12192 record.result = Some(json!({"preview":"🦀\u{0001}".repeat(10_000)}));
12193 assert_eq!(live_workflow_count(tmp.path(), &context.state_namespace), 1);
12194 assert_eq!(live_workflow_count(tmp.path(), "other-session"), 0);
12195 finish_workflow_controller(&state, &record);
12196 assert_eq!(live_workflow_count(tmp.path(), &context.state_namespace), 0);
12197 let completion = rx
12198 .try_recv()
12199 .expect("receipt already enqueued when idle is observable");
12200 assert!(
12201 completion.payload.len() <= WORKFLOW_COMPLETION_MAX_BYTES,
12202 "{} bytes",
12203 completion.payload.len()
12204 );
12205 assert_eq!(
12206 terminal_workflow_receipt(&completion.payload)["status"],
12207 "failed"
12208 );
12209 finish_workflow_controller(&state, &record);
12210 assert!(rx.try_recv().is_err(), "terminalization is exactly once");
12211 }
12212
12213 #[tokio::test]
12214 async fn native_parent_cancellation_between_children_settles_the_vm() {
12215 let tmp = tempfile::tempdir().expect("tempdir");
12216 let (mut tool, context, mut completion_rx) = native_lifecycle_tool(tmp.path());
12217 let parent_cancel = tool.runtime.cancel_token.clone();
12218 let (event_tx, mut event_rx) = mpsc::channel(32);
12219 tool.runtime.event_tx = Some(event_tx);
12220 tool.execute(
12221 json!({"action":"start", "script":"phase('between-children'); while (true) {}"}),
12222 &context,
12223 )
12224 .await
12225 .expect("detached start");
12226 tokio::time::timeout(std::time::Duration::from_secs(5), async {
12227 loop {
12228 if let Event::WorkflowUi { event, .. } =
12229 event_rx.recv().await.expect("event channel")
12230 && event["type"] == "phase_started"
12231 {
12232 break;
12233 }
12234 }
12235 })
12236 .await
12237 .expect("VM entered a phase with no child provider in flight");
12238 assert_eq!(live_workflow_count(tmp.path(), &context.state_namespace), 1);
12239 parent_cancel.cancel();
12240 let completion =
12241 tokio::time::timeout(std::time::Duration::from_secs(5), completion_rx.recv())
12242 .await
12243 .expect("cancel stops VM bytecode")
12244 .expect("cancel receipt");
12245 assert_eq!(
12246 terminal_workflow_receipt(&completion.payload)["status"],
12247 "cancelled"
12248 );
12249 assert_eq!(live_workflow_count(tmp.path(), &context.state_namespace), 0);
12250 assert_eq!(tool.manager.read().await.active_count(), 0);
12251 }
12252
12253 #[tokio::test]
12254 async fn native_cancel_closes_queued_admission_without_cancelling_parent() {
12255 let tmp = tempfile::tempdir().expect("tempdir");
12256 set_session_workflow_config(
12257 tmp.path(),
12258 codewhale_config::WorkflowConfigToml {
12259 max_concurrent: 1,
12260 ..Default::default()
12261 },
12262 );
12263 let driver = native_lifecycle_driver(tmp.path(), Vec::new());
12264 assert_eq!(driver.concurrent_gate.available_permits(), 1);
12265 let held = driver
12266 .concurrent_gate
12267 .clone()
12268 .acquire_owned()
12269 .await
12270 .expect("hold capacity");
12271 let mut queued = Box::pin(driver.spawn_task(exact_task_request("reviewer")));
12272 std::future::poll_fn(|cx| {
12273 assert!(std::future::Future::poll(queued.as_mut(), cx).is_pending());
12274 std::task::Poll::Ready(())
12275 })
12276 .await;
12277 driver.force_cancel_all();
12278 drop(held);
12279 let error = match queued.await {
12280 Ok(_) => panic!("cancelled admission spawned"),
12281 Err(err) => err,
12282 };
12283 assert!(error.to_string().contains("admission closed"), "{error}");
12284 assert_eq!(driver.child_counter.load(Ordering::SeqCst), 0);
12285 assert!(driver.child_ids.lock().expect("children").is_empty());
12286 assert!(
12287 !driver.parent_cancel_token.is_cancelled(),
12288 "workflow stop is scoped to its subtree"
12289 );
12290 }
12291
12292 #[tokio::test]
12293 async fn native_handoff_survives_rejected_spawn_and_pending_gates_block_completion() {
12294 let tmp = tempfile::tempdir().expect("tempdir");
12295 let gate: GateSpec = serde_json::from_value(json!({
12296 "id":"findings", "role":"scout", "on":"role_complete", "gate":"approve", "on_fail":"block",
12297 "blocks_role":"reviewer", "artifact_kind":"findings"
12298 })).expect("gate fixture");
12299 let driver = native_lifecycle_driver(tmp.path(), vec![gate]);
12300 assert!(
12301 driver.terminal_gate_failure().is_some(),
12302 "unevaluated prerequisite blocks terminal success"
12303 );
12304 driver.gate_board.lock().expect("board").gates.clear();
12305 assert!(
12306 driver.terminal_gate_failure().is_some(),
12307 "missing persisted gate also blocks success"
12308 );
12309 driver.evaluate_gates_for_completed_role(&RuntimeTaskRecord {
12310 agent_id: "scout-1".to_string(),
12311 label: None,
12312 role: Some("scout".to_string()),
12313 status: IrWorkflowRunStatus::Succeeded,
12314 output: Some("evidence to retain".to_string()),
12315 schema_error: None,
12316 usage: None,
12317 });
12318 assert!(
12319 driver.terminal_gate_failure().is_none(),
12320 "passed prerequisite releases the run"
12321 );
12322 let mut rejected = exact_task_request("reviewer");
12323 rejected.profile = Some("native-fixture-missing-profile".to_string());
12324 let error = match driver.spawn_task(rejected).await {
12325 Ok(_) => panic!("missing profile spawned"),
12326 Err(err) => err,
12327 };
12328 assert!(
12329 error.to_string().contains("native-fixture-missing-profile"),
12330 "{error}"
12331 );
12332 assert_eq!(driver.gate_board.lock().expect("board").artifacts.len(), 1);
12333 let mut retry = exact_task_request("reviewer");
12334 let handoffs = driver
12335 .prepare_request_for_gates(&mut retry)
12336 .expect("retry retains evidence");
12337 assert_eq!(handoffs.len(), 1);
12338 assert!(retry.description.contains("evidence to retain"));
12339 assert!(
12340 driver.state.runs.lock().expect("runs")[&driver.run_id]
12341 .events
12342 .iter()
12343 .all(|event| event.event_type() != "handoff_consumed")
12344 );
12345 }
12346
12347 #[tokio::test]
12348 async fn native_configured_shape_and_admission_limits_are_enforced() {
12349 let tmp = tempfile::tempdir().expect("tempdir");
12350 let (tool, context, _rx) = native_lifecycle_tool(tmp.path());
12351 set_session_workflow_config(
12352 tmp.path(),
12353 codewhale_config::WorkflowConfigToml {
12354 max_children: 2,
12355 max_depth: 1,
12356 max_concurrent: 1,
12357 ..Default::default()
12358 },
12359 );
12360 for children in [
12361 json!([{"prompt":"one"},{"prompt":"two"},{"prompt":"three"}]),
12362 json!([{"prompt":"one"},{"prompt":"two"}]),
12363 ] {
12364 let error = tool
12365 .execute(
12366 json!({"action":"start", "plan":{"goal":"bounded", "children":children}}),
12367 &context,
12368 )
12369 .await
12370 .expect_err("configured total or structural depth rejects before launch");
12371 assert!(error.to_string().contains("workflow.max_"), "{error}");
12372 }
12373 assert!(
12374 shared_workflow_state(tmp.path())
12375 .runs
12376 .lock()
12377 .expect("runs")
12378 .is_empty()
12379 );
12380 let initial_spawn_depth = tool.runtime.spawn_depth;
12381 let accepted = tool.execute(json!({"action":"run", "plan":{
12382 "goal":"two sequential leaves fit structural depth one", "phases":[
12383 {"id":"first", "children":[{"prompt":"one", "profile":"missing-boundary-profile"}]},
12384 {"id":"second", "children":[{"prompt":"two", "profile":"missing-boundary-profile"}]}
12385 ]
12386 }}), &context).await.expect("sequence wrappers do not increase structural depth");
12387 let accepted: Value = serde_json::from_str(&accepted.content).expect("accepted run");
12388 assert_eq!(
12389 accepted["status"], "failed",
12390 "admitted plan, then deliberate missing-profile rejection"
12391 );
12392 assert_eq!(
12393 tool.runtime.spawn_depth, initial_spawn_depth,
12394 "plan shape never rewrites delegation depth"
12395 );
12396 set_session_workflow_config(
12397 tmp.path(),
12398 codewhale_config::WorkflowConfigToml {
12399 max_children: 1,
12400 max_concurrent: 1,
12401 ..Default::default()
12402 },
12403 );
12404 let driver = native_lifecycle_driver(tmp.path(), Vec::new());
12405 let mut invalid = exact_task_request("reviewer");
12406 invalid.profile = Some("missing-first-attempt-profile".to_string());
12407 let first = driver.spawn_task(invalid.clone()).await;
12408 assert!(first.is_err());
12409 let second = match driver.spawn_task(invalid).await {
12410 Ok(_) => panic!("cap bypassed"),
12411 Err(err) => err,
12412 };
12413 assert!(
12414 second
12415 .to_string()
12416 .contains("workflow.max_children limit (1)"),
12417 "{second}"
12418 );
12419 assert_eq!(driver.child_counter.load(Ordering::SeqCst), 1);
12420 assert!(driver.child_ids.lock().expect("children").is_empty());
12421 for name in ["max_children", "max_concurrent", "max_depth"] {
12422 let mut config = serde_json::to_value(codewhale_config::WorkflowConfigToml::default())
12423 .expect("config");
12424 config[name] = json!(0);
12425 set_session_workflow_config(
12426 tmp.path(),
12427 serde_json::from_value(config).expect("zero config"),
12428 );
12429 let error = tool
12430 .execute(json!({"action":"start", "script":"return 1;"}), &context)
12431 .await
12432 .expect_err("zero ceiling fails without waiting on a semaphore");
12433 assert!(
12434 error.to_string().contains(&format!("workflow.{name}")),
12435 "{error}"
12436 );
12437 }
12438 }
12439
12440 #[test]
12441 fn native_gate_identity_is_normalized_and_unsupported_or_duplicate_gates_reject() {
12442 let gate = json!({"id":" check ","role":" Scout ","on":"role_complete", "gate":"approve", "on_fail":"block", "blocks_role":" Reviewer "});
12443 let mut gates: Vec<GateSpec> = serde_json::from_value(json!([gate])).expect("gates");
12444 validate_runtime_gates(&mut gates).expect("normalize gate");
12445 assert_eq!(gates[0].id, "check");
12446 assert_eq!(gates[0].role, "scout");
12447 assert_eq!(gates[0].blocks_role.as_deref(), Some("reviewer"));
12448 let mut duplicate = vec![gates[0].clone(), gates[0].clone()];
12449 assert!(validate_runtime_gates(&mut duplicate).is_err());
12450 for (field, value) in [
12451 ("id", " "),
12452 ("role", " "),
12453 ("blocks_role", " "),
12454 ("on", "role_start"),
12455 ] {
12456 let mut candidate = serde_json::to_value(&gates[0]).expect("gate json");
12457 candidate[field] = json!(value);
12458 let mut candidate =
12459 [serde_json::from_value(candidate).expect("syntactically valid gate")];
12460 assert!(validate_runtime_gates(&mut candidate).is_err(), "{field}");
12461 }
12462 }
12463
12464 #[tokio::test]
12465 async fn native_default_shared_budget_applies_only_without_an_explicit_budget() {
12466 let tmp = tempfile::tempdir().expect("tempdir");
12467 let (tool, context, _rx) = native_lifecycle_tool(tmp.path());
12468 set_session_workflow_config(
12469 tmp.path(),
12470 codewhale_config::WorkflowConfigToml {
12471 default_token_budget: 123,
12472 ..Default::default()
12473 },
12474 );
12475 for (explicit, expected) in [(None, 123), (Some(17), 17), (Some(246), 246)] {
12476 let mut input = json!({"action":"run", "script":"return budget.remaining();"});
12477 if let Some(explicit) = explicit {
12478 input["token_budget"] = json!(explicit);
12479 }
12480 let receipt = tool.execute(input, &context).await.expect("budget run");
12481 let receipt: Value = serde_json::from_str(&receipt.content).expect("budget receipt");
12482 assert_eq!(receipt["token_budget"], expected);
12483 assert_eq!(
12484 receipt["result"], expected,
12485 "the same effective cap reaches the VM"
12486 );
12487 }
12488 }
12489
12490 #[tokio::test]
12491 async fn native_cancel_preserves_completed_outputs_and_late_completion_cannot_reverse_it() {
12492 let tmp = tempfile::tempdir().expect("tempdir");
12493 let driver = native_lifecycle_driver(tmp.path(), Vec::new());
12494 for id in ["completed-child", "running-child"] {
12495 driver.record_child(id);
12496 driver.task_records.lock().expect("records").insert(
12497 id.to_string(),
12498 RuntimeTaskRecord {
12499 agent_id: id.to_string(),
12500 label: None,
12501 role: None,
12502 status: IrWorkflowRunStatus::Running,
12503 output: None,
12504 schema_error: None,
12505 usage: None,
12506 },
12507 );
12508 }
12509 driver.record_task_completion(
12510 "completed-child",
12511 &TaskCompletion::Completed {
12512 text: "completed evidence must survive".to_string(),
12513 },
12514 None,
12515 );
12516 driver.finalize_running_tasks_cancelled();
12517 driver.record_task_completion(
12518 "running-child",
12519 &TaskCompletion::Completed {
12520 text: "late completion after cancellation".to_string(),
12521 },
12522 None,
12523 );
12524 let records = driver.task_records.lock().expect("records");
12525 assert_eq!(
12526 records["completed-child"].status,
12527 IrWorkflowRunStatus::Succeeded
12528 );
12529 assert_eq!(
12530 records["completed-child"].output.as_deref(),
12531 Some("completed evidence must survive")
12532 );
12533 assert_eq!(
12534 records["running-child"].status,
12535 IrWorkflowRunStatus::Cancelled
12536 );
12537 assert!(records["running-child"].output.is_none());
12538 }
12539
12540 #[tokio::test]
12541 async fn native_completion_before_registration_releases_capacity_and_cancelled_waiters() {
12542 let tmp = tempfile::tempdir().expect("tempdir");
12543 set_session_workflow_config(
12544 tmp.path(),
12545 codewhale_config::WorkflowConfigToml {
12546 max_concurrent: 1,
12547 ..Default::default()
12548 },
12549 );
12550 let driver = native_lifecycle_driver(tmp.path(), Vec::new());
12551 let held = driver
12552 .concurrent_gate
12553 .clone()
12554 .acquire_owned()
12555 .await
12556 .expect("capacity");
12557 driver.deliver_completion(
12558 "instant".to_string(),
12559 TaskCompletion::Completed {
12560 text: "ready".to_string(),
12561 },
12562 None,
12563 );
12564 driver
12565 .spawn_permits
12566 .lock()
12567 .expect("permits")
12568 .insert("instant".to_string(), held);
12569 driver.record_task_request("instant", &exact_task_request("reviewer"));
12570 let (tx, rx) = oneshot::channel();
12571 driver.add_waiter_or_complete("instant".to_string(), tx);
12572 assert!(
12573 matches!(rx.await.expect("early completion delivered"), TaskCompletion::Completed {text} if text == "ready")
12574 );
12575 assert_eq!(
12576 driver.concurrent_gate.available_permits(),
12577 1,
12578 "instant completion must not leak the sole permit"
12579 );
12580 // Reproduce the other interleaving: the completion checked records
12581 // before registration, but published pending after its first peek.
12582 driver.record_task_request("pending-after-peek", &exact_task_request("reviewer"));
12583 driver
12584 .completion_state
12585 .lock()
12586 .expect("completion state")
12587 .pending
12588 .insert(
12589 "pending-after-peek".to_string(),
12590 PendingCompletion {
12591 completion: TaskCompletion::Completed {
12592 text: "preserved receipt".to_string(),
12593 },
12594 usage: None,
12595 },
12596 );
12597 let (tx, rx) = oneshot::channel();
12598 driver.add_waiter_or_complete("pending-after-peek".to_string(), tx);
12599 assert!(matches!(
12600 rx.await.expect("completion"),
12601 TaskCompletion::Completed { .. }
12602 ));
12603 assert_eq!(
12604 driver.task_records.lock().expect("records")["pending-after-peek"]
12605 .output
12606 .as_deref(),
12607 Some("preserved receipt")
12608 );
12609 driver.force_cancel_all();
12610 driver.record_task_request("late-registration", &exact_task_request("reviewer"));
12611 let (tx, rx) = oneshot::channel();
12612 driver.add_waiter_or_complete("late-registration".to_string(), tx);
12613 assert!(matches!(
12614 rx.await.expect("late waiter released"),
12615 TaskCompletion::Cancelled
12616 ));
12617 assert_eq!(
12618 driver.task_records.lock().expect("records")["late-registration"].status,
12619 IrWorkflowRunStatus::Cancelled
12620 );
12621 }
12622
12623 fn stub_client() -> CodewhaleClient {
12624 let _ = rustls::crypto::ring::default_provider().install_default();
12625 let config = crate::config::Config {
12626 api_key: Some("test-key".to_string()),
12627 ..crate::config::Config::default()
12628 };
12629 CodewhaleClient::new(&config).expect("stub client should construct")
12630 }
12631
12632 async fn fake_chat_client(response_text: &str) -> (CodewhaleClient, Arc<AtomicUsize>) {
12633 let (client, calls, _) = fake_chat_client_capturing(response_text).await;
12634 (client, calls)
12635 }
12636
12637 async fn fake_chat_client_responses(
12638 response_texts: &[&str],
12639 ) -> (CodewhaleClient, Arc<AtomicUsize>) {
12640 let (client, calls, _) = fake_chat_client_capturing_responses(response_texts).await;
12641 (client, calls)
12642 }
12643
12644 pub(super) async fn fake_chat_client_capturing(
12645 response_text: &str,
12646 ) -> (CodewhaleClient, Arc<AtomicUsize>, Arc<Mutex<Vec<Value>>>) {
12647 fake_chat_client_capturing_responses(&[response_text]).await
12648 }
12649
12650 async fn fake_chat_client_capturing_responses(
12651 response_texts: &[&str],
12652 ) -> (CodewhaleClient, Arc<AtomicUsize>, Arc<Mutex<Vec<Value>>>) {
12653 assert!(
12654 !response_texts.is_empty(),
12655 "fake chat client needs at least one response"
12656 );
12657 let calls = Arc::new(AtomicUsize::new(0));
12658 let bodies = Arc::new(Mutex::new(Vec::new()));
12659 let response_texts = Arc::new(
12660 response_texts
12661 .iter()
12662 .map(|response| (*response).to_string())
12663 .collect::<Vec<_>>(),
12664 );
12665 let app = Router::new().route(
12666 "/{*path}",
12667 post({
12668 let calls = Arc::clone(&calls);
12669 let bodies = Arc::clone(&bodies);
12670 let response_texts = Arc::clone(&response_texts);
12671 move |Json(body): Json<Value>| {
12672 let calls = Arc::clone(&calls);
12673 let bodies = Arc::clone(&bodies);
12674 let response_texts = Arc::clone(&response_texts);
12675 async move {
12676 bodies.lock().expect("capture body").push(body);
12677 let attempt = calls.fetch_add(1, Ordering::SeqCst) + 1;
12678 let response_text = if response_texts.len() == 1 {
12679 response_texts[0].clone()
12680 } else {
12681 response_texts
12682 .get(attempt - 1)
12683 .unwrap_or_else(|| {
12684 panic!(
12685 "fake chat server received call {attempt} but only {} responses were supplied",
12686 response_texts.len()
12687 )
12688 })
12689 .clone()
12690 };
12691 Json(json!({
12692 "id": format!("chatcmpl-workflow-test-{attempt}"),
12693 "model": "deepseek-v4-flash",
12694 "choices": [{
12695 "index": 0,
12696 "message": {
12697 "role": "assistant",
12698 "content": response_text
12699 },
12700 "finish_reason": "stop"
12701 }],
12702 "usage": {
12703 "prompt_tokens": 1,
12704 "completion_tokens": 1,
12705 "total_tokens": 2
12706 }
12707 }))
12708 }
12709 }
12710 }),
12711 );
12712
12713 let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
12714 .await
12715 .expect("bind fake chat server");
12716 let addr = listener.local_addr().expect("fake chat server addr");
12717 tokio::spawn(async move {
12718 let _ = axum::serve(listener, app).await;
12719 });
12720
12721 let config = crate::config::Config {
12722 api_key: Some("test-key".to_string()),
12723 base_url: Some(format!("http://{addr}/v1")),
12724 ..crate::config::Config::default()
12725 };
12726 (
12727 CodewhaleClient::new(&config).expect("fake chat client"),
12728 calls,
12729 bodies,
12730 )
12731 }
12732
12733 pub(super) fn workflow_test_retry_guard() -> std::sync::MutexGuard<'static, ()> {
12734 let guard = crate::retry_status::test_guard();
12735 crate::retry_status::clear();
12736 crate::retry_status::clear_rate_limit();
12737 guard
12738 }
12739 }
12740
12740 lines RUST