返回 CodeWhale
mailbox.rs
根目录 / crates / tui / src / tools / subagent / mailbox.rs
1 //! Mailbox abstraction for sub-agent runtime coordination.
2 //!
3 //! Monotonic sequence numbers give every consumer a consistent ordering even
4 //! when multiple subscribers (e.g. UI card + parent agent) drain
5 //! independently; close-as-cancel lets a single signal both stop new mail and
6 //! propagate cancellation through nested children.
7
8 use std::collections::VecDeque;
9 use std::sync::Arc;
10 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
11 #[cfg(test)]
12 use std::time::Duration;
13
14 use serde::{Deserialize, Serialize};
15 use tokio::sync::{mpsc, watch};
16 use tokio_util::sync::CancellationToken;
17
18 #[cfg(test)]
19 use crate::config::ApiProvider;
20 use crate::tools::todo::TodoListSnapshot;
21 use codewhale_models::Usage;
22
23 use super::FleetRole;
24
25 /// Stable, structured progress envelope shared across the sub-agent surface.
26 ///
27 /// Tracks the lifecycle of a single agent (identified by `agent_id`) end to
28 /// end: spawn, per-step progress, tool execution, completion / failure /
29 /// cancellation, and parent → child topology so consumers can render trees.
30 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
31 #[serde(tag = "kind", rename_all = "snake_case")]
32 pub enum MailboxMessage {
33 /// Agent has been started (background task is running).
34 Started {
35 agent_id: String,
36 agent_type: String,
37 },
38 /// Free-form human-readable progress (mirrors `Event::AgentProgress`).
39 Progress { agent_id: String, status: String },
40 /// A tool call inside the agent has started.
41 ToolCallStarted {
42 agent_id: String,
43 tool_name: String,
44 step: u32,
45 },
46 /// A tool call inside the agent has finished.
47 ToolCallCompleted {
48 agent_id: String,
49 tool_name: String,
50 step: u32,
51 ok: bool,
52 },
53 /// A child agent was spawned by this agent.
54 ChildSpawned { parent_id: String, child_id: String },
55 /// Agent completed successfully (carries the summary line shown in the
56 /// transcript; full result is still available through the transcript handle).
57 Completed { agent_id: String, summary: String },
58 /// Agent failed with the carried error message.
59 Failed { agent_id: String, error: String },
60 /// Agent was interrupted (e.g. API timeout) with a continuable
61 /// checkpoint; the worker is parked waiting for continuation input.
62 Interrupted { agent_id: String, reason: String },
63 /// Cancellation propagated to this agent.
64 Cancelled { agent_id: String },
65 /// This agent's **own** bounded To-do snapshot (#4810).
66 ///
67 /// Published by the agent that owns the ledger, from its private list, so a
68 /// consumer keyed on `agent_id` can never attribute a parent's or a
69 /// sibling's work to this agent. Emitted only when the snapshot actually
70 /// changes; the payload is the canonical [`TodoListSnapshot`], not a second
71 /// ledger.
72 WorkState {
73 agent_id: String,
74 /// Absent in older persisted payloads, which predate per-agent Work
75 /// state; those deserialize to an empty (no work stated) snapshot.
76 #[serde(default)]
77 todo: TodoListSnapshot,
78 },
79 /// Incremental token usage from a sub-agent's API call.
80 /// Published after each turn so the parent's cost counter updates live.
81 TokenUsage {
82 agent_id: String,
83 /// Stable identity of the provider response. Runtime accounting uses
84 /// this across direct durability, mailbox replay, and restart dedupe.
85 source_id: String,
86 /// Immutable provider/model/billing evidence captured before the
87 /// child request was sent. Boxed: the envelope dwarfs every other
88 /// variant, and mailboxes queue many messages.
89 route: Box<crate::cost_status::EffectiveRouteEnvelope>,
90 /// Provider usage payload, including cache-hit/cache-miss fields.
91 usage: Usage,
92 },
93 }
94
95 impl MailboxMessage {
96 /// `agent_id` of the message subject (for `ChildSpawned` this is the
97 /// child, since that's the new lifecycle being announced).
98 #[must_use]
99 pub fn agent_id(&self) -> &str {
100 match self {
101 Self::Started { agent_id, .. }
102 | Self::Progress { agent_id, .. }
103 | Self::ToolCallStarted { agent_id, .. }
104 | Self::ToolCallCompleted { agent_id, .. }
105 | Self::Completed { agent_id, .. }
106 | Self::Failed { agent_id, .. }
107 | Self::Interrupted { agent_id, .. }
108 | Self::Cancelled { agent_id }
109 | Self::WorkState { agent_id, .. }
110 | Self::TokenUsage { agent_id, .. } => agent_id,
111 Self::ChildSpawned { child_id, .. } => child_id,
112 }
113 }
114
115 pub(crate) fn started(agent_id: impl Into<String>, agent_type: FleetRole) -> Self {
116 Self::Started {
117 agent_id: agent_id.into(),
118 agent_type: agent_type.as_str().to_string(),
119 }
120 }
121
122 pub(crate) fn progress(agent_id: impl Into<String>, status: impl Into<String>) -> Self {
123 Self::Progress {
124 agent_id: agent_id.into(),
125 status: status.into(),
126 }
127 }
128
129 pub(crate) fn work_state(agent_id: impl Into<String>, todo: TodoListSnapshot) -> Self {
130 Self::WorkState {
131 agent_id: agent_id.into(),
132 todo,
133 }
134 }
135
136 pub(crate) fn token_usage(
137 agent_id: impl Into<String>,
138 source_id: impl Into<String>,
139 route: crate::cost_status::EffectiveRouteEnvelope,
140 usage: Usage,
141 ) -> Self {
142 Self::TokenUsage {
143 agent_id: agent_id.into(),
144 source_id: source_id.into(),
145 route: Box::new(route),
146 usage,
147 }
148 }
149 }
150
151 /// One delivery: a sequence number plus the message. The sequence is
152 /// monotonic across the entire mailbox (not per-agent) so a single ordering
153 /// is well-defined even when multiple sub-agents share one mailbox.
154 #[derive(Debug, Clone, PartialEq, Eq)]
155 pub struct MailboxEnvelope {
156 pub seq: u64,
157 pub message: MailboxMessage,
158 }
159
160 /// Sender side of the mailbox.
161 ///
162 /// Cheaply cloneable (everything inside is `Arc`/atomic). Cloning a
163 /// `Mailbox` shares the same delivery channel, sequence counter, watch
164 /// notifier, and close/cancel state — so a child runtime that clones its
165 /// parent's `Mailbox` participates in the same stream.
166 #[derive(Clone)]
167 pub struct Mailbox {
168 inner: Arc<MailboxInner>,
169 }
170
171 struct MailboxInner {
172 tx: mpsc::UnboundedSender<MailboxEnvelope>,
173 next_seq: AtomicU64,
174 seq_tx: watch::Sender<u64>,
175 closed: AtomicBool,
176 /// Linearizes publication against turn-end sealing. Without this gate a
177 /// producer could observe `closed = false`, lose the race to the turn
178 /// completion barrier, and enqueue usage after `TurnComplete`.
179 send_gate: std::sync::Mutex<()>,
180 #[cfg(test)]
181 cancel_token: CancellationToken,
182 }
183
184 /// Receiver side of the mailbox. Not `Clone` — only the original creator
185 /// can drain. Use `Mailbox::subscribe()` for fanout (UI cards + parent both
186 /// observing the same stream).
187 pub struct MailboxReceiver {
188 rx: mpsc::UnboundedReceiver<MailboxEnvelope>,
189 pending: VecDeque<MailboxEnvelope>,
190 }
191
192 impl Mailbox {
193 /// Create a new mailbox bound to the given cancellation token. Closing
194 /// the mailbox (or dropping the last sender) cancels this token. Runtimes
195 /// that derive from the same token observe that cancellation; detached
196 /// background `agent` sessions use their own runtime token.
197 #[must_use]
198 pub fn new(cancel_token: CancellationToken) -> (Self, MailboxReceiver) {
199 #[cfg(not(test))]
200 let _ = cancel_token;
201 let (tx, rx) = mpsc::unbounded_channel();
202 let (seq_tx, _) = watch::channel(0);
203 let inner = MailboxInner {
204 tx,
205 next_seq: AtomicU64::new(0),
206 seq_tx,
207 closed: AtomicBool::new(false),
208 send_gate: std::sync::Mutex::new(()),
209 #[cfg(test)]
210 cancel_token,
211 };
212 (
213 Self {
214 inner: Arc::new(inner),
215 },
216 MailboxReceiver {
217 rx,
218 pending: VecDeque::new(),
219 },
220 )
221 }
222
223 /// Subscribe to seq-bump notifications. Each `recv()` returns when the
224 /// sequence counter advances, signaling new mail without copying it —
225 /// the consumer then calls `drain` (or `recv_one` on its own receiver).
226 /// Multiple subscribers may exist; this is the fanout primitive.
227 #[cfg(test)]
228 #[must_use]
229 pub fn subscribe(&self) -> watch::Receiver<u64> {
230 self.inner.seq_tx.subscribe()
231 }
232
233 /// Send a message; returns `Some(seq)` on success, `None` if the
234 /// mailbox is already closed (callers should treat this as "the
235 /// receiver is gone, stop publishing").
236 pub fn send(&self, message: MailboxMessage) -> Option<u64> {
237 let _send_gate = self
238 .inner
239 .send_gate
240 .lock()
241 .unwrap_or_else(|error| error.into_inner());
242 if self.inner.closed.load(Ordering::Acquire) {
243 return None;
244 }
245 let seq = self.inner.next_seq.fetch_add(1, Ordering::Relaxed) + 1;
246 let envelope = MailboxEnvelope { seq, message };
247 if self.inner.tx.send(envelope).is_err() {
248 return None;
249 }
250 let _ = self.inner.seq_tx.send_replace(seq);
251 Some(seq)
252 }
253
254 /// Stop publication for this turn without cancelling detached workers.
255 ///
256 /// The engine seals, drains, and awaits the mailbox before it emits
257 /// `TurnComplete`. The send gate makes this a hard ordering boundary:
258 /// once this returns, every accepted envelope is already in the receiver
259 /// and no later worker message can attach itself to the completed turn.
260 pub(crate) fn seal(&self) {
261 let _send_gate = self
262 .inner
263 .send_gate
264 .lock()
265 .unwrap_or_else(|error| error.into_inner());
266 self.inner.closed.store(true, Ordering::Release);
267 }
268
269 /// Whether the mailbox has been closed.
270 #[cfg(test)]
271 #[must_use]
272 pub fn is_closed(&self) -> bool {
273 self.inner.closed.load(Ordering::Acquire)
274 }
275
276 /// Close the mailbox AND cancel the bound cancellation token.
277 ///
278 /// "Close-as-cancel": there's no useful state where the consumer is gone
279 /// but producers bound to this mailbox token should keep publishing.
280 /// Closing cancels the bound token; directly derived `child_runtime()`
281 /// children observe it, while detached `agent` sessions rely on their
282 /// own explicit cancellation.
283 #[cfg(test)]
284 pub fn close(&self) {
285 let was_closed = self.inner.closed.load(Ordering::Acquire);
286 self.seal();
287 if !was_closed {
288 self.inner.cancel_token.cancel();
289 }
290 }
291 }
292
293 impl MailboxReceiver {
294 #[cfg(test)]
295 fn sync_pending(&mut self) {
296 while let Ok(env) = self.rx.try_recv() {
297 self.pending.push_back(env);
298 }
299 }
300
301 /// Whether any envelopes are buffered (or arrived since last check).
302 #[cfg(test)]
303 pub fn has_pending(&mut self) -> bool {
304 self.sync_pending();
305 !self.pending.is_empty()
306 }
307
308 /// Drain all currently available envelopes, in delivery order.
309 #[cfg(test)]
310 pub fn drain(&mut self) -> Vec<MailboxEnvelope> {
311 self.sync_pending();
312 self.pending.drain(..).collect()
313 }
314
315 /// Await the next envelope, with backpressure-aware blocking. Returns
316 /// `None` when every sender has been dropped and the buffer is drained.
317 pub async fn recv(&mut self) -> Option<MailboxEnvelope> {
318 if let Some(env) = self.pending.pop_front() {
319 return Some(env);
320 }
321 self.rx.recv().await
322 }
323
324 /// Drain all envelopes accepted before a mailbox was sealed.
325 pub(crate) fn drain_available(&mut self) -> Vec<MailboxEnvelope> {
326 while let Ok(envelope) = self.rx.try_recv() {
327 self.pending.push_back(envelope);
328 }
329 self.pending.drain(..).collect()
330 }
331
332 /// Awaits the next envelope with a timeout. Useful in tests.
333 #[cfg(test)]
334 pub async fn recv_timeout(&mut self, timeout: Duration) -> Option<MailboxEnvelope> {
335 tokio::time::timeout(timeout, self.recv())
336 .await
337 .ok()
338 .flatten()
339 }
340 }
341
342 #[cfg(test)]
343 mod tests {
344 use super::*;
345 use tokio::time::Duration;
346
347 fn open() -> (Mailbox, MailboxReceiver, CancellationToken) {
348 let token = CancellationToken::new();
349 let (mb, rx) = Mailbox::new(token.clone());
350 (mb, rx, token)
351 }
352
353 fn test_route(
354 provider: ApiProvider,
355 model: &str,
356 ) -> crate::cost_status::EffectiveRouteEnvelope {
357 crate::cost_status::EffectiveRouteEnvelope::capture(
358 None,
359 provider,
360 provider.as_str(),
361 model,
362 Some(provider.default_base_url()),
363 chrono::Utc::now(),
364 )
365 }
366
367 #[tokio::test]
368 async fn mailbox_assigns_monotonic_sequence_numbers() {
369 let (mb, _rx, _tok) = open();
370 let s1 = mb
371 .send(MailboxMessage::progress("a", "one"))
372 .expect("seq 1");
373 let s2 = mb
374 .send(MailboxMessage::progress("a", "two"))
375 .expect("seq 2");
376 let s3 = mb
377 .send(MailboxMessage::progress("b", "three"))
378 .expect("seq 3");
379 assert_eq!(s1, 1);
380 assert_eq!(s2, 2);
381 assert_eq!(s3, 3);
382 assert!(s2 > s1 && s3 > s2);
383 }
384
385 #[tokio::test]
386 async fn mailbox_drains_in_delivery_order() {
387 let (mb, mut rx, _tok) = open();
388 mb.send(MailboxMessage::progress("a", "first"));
389 mb.send(MailboxMessage::progress("a", "second"));
390 mb.send(MailboxMessage::Completed {
391 agent_id: "a".into(),
392 summary: "done".into(),
393 });
394 let drained = rx.drain();
395 assert_eq!(drained.len(), 3);
396 assert_eq!(drained[0].seq, 1);
397 assert_eq!(drained[1].seq, 2);
398 assert_eq!(drained[2].seq, 3);
399 assert!(matches!(
400 drained[0].message,
401 MailboxMessage::Progress { .. }
402 ));
403 assert!(matches!(
404 drained[2].message,
405 MailboxMessage::Completed { .. }
406 ));
407 assert!(!rx.has_pending());
408 }
409
410 #[tokio::test]
411 async fn subscribers_receive_seq_bumps_for_backpressure() {
412 let (mb, _rx, _tok) = open();
413 let mut sub_a = mb.subscribe();
414 let mut sub_b = mb.subscribe();
415 // Initial state: both at 0.
416 assert_eq!(*sub_a.borrow(), 0);
417 assert_eq!(*sub_b.borrow(), 0);
418
419 mb.send(MailboxMessage::progress("x", "tick"));
420 sub_a.changed().await.expect("subscriber a sees bump");
421 sub_b.changed().await.expect("subscriber b sees bump");
422 assert_eq!(*sub_a.borrow(), 1);
423 assert_eq!(*sub_b.borrow(), 1);
424
425 // A second send updates both subscribers' watch values too — even
426 // though they share a single watch channel, fanout is N-to-many.
427 mb.send(MailboxMessage::progress("x", "tick2"));
428 sub_a.changed().await.expect("a sees second bump");
429 assert_eq!(*sub_a.borrow(), 2);
430 }
431
432 #[tokio::test]
433 async fn close_cancels_bound_token_and_blocks_further_sends() {
434 let (mb, _rx, token) = open();
435 assert!(!token.is_cancelled());
436 mb.send(MailboxMessage::progress("a", "before close"));
437 mb.close();
438 assert!(token.is_cancelled(), "close-as-cancel: token must fire");
439 assert!(mb.is_closed());
440 // Further sends are no-ops, returning None instead of poisoning seq.
441 assert!(
442 mb.send(MailboxMessage::progress("a", "after close"))
443 .is_none()
444 );
445 }
446
447 #[test]
448 fn turn_end_seal_forms_a_flush_barrier_without_cancelling_worker() {
449 let (mb, mut rx, token) = open();
450 assert_eq!(
451 mb.send(MailboxMessage::progress("a", "accepted before barrier")),
452 Some(1)
453 );
454 mb.seal();
455
456 assert!(!token.is_cancelled(), "detached worker is not cancelled");
457 assert!(
458 mb.send(MailboxMessage::progress("a", "too late")).is_none(),
459 "no event may be accepted after the completion barrier"
460 );
461 let drained = rx.drain_available();
462 assert_eq!(drained.len(), 1);
463 assert_eq!(drained[0].seq, 1);
464 }
465
466 #[tokio::test]
467 async fn close_propagates_to_child_tokens_across_max_spawn_depth() {
468 // Mirror the runtime: root → child → grandchild (default depth 3).
469 let root = CancellationToken::new();
470 let child = root.child_token();
471 let grandchild = child.child_token();
472 let (mb, _rx) = Mailbox::new(root.clone());
473
474 assert!(!child.is_cancelled());
475 assert!(!grandchild.is_cancelled());
476 mb.close();
477 assert!(child.is_cancelled(), "child inherits root close");
478 assert!(
479 grandchild.is_cancelled(),
480 "grandchild inherits too — covers default max_spawn_depth = 3"
481 );
482 }
483
484 #[tokio::test]
485 async fn recv_returns_envelope_then_none_after_close_and_drop() {
486 let (mb, mut rx, _tok) = open();
487 mb.send(MailboxMessage::progress("a", "queued"));
488 let env = rx.recv().await.expect("buffered envelope");
489 assert_eq!(env.seq, 1);
490
491 // After closing AND dropping the sender, recv must yield None.
492 mb.close();
493 drop(mb);
494 let next = rx.recv_timeout(Duration::from_millis(100)).await;
495 assert!(next.is_none(), "drained + dropped → recv yields None");
496 }
497
498 #[tokio::test]
499 async fn cloned_mailbox_shares_sequence_and_close_state() {
500 let (mb, mut rx, token) = open();
501 let mb_clone = mb.clone();
502 let s1 = mb
503 .send(MailboxMessage::progress("a", "from original"))
504 .unwrap();
505 let s2 = mb_clone
506 .send(MailboxMessage::progress("a", "from clone"))
507 .unwrap();
508 assert_eq!(s1, 1);
509 assert_eq!(s2, 2, "clones share the seq counter");
510
511 let drained = rx.drain();
512 assert_eq!(drained.len(), 2);
513
514 // Closing through one clone closes them all (the AtomicBool is shared).
515 mb_clone.close();
516 assert!(mb.is_closed());
517 assert!(token.is_cancelled());
518 }
519
520 #[test]
521 fn work_state_payload_round_trips_and_tolerates_a_missing_snapshot() {
522 use crate::tools::todo::{TodoItem, TodoStatus};
523
524 let message = MailboxMessage::work_state(
525 "agent_child",
526 TodoListSnapshot {
527 items: vec![TodoItem {
528 id: 2,
529 content: "write the projection".to_string(),
530 status: TodoStatus::InProgress,
531 }],
532 completion_pct: 50,
533 in_progress_id: Some(2),
534 },
535 );
536 let encoded = serde_json::to_string(&message).expect("encode");
537 let decoded: MailboxMessage = serde_json::from_str(&encoded).expect("decode");
538 assert_eq!(decoded, message);
539
540 // An older payload that predates per-agent Work state decodes to an
541 // empty snapshot rather than failing the whole stream.
542 let legacy: MailboxMessage =
543 serde_json::from_str(r#"{"kind":"work_state","agent_id":"agent_old"}"#)
544 .expect("legacy");
545 assert_eq!(legacy.agent_id(), "agent_old");
546 match legacy {
547 MailboxMessage::WorkState { todo, .. } => assert!(todo.is_empty()),
548 other => panic!("expected work state, got {other:?}"),
549 }
550
551 // Every pre-existing variant still decodes unchanged.
552 let started: MailboxMessage =
553 serde_json::from_str(r#"{"kind":"started","agent_id":"a","agent_type":"worker"}"#)
554 .expect("started");
555 assert_eq!(started.agent_id(), "a");
556 }
557
558 #[tokio::test]
559 async fn agent_id_is_extractable_from_every_variant() {
560 let cases: Vec<(MailboxMessage, &str)> = vec![
561 (MailboxMessage::started("a1", FleetRole::Worker), "a1"),
562 (MailboxMessage::progress("a2", "x"), "a2"),
563 (
564 MailboxMessage::ToolCallStarted {
565 agent_id: "a3".into(),
566 tool_name: "read_file".into(),
567 step: 1,
568 },
569 "a3",
570 ),
571 (
572 MailboxMessage::ToolCallCompleted {
573 agent_id: "a4".into(),
574 tool_name: "read_file".into(),
575 step: 1,
576 ok: true,
577 },
578 "a4",
579 ),
580 (
581 MailboxMessage::ChildSpawned {
582 parent_id: "parent".into(),
583 child_id: "a5".into(),
584 },
585 "a5",
586 ),
587 (
588 MailboxMessage::Completed {
589 agent_id: "a6".into(),
590 summary: "done".into(),
591 },
592 "a6",
593 ),
594 (
595 MailboxMessage::Failed {
596 agent_id: "a7".into(),
597 error: "boom".into(),
598 },
599 "a7",
600 ),
601 (
602 MailboxMessage::Cancelled {
603 agent_id: "a8".into(),
604 },
605 "a8",
606 ),
607 (
608 MailboxMessage::Interrupted {
609 agent_id: "a10".into(),
610 reason: "API call timed out".into(),
611 },
612 "a10",
613 ),
614 (
615 MailboxMessage::work_state("a11", TodoListSnapshot::default()),
616 "a11",
617 ),
618 (
619 MailboxMessage::TokenUsage {
620 agent_id: "a9".into(),
621 source_id: "response-a9".into(),
622 route: Box::new(test_route(ApiProvider::Deepseek, "deepseek-v4-flash")),
623 usage: Usage {
624 input_tokens: 100,
625 output_tokens: 50,
626 ..Default::default()
627 },
628 },
629 "a9",
630 ),
631 ];
632 for (msg, expected) in cases {
633 assert_eq!(msg.agent_id(), expected, "extract failed for {msg:?}");
634 }
635 }
636
637 #[test]
638 fn token_usage_serde_round_trip_preserves_immutable_route_evidence() {
639 let route = crate::cost_status::EffectiveRouteEnvelope {
640 openrouter_vendor: None,
641 provider: ApiProvider::Moonshot,
642 provider_identity: "kimi-membership".to_string(),
643 model: "k3".to_string(),
644 billing_surface: Some(crate::pricing::MOONSHOT_KIMI_CODE_BILLING_SURFACE.to_string()),
645 endpoint_fingerprint: Some("a".repeat(64)),
646 provider_live_pricing: None,
647 billing_mode: crate::cost_status::RouteBillingMode::Subscription,
648 dispatched_at: chrono::DateTime::<chrono::Utc>::from_timestamp(1_234, 0)
649 .expect("timestamp"),
650 };
651 let message =
652 MailboxMessage::token_usage("agent-k3", "response-k3", route, Usage::default());
653 let json = serde_json::to_string(&message).expect("serialize token usage");
654 let restored: MailboxMessage =
655 serde_json::from_str(&json).expect("deserialize token usage");
656 assert_eq!(restored, message);
657 }
658 }
659
659 lines RUST