| 1 | //! Dedicated persistence actor for session save / checkpoint I/O. |
| 2 | //! |
| 3 | //! ## Motivation |
| 4 | //! |
| 5 | //! Before this module, `persist_checkpoint` and `persist_session_snapshot` ran |
| 6 | //! synchronously on the tokio worker thread that drives the TUI event loop. |
| 7 | //! Each call serialised all API messages to JSON, wrote a temp file, and |
| 8 | //! renamed it atomically — blocking keyboard input for the duration. |
| 9 | //! `save_session` additionally called `cleanup_old_sessions`, which listed all |
| 10 | //! session files, parsed metadata from every one, sorted, and deleted the |
| 11 | //! oldest — scaling O(session-bytes + file-count) with every turn. |
| 12 | //! |
| 13 | //! ## Design |
| 14 | //! |
| 15 | //! - **One dedicated tokio task** owns disk I/O. The UI only sends requests; |
| 16 | //! keystrokes never wait for writes. |
| 17 | //! - **Latest-wins coalescing per session**: when multiple `SaveCheckpoint`, |
| 18 | //! `SessionSnapshot`, or offline-queue requests pile up before the actor's |
| 19 | //! next write cycle, only the most recent one per session is written. |
| 20 | //! Checkpoints and clears are keyed by session id, so concurrent sessions |
| 21 | //! never coalesce into (or clear) each other's slot. |
| 22 | //! - **Durability reporting**: `FlushAndReport` returns accumulated results; |
| 23 | //! cycles without a listener log failures instead of discarding them. |
| 24 | //! - **Bounded command channel with sender-side coalescing** (#6212): |
| 25 | //! `try_send` absorbs each request into a shared latest-wins state at send |
| 26 | //! time and wakes the actor through a small bounded channel, so a paused |
| 27 | //! consumer retains one snapshot per session instead of one per send. |
| 28 | //! Queued snapshots retain only the canonical journal; legacy `messages` |
| 29 | //! are derived at the disk boundary instead of doubling every paused |
| 30 | //! request. |
| 31 | |
| 32 | use std::collections::{BTreeMap, BTreeSet}; |
| 33 | use std::sync::{Arc, OnceLock}; |
| 34 | |
| 35 | use tokio::sync::{mpsc, oneshot}; |
| 36 | |
| 37 | use crate::session_manager::{OfflineQueueLease, OfflineQueueState, SavedSession, SessionManager}; |
| 38 | use crate::utils::spawn_supervised; |
| 39 | |
| 40 | // --------------------------------------------------------------------------- |
| 41 | // Request type |
| 42 | // --------------------------------------------------------------------------- |
| 43 | |
| 44 | /// Persistence work item sent to the actor. |
| 45 | #[derive(Debug)] |
| 46 | pub enum PersistRequest { |
| 47 | /// Write a crash-recovery checkpoint (in-flight turn state) to the |
| 48 | /// session's own file (`checkpoints/<session_id>.json`). |
| 49 | SaveCheckpoint { session: SavedSession }, |
| 50 | /// Write a full session snapshot (completed turn, durable save). |
| 51 | SessionSnapshot(SavedSession), |
| 52 | /// Compound completion commit: write the completed session snapshot, |
| 53 | /// and only if that write succeeds, clear that same session's |
| 54 | /// crash-recovery checkpoint. A failed snapshot write RETAINS the |
| 55 | /// checkpoint as the only surviving recovery record, and the clear is |
| 56 | /// scoped to the committed session's id — it can never remove another |
| 57 | /// session's checkpoint. Turn completion must send this instead of a |
| 58 | /// `SessionSnapshot` + `ClearCheckpoint` pair, which the actor could |
| 59 | /// otherwise apply with the clear first, erasing the recovery record |
| 60 | /// before the snapshot safely landed. |
| 61 | CompletedCommit { session: SavedSession }, |
| 62 | /// Write queued/draft offline input for crash recovery. |
| 63 | OfflineQueue { |
| 64 | state: OfflineQueueState, |
| 65 | lease: Arc<OfflineQueueLease>, |
| 66 | }, |
| 67 | /// Remove the queued/draft offline input file. |
| 68 | ClearOfflineQueue { |
| 69 | /// Captures the exact owner and retains its exclusive editor lease |
| 70 | /// until the removal finishes. An unowned clear is unrepresentable. |
| 71 | lease: Arc<OfflineQueueLease>, |
| 72 | }, |
| 73 | /// Remove one session's crash-recovery checkpoint file. Scoped: cannot |
| 74 | /// remove another session's checkpoint. |
| 75 | ClearCheckpoint { session_id: String }, |
| 76 | /// Flush all pending work now and report durability results through |
| 77 | /// `reply`. The report aggregates every write/removal result since the |
| 78 | /// previous report (including background write cycles) — errors are |
| 79 | /// collected and surfaced, never discarded. |
| 80 | FlushAndReport { reply: oneshot::Sender<FlushReport> }, |
| 81 | /// Graceful shutdown — flush pending writes, then exit the actor loop. |
| 82 | Shutdown, |
| 83 | } |
| 84 | |
| 85 | /// Aggregated durability results: how many writes/removals completed and |
| 86 | /// which failed (labelled by what was being persisted, with the I/O error |
| 87 | /// kind). |
| 88 | #[derive(Debug, Default)] |
| 89 | pub struct FlushReport { |
| 90 | pub completed: usize, |
| 91 | pub failures: Vec<(String, std::io::ErrorKind)>, |
| 92 | } |
| 93 | |
| 94 | impl FlushReport { |
| 95 | /// Upper bound on retained failure entries when accumulating across |
| 96 | /// write cycles. Every failure is logged at the cycle it happened, so |
| 97 | /// dropping older-than-bound entries from the reply loses no evidence. |
| 98 | const MAX_ACCUMULATED_FAILURES: usize = 256; |
| 99 | |
| 100 | fn merge(&mut self, other: FlushReport) { |
| 101 | self.completed += other.completed; |
| 102 | self.failures.extend(other.failures); |
| 103 | if self.failures.len() > Self::MAX_ACCUMULATED_FAILURES { |
| 104 | let excess = self.failures.len() - Self::MAX_ACCUMULATED_FAILURES; |
| 105 | self.failures.drain(..excess); |
| 106 | } |
| 107 | } |
| 108 | } |
| 109 | |
| 110 | #[derive(Debug)] |
| 111 | enum PendingOfflineQueue { |
| 112 | Save { |
| 113 | state: Box<OfflineQueueState>, |
| 114 | lease: Arc<OfflineQueueLease>, |
| 115 | }, |
| 116 | Clear { |
| 117 | lease: Arc<OfflineQueueLease>, |
| 118 | }, |
| 119 | } |
| 120 | |
| 121 | // --------------------------------------------------------------------------- |
| 122 | // Handle (held by the TUI) |
| 123 | // --------------------------------------------------------------------------- |
| 124 | |
| 125 | /// Control commands the actor reacts to. Work itself never crosses the |
| 126 | /// channel: requests are coalesced into the shared [`PendingState`] at send |
| 127 | /// time, so a paused consumer retains at most the latest request per session |
| 128 | /// instead of every snapshot ever sent (#6212). |
| 129 | enum ActorCommand { |
| 130 | /// The shared pending state has work the actor has not taken yet. |
| 131 | WorkReady, |
| 132 | FlushAndReport { |
| 133 | reply: oneshot::Sender<FlushReport>, |
| 134 | }, |
| 135 | Shutdown, |
| 136 | } |
| 137 | |
| 138 | /// Command-channel capacity. `WorkReady` is deduplicated by the `notified` |
| 139 | /// flag, so only `FlushAndReport`/`Shutdown` can occupy slots unplanned; the |
| 140 | /// capacity exists so those never observe a full channel in practice. |
| 141 | const ACTOR_COMMAND_CAPACITY: usize = 8; |
| 142 | |
| 143 | /// The coalescing state shared between senders and the actor. Senders absorb |
| 144 | /// under the lock; the actor `take`s (swap to empty) under the same lock and |
| 145 | /// flushes outside it, so disk I/O never blocks a sender. |
| 146 | #[derive(Debug, Default)] |
| 147 | struct SharedPending { |
| 148 | pending: PendingState, |
| 149 | /// Whether a `WorkReady` is already queued (or being queued) and no |
| 150 | /// `take_pending` has observed it since. Reset only by `take_pending` — |
| 151 | /// the receiver side — so the flag always reflects the channel the actor |
| 152 | /// drains. |
| 153 | notified: bool, |
| 154 | } |
| 155 | |
| 156 | #[derive(Clone)] |
| 157 | struct PersistRequestSender { |
| 158 | shared: Arc<std::sync::Mutex<SharedPending>>, |
| 159 | cmd_tx: mpsc::Sender<ActorCommand>, |
| 160 | } |
| 161 | |
| 162 | impl std::fmt::Debug for PersistRequestSender { |
| 163 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { |
| 164 | f.debug_struct("PersistRequestSender") |
| 165 | .finish_non_exhaustive() |
| 166 | } |
| 167 | } |
| 168 | |
| 169 | struct PersistRequestReceiver { |
| 170 | shared: Arc<std::sync::Mutex<SharedPending>>, |
| 171 | cmd_rx: mpsc::Receiver<ActorCommand>, |
| 172 | } |
| 173 | |
| 174 | /// Single construction seam for the production persistence request channel. |
| 175 | /// |
| 176 | /// The ignored backlog measurement uses this same factory with the receiver |
| 177 | /// deliberately paused, so the measurement always characterizes whatever |
| 178 | /// representation the seam actually retains. |
| 179 | fn persistence_request_channel() -> (PersistRequestSender, PersistRequestReceiver) { |
| 180 | let (cmd_tx, cmd_rx) = mpsc::channel(ACTOR_COMMAND_CAPACITY); |
| 181 | let shared = Arc::new(std::sync::Mutex::new(SharedPending::default())); |
| 182 | ( |
| 183 | PersistRequestSender { |
| 184 | shared: Arc::clone(&shared), |
| 185 | cmd_tx, |
| 186 | }, |
| 187 | PersistRequestReceiver { shared, cmd_rx }, |
| 188 | ) |
| 189 | } |
| 190 | |
| 191 | impl PersistRequestReceiver { |
| 192 | /// Await the next actor command. `None` means every sender is gone. |
| 193 | async fn recv(&mut self) -> Option<ActorCommand> { |
| 194 | self.cmd_rx.recv().await |
| 195 | } |
| 196 | |
| 197 | /// Atomically take everything coalesced so far. Resets the `notified` |
| 198 | /// flag so the next absorbed request queues a fresh `WorkReady`. |
| 199 | fn take_pending(&mut self) -> PendingState { |
| 200 | let mut guard = self |
| 201 | .shared |
| 202 | .lock() |
| 203 | .unwrap_or_else(|poisoned| poisoned.into_inner()); |
| 204 | guard.notified = false; |
| 205 | std::mem::take(&mut guard.pending) |
| 206 | } |
| 207 | } |
| 208 | |
| 209 | /// Lightweight handle that the UI holds to queue persistence work. |
| 210 | #[derive(Debug, Clone)] |
| 211 | pub struct PersistActorHandle { |
| 212 | tx: PersistRequestSender, |
| 213 | } |
| 214 | |
| 215 | impl PersistActorHandle { |
| 216 | /// Queue a persistence request without blocking. The request is |
| 217 | /// coalesced into the shared pending state immediately (latest-wins per |
| 218 | /// session), so repeated snapshots of one session retain only the |
| 219 | /// newest. Returns `false` when the actor is already shut down. |
| 220 | pub fn try_send(&self, mut request: PersistRequest) -> bool { |
| 221 | match &mut request { |
| 222 | PersistRequest::SaveCheckpoint { session } |
| 223 | | PersistRequest::SessionSnapshot(session) |
| 224 | | PersistRequest::CompletedCommit { session } => { |
| 225 | session.compact_for_persistence_queue(); |
| 226 | } |
| 227 | _ => {} |
| 228 | } |
| 229 | let control = { |
| 230 | let mut guard = self |
| 231 | .tx |
| 232 | .shared |
| 233 | .lock() |
| 234 | .unwrap_or_else(|poisoned| poisoned.into_inner()); |
| 235 | guard.pending.absorb(request) |
| 236 | }; |
| 237 | match control { |
| 238 | Control::Continue => { |
| 239 | // A WorkReady the actor has not taken yet is already queued; |
| 240 | // that take will observe this request too. `notified` resets |
| 241 | // only when the actor takes, so it always mirrors the |
| 242 | // channel the actor drains. |
| 243 | { |
| 244 | let mut guard = self |
| 245 | .tx |
| 246 | .shared |
| 247 | .lock() |
| 248 | .unwrap_or_else(|poisoned| poisoned.into_inner()); |
| 249 | if guard.notified { |
| 250 | return true; |
| 251 | } |
| 252 | guard.notified = true; |
| 253 | } |
| 254 | if self.tx.cmd_tx.try_send(ActorCommand::WorkReady).is_ok() { |
| 255 | true |
| 256 | } else { |
| 257 | // Roll the flag back so later sends re-attempt (and |
| 258 | // re-fail) honestly instead of riding a dead wake. |
| 259 | let mut guard = self |
| 260 | .tx |
| 261 | .shared |
| 262 | .lock() |
| 263 | .unwrap_or_else(|poisoned| poisoned.into_inner()); |
| 264 | guard.notified = false; |
| 265 | false |
| 266 | } |
| 267 | } |
| 268 | Control::Flush(reply) => self |
| 269 | .tx |
| 270 | .cmd_tx |
| 271 | .try_send(ActorCommand::FlushAndReport { reply }) |
| 272 | .is_ok(), |
| 273 | Control::Shutdown => self.tx.cmd_tx.try_send(ActorCommand::Shutdown).is_ok(), |
| 274 | } |
| 275 | } |
| 276 | } |
| 277 | |
| 278 | // --------------------------------------------------------------------------- |
| 279 | // Global singleton (avoid threading through App) |
| 280 | // --------------------------------------------------------------------------- |
| 281 | |
| 282 | static ACTOR_TX: OnceLock<PersistActorHandle> = OnceLock::new(); |
| 283 | |
| 284 | /// Initialise the global persistence actor handle. Must be called once at |
| 285 | /// startup, before the event loop starts. |
| 286 | pub fn init_actor(handle: PersistActorHandle) { |
| 287 | let _ = ACTOR_TX.set(handle); |
| 288 | } |
| 289 | |
| 290 | /// Queue a persistence request through the global handle. When the request |
| 291 | /// cannot be queued — actor not initialised yet (tests, early startup) or |
| 292 | /// already shut down — the drop is logged instead of discarded silently, so |
| 293 | /// lost session/work-graph state is diagnosable after the fact. |
| 294 | pub fn persist(request: PersistRequest) { |
| 295 | let label = request_label(&request); |
| 296 | if try_persist(request) { |
| 297 | return; |
| 298 | } |
| 299 | if ACTOR_TX.get().is_some() { |
| 300 | tracing::warn!( |
| 301 | request = label, |
| 302 | "persistence request dropped: actor channel is closed (shutdown already happened)" |
| 303 | ); |
| 304 | } else { |
| 305 | tracing::debug!( |
| 306 | request = label, |
| 307 | "persistence request dropped: actor not initialised yet" |
| 308 | ); |
| 309 | } |
| 310 | } |
| 311 | |
| 312 | /// Order synchronous lifecycle saves after all previously queued snapshots. |
| 313 | /// Refuse a single-thread runtime rather than deadlocking its persistence task. |
| 314 | pub(crate) fn flush_before_transition() -> Result<(), String> { |
| 315 | let Some(handle) = ACTOR_TX.get() else { |
| 316 | return Ok(()); |
| 317 | }; |
| 318 | let runtime = tokio::runtime::Handle::try_current().ok(); |
| 319 | if runtime |
| 320 | .as_ref() |
| 321 | .is_some_and(|r| r.runtime_flavor() != tokio::runtime::RuntimeFlavor::MultiThread) |
| 322 | { |
| 323 | return Err("session transition requires an asynchronous persistence barrier".into()); |
| 324 | } |
| 325 | let (reply, receiver) = oneshot::channel(); |
| 326 | if !handle.try_send(PersistRequest::FlushAndReport { reply }) { |
| 327 | return Err("session transition could not queue persistence barrier".into()); |
| 328 | } |
| 329 | let receive = || receiver.blocking_recv(); |
| 330 | let report = if runtime.is_some() { |
| 331 | tokio::task::block_in_place(receive) |
| 332 | } else { |
| 333 | receive() |
| 334 | } |
| 335 | .map_err(|_| "session persistence stopped before the transition".to_string())?; |
| 336 | if !report.failures.is_empty() { |
| 337 | return Err(format!( |
| 338 | "session transition refused after persistence failures: {:?}", |
| 339 | report.failures |
| 340 | )); |
| 341 | } |
| 342 | Ok(()) |
| 343 | } |
| 344 | |
| 345 | fn request_label(request: &PersistRequest) -> &'static str { |
| 346 | match request { |
| 347 | PersistRequest::SaveCheckpoint { .. } => "SaveCheckpoint", |
| 348 | PersistRequest::SessionSnapshot(_) => "SessionSnapshot", |
| 349 | PersistRequest::CompletedCommit { .. } => "CompletedCommit", |
| 350 | PersistRequest::OfflineQueue { .. } => "OfflineQueue", |
| 351 | PersistRequest::ClearOfflineQueue { .. } => "ClearOfflineQueue", |
| 352 | PersistRequest::ClearCheckpoint { .. } => "ClearCheckpoint", |
| 353 | PersistRequest::FlushAndReport { .. } => "FlushAndReport", |
| 354 | PersistRequest::Shutdown => "Shutdown", |
| 355 | } |
| 356 | } |
| 357 | |
| 358 | /// Queue persistence and report whether the actor accepted ownership. Work |
| 359 | /// Graph projections use this acknowledgement as their publish boundary. |
| 360 | pub fn try_persist(request: PersistRequest) -> bool { |
| 361 | ACTOR_TX |
| 362 | .get() |
| 363 | .is_some_and(|handle| handle.try_send(request)) |
| 364 | } |
| 365 | |
| 366 | // --------------------------------------------------------------------------- |
| 367 | // Actor spawn |
| 368 | // --------------------------------------------------------------------------- |
| 369 | |
| 370 | /// Spawn the persistence actor task and return a handle for the caller to |
| 371 | /// store and initialise. |
| 372 | /// |
| 373 | /// The returned handle should be passed to [`init_actor`] so that the |
| 374 | /// `persist()` free function can reach it from anywhere in the TUI. |
| 375 | pub fn spawn_persistence_actor( |
| 376 | manager: SessionManager, |
| 377 | ) -> (PersistActorHandle, tokio::task::JoinHandle<()>) { |
| 378 | let (tx, mut rx) = persistence_request_channel(); |
| 379 | let handle = PersistActorHandle { tx }; |
| 380 | |
| 381 | let task = spawn_supervised( |
| 382 | "persistence-actor", |
| 383 | std::panic::Location::caller(), |
| 384 | async move { |
| 385 | let mut unreported = FlushReport::default(); |
| 386 | |
| 387 | // Flush pending work, log new failures, and fold the cycle's |
| 388 | // results into the unreported accumulator. |
| 389 | fn flush_cycle( |
| 390 | manager: &SessionManager, |
| 391 | pending: &mut PendingState, |
| 392 | unreported: &mut FlushReport, |
| 393 | ) { |
| 394 | let cycle = flush_inner(manager, pending); |
| 395 | log_flush_failures(&cycle); |
| 396 | unreported.merge(cycle); |
| 397 | } |
| 398 | |
| 399 | // Work is coalesced at send time into the shared pending state; |
| 400 | // every command handler takes whatever has accumulated and |
| 401 | // flushes it outside the sender lock. |
| 402 | while let Some(command) = rx.recv().await { |
| 403 | let mut pending = rx.take_pending(); |
| 404 | match command { |
| 405 | ActorCommand::WorkReady => { |
| 406 | if !pending.is_empty() { |
| 407 | flush_cycle(&manager, &mut pending, &mut unreported); |
| 408 | } |
| 409 | } |
| 410 | ActorCommand::FlushAndReport { reply } => { |
| 411 | flush_cycle(&manager, &mut pending, &mut unreported); |
| 412 | let _ = reply.send(std::mem::take(&mut unreported)); |
| 413 | } |
| 414 | ActorCommand::Shutdown => { |
| 415 | flush_cycle(&manager, &mut pending, &mut unreported); |
| 416 | return; |
| 417 | } |
| 418 | } |
| 419 | } |
| 420 | // Every sender is gone — final flush and exit. Each absorb |
| 421 | // guarantees a queued WorkReady, so this is normally empty. |
| 422 | let mut pending = rx.take_pending(); |
| 423 | flush_cycle(&manager, &mut pending, &mut unreported); |
| 424 | }, |
| 425 | ); |
| 426 | |
| 427 | (handle, task) |
| 428 | } |
| 429 | |
| 430 | /// Coalesced work waiting for the next write cycle. |
| 431 | #[derive(Debug, Default)] |
| 432 | struct PendingState { |
| 433 | /// Latest-wins per session id. Crash checkpoints are keyed per session |
| 434 | /// (mirroring `sessions` below) so concurrent sessions can interleave |
| 435 | /// saves and clears without clobbering each other. |
| 436 | checkpoints: BTreeMap<String, SavedSession>, |
| 437 | /// Session ids whose checkpoint file should be removed. |
| 438 | checkpoint_clears: BTreeSet<String>, |
| 439 | /// Latest-wins per session id. Coalescing into one global slot can |
| 440 | /// drop session A when an immediate `/new` queues session B before |
| 441 | /// the actor drains. |
| 442 | sessions: BTreeMap<String, SavedSession>, |
| 443 | /// Compound completion commits, latest-wins per session id: the |
| 444 | /// completed snapshot body to write, followed by that session's own |
| 445 | /// checkpoint clear — the clear only if the write succeeded. Kept |
| 446 | /// separate from `sessions` so a plain snapshot can never be drained |
| 447 | /// as a completion (or vice versa) and the clear intent stays bound to |
| 448 | /// exactly the session that completed. |
| 449 | completed_commits: BTreeMap<String, SavedSession>, |
| 450 | /// Latest-wins per session id, for the same reason `sessions` above is: |
| 451 | /// a single global slot dropped session A's queued text when session B |
| 452 | /// queued before the actor drained, which defeats the per-session file |
| 453 | /// naming entirely. Each pending request retains its editor lease, so a |
| 454 | /// window changing session cannot release ownership ahead of its writes. |
| 455 | offline_queue: BTreeMap<String, PendingOfflineQueue>, |
| 456 | } |
| 457 | |
| 458 | /// What the actor loop should do after absorbing a request. |
| 459 | enum Control { |
| 460 | Continue, |
| 461 | Flush(oneshot::Sender<FlushReport>), |
| 462 | Shutdown, |
| 463 | } |
| 464 | |
| 465 | impl PendingState { |
| 466 | /// True when nothing coalesced is waiting. An empty `WorkReady` take is |
| 467 | /// skipped instead of running a no-op flush cycle. |
| 468 | fn is_empty(&self) -> bool { |
| 469 | self.checkpoints.is_empty() |
| 470 | && self.checkpoint_clears.is_empty() |
| 471 | && self.sessions.is_empty() |
| 472 | && self.completed_commits.is_empty() |
| 473 | && self.offline_queue.is_empty() |
| 474 | } |
| 475 | |
| 476 | fn absorb(&mut self, req: PersistRequest) -> Control { |
| 477 | match req { |
| 478 | PersistRequest::SaveCheckpoint { session } => { |
| 479 | // Last-writer-wins per session: a fresh checkpoint supersedes |
| 480 | // a pending clear for the same session so the two never both |
| 481 | // apply in one drain (which previously cleared then re-wrote |
| 482 | // the stale checkpoint, undoing the clear). |
| 483 | let id = session.metadata.id.clone(); |
| 484 | self.checkpoint_clears.remove(&id); |
| 485 | // A new in-flight checkpoint means newer turn work started |
| 486 | // after the completion it would have cleared: the compound's |
| 487 | // clear-on-success intent is stale now (it would erase the |
| 488 | // fresher recovery record), so drop the pending compound and |
| 489 | // keep only the newer checkpoint body. |
| 490 | if self.completed_commits.remove(&id).is_some() { |
| 491 | tracing::debug!( |
| 492 | session_id = %id, |
| 493 | "pending completed commit superseded by a newer in-flight checkpoint" |
| 494 | ); |
| 495 | } |
| 496 | self.checkpoints.insert(id, session); |
| 497 | } |
| 498 | PersistRequest::SessionSnapshot(session) => { |
| 499 | // A newer full snapshot of a session with a pending |
| 500 | // completed commit refreshes the commit's body (and keeps |
| 501 | // its clear-on-success intent): the in-flight checkpoint it |
| 502 | // guards is already captured by the newer completed state. |
| 503 | let id = session.metadata.id.clone(); |
| 504 | if let Some(pending) = self.completed_commits.get_mut(&id) { |
| 505 | *pending = session; |
| 506 | } else { |
| 507 | self.sessions.insert(id, session); |
| 508 | } |
| 509 | } |
| 510 | PersistRequest::CompletedCommit { session } => { |
| 511 | let id = session.metadata.id.clone(); |
| 512 | // The compound owns this session's snapshot and clear: a |
| 513 | // pending plain snapshot is superseded, a pending standalone |
| 514 | // clear would erase the recovery record even when the save |
| 515 | // fails, and a pending checkpoint write would re-create the |
| 516 | // record after the compound cleared it. |
| 517 | self.sessions.remove(&id); |
| 518 | self.checkpoint_clears.remove(&id); |
| 519 | self.checkpoints.remove(&id); |
| 520 | self.completed_commits.insert(id, session); |
| 521 | } |
| 522 | PersistRequest::OfflineQueue { state, lease } => { |
| 523 | self.offline_queue.insert( |
| 524 | lease.session_id().to_string(), |
| 525 | PendingOfflineQueue::Save { |
| 526 | state: Box::new(state), |
| 527 | lease, |
| 528 | }, |
| 529 | ); |
| 530 | } |
| 531 | PersistRequest::ClearOfflineQueue { lease } => { |
| 532 | // A clear supersedes a pending save for its OWN session only. |
| 533 | self.offline_queue.insert( |
| 534 | lease.session_id().to_string(), |
| 535 | PendingOfflineQueue::Clear { lease }, |
| 536 | ); |
| 537 | } |
| 538 | PersistRequest::ClearCheckpoint { session_id } => { |
| 539 | // A clear supersedes a pending checkpoint write for the same |
| 540 | // session only — other sessions' pending work is untouched. |
| 541 | // An explicit clear is also a user-owned boundary (e.g. |
| 542 | // `/new`): it supersedes a pending compound for the same |
| 543 | // session so the discarded session is not re-written. |
| 544 | self.checkpoints.remove(&session_id); |
| 545 | self.completed_commits.remove(&session_id); |
| 546 | self.checkpoint_clears.insert(session_id); |
| 547 | } |
| 548 | PersistRequest::FlushAndReport { reply } => return Control::Flush(reply), |
| 549 | PersistRequest::Shutdown => return Control::Shutdown, |
| 550 | } |
| 551 | Control::Continue |
| 552 | } |
| 553 | } |
| 554 | |
| 555 | /// Write all pending work to disk, draining `pending`. Every write and |
| 556 | /// removal result is collected into the returned [`FlushReport`] — failures |
| 557 | /// are reported, never silently discarded. |
| 558 | /// |
| 559 | /// Ordering is durability-critical: every session snapshot write (plain or |
| 560 | /// completion commit) happens BEFORE any checkpoint clear. A completion |
| 561 | /// commit clears its session's checkpoint only after that session's own |
| 562 | /// write succeeded, so a failed save always leaves the crash-recovery |
| 563 | /// checkpoint in place. |
| 564 | fn flush_inner(manager: &SessionManager, pending: &mut PendingState) -> FlushReport { |
| 565 | let mut report = FlushReport::default(); |
| 566 | let mut record = |what: String, result: std::io::Result<()>| match result { |
| 567 | Ok(()) => report.completed += 1, |
| 568 | Err(err) => report.failures.push((what, err.kind())), |
| 569 | }; |
| 570 | |
| 571 | for (session_id, session) in std::mem::take(&mut pending.sessions) { |
| 572 | record( |
| 573 | format!("session:{session_id}"), |
| 574 | manager.save_session_owned(session).map(|_| ()), |
| 575 | ); |
| 576 | } |
| 577 | for (session_id, session) in std::mem::take(&mut pending.completed_commits) { |
| 578 | let commit_result = manager.save_session_owned(session); |
| 579 | let save_succeeded = commit_result.is_ok(); |
| 580 | record( |
| 581 | format!("completed-commit:{session_id}"), |
| 582 | commit_result.map(|_| ()), |
| 583 | ); |
| 584 | if save_succeeded { |
| 585 | // Only the committed session's own checkpoint is cleared, and |
| 586 | // only because its snapshot safely landed. A failure above |
| 587 | // retains the checkpoint as the sole recovery record. |
| 588 | record( |
| 589 | format!("clear-checkpoint:{session_id}"), |
| 590 | manager.clear_session_checkpoint(&session_id), |
| 591 | ); |
| 592 | } |
| 593 | } |
| 594 | for session_id in std::mem::take(&mut pending.checkpoint_clears) { |
| 595 | record( |
| 596 | format!("clear-checkpoint:{session_id}"), |
| 597 | manager.clear_session_checkpoint(&session_id), |
| 598 | ); |
| 599 | } |
| 600 | for (session_id, session) in std::mem::take(&mut pending.checkpoints) { |
| 601 | record( |
| 602 | format!("checkpoint:{session_id}"), |
| 603 | manager.save_checkpoint_owned(session).map(|_| ()), |
| 604 | ); |
| 605 | } |
| 606 | for (_, request) in std::mem::take(&mut pending.offline_queue) { |
| 607 | match request { |
| 608 | PendingOfflineQueue::Save { state, lease } => record( |
| 609 | "offline-queue".to_string(), |
| 610 | manager |
| 611 | .save_offline_queue_state(&state, Some(lease.session_id())) |
| 612 | .map(|_| ()), |
| 613 | ), |
| 614 | PendingOfflineQueue::Clear { lease } => record( |
| 615 | "clear-offline-queue".to_string(), |
| 616 | manager.clear_offline_queue_state_for(lease.session_id()), |
| 617 | ), |
| 618 | } |
| 619 | } |
| 620 | report |
| 621 | } |
| 622 | |
| 623 | /// Surface flush failures in the log for write cycles that have no caller |
| 624 | /// waiting on a [`FlushReport`]. |
| 625 | fn log_flush_failures(report: &FlushReport) { |
| 626 | for (what, kind) in &report.failures { |
| 627 | tracing::warn!( |
| 628 | target: "persistence", |
| 629 | what = %what, |
| 630 | error_kind = ?kind, |
| 631 | "persistence write failed", |
| 632 | ); |
| 633 | } |
| 634 | } |
| 635 | |
| 636 | #[cfg(test)] |
| 637 | #[path = "persistence_actor/tests.rs"] |
| 638 | mod backlog_measurement_tests; |
| 639 | |
| 640 | #[cfg(test)] |
| 641 | mod tests { |
| 642 | use super::*; |
| 643 | use std::time::Duration; |
| 644 | |
| 645 | use crate::session_manager::{OfflineQueueState, QueuedSessionMessage}; |
| 646 | |
| 647 | async fn wait_until(mut predicate: impl FnMut() -> bool) { |
| 648 | let deadline = tokio::time::Instant::now() + Duration::from_secs(2); |
| 649 | loop { |
| 650 | if predicate() { |
| 651 | return; |
| 652 | } |
| 653 | assert!( |
| 654 | tokio::time::Instant::now() < deadline, |
| 655 | "timed out waiting for persistence actor" |
| 656 | ); |
| 657 | tokio::time::sleep(Duration::from_millis(10)).await; |
| 658 | } |
| 659 | } |
| 660 | |
| 661 | #[tokio::test] |
| 662 | async fn two_sessions_queueing_before_a_drain_both_survive() { |
| 663 | // Per-session FILENAMES are not enough on their own: the actor |
| 664 | // coalesces pending work before those names are ever used, and the |
| 665 | // queue used one global slot while its checkpoint/session neighbours |
| 666 | // were already keyed per session. Session A's unsent text was |
| 667 | // therefore dropped whenever session B queued first. |
| 668 | let tmp = tempfile::tempdir().expect("tempdir"); |
| 669 | let sessions_dir = tmp.path().join("sessions"); |
| 670 | let manager = SessionManager::new(sessions_dir.clone()).expect("manager"); |
| 671 | let (handle, task) = spawn_persistence_actor(manager); |
| 672 | |
| 673 | let queue_manager = SessionManager::new(sessions_dir.clone()).expect("queue manager"); |
| 674 | let lease_a = queue_manager |
| 675 | .acquire_offline_queue_lease("session-A") |
| 676 | .expect("lease A"); |
| 677 | let lease_b = queue_manager |
| 678 | .acquire_offline_queue_lease("session-B") |
| 679 | .expect("lease B"); |
| 680 | for (session, body) in [("session-A", "text from A"), ("session-B", "text from B")] { |
| 681 | let state = OfflineQueueState { |
| 682 | messages: vec![QueuedSessionMessage { |
| 683 | display: body.to_string(), |
| 684 | skill_instruction: None, |
| 685 | skill_provenance: None, |
| 686 | }], |
| 687 | ..OfflineQueueState::default() |
| 688 | }; |
| 689 | handle.try_send(PersistRequest::OfflineQueue { |
| 690 | state, |
| 691 | lease: Arc::clone(if session == "session-A" { |
| 692 | &lease_a |
| 693 | } else { |
| 694 | &lease_b |
| 695 | }), |
| 696 | }); |
| 697 | } |
| 698 | |
| 699 | let checkpoints = sessions_dir.join("checkpoints"); |
| 700 | for (session, body) in [("session-A", "text from A"), ("session-B", "text from B")] { |
| 701 | let path = checkpoints.join(format!("{session}.offline_queue.json")); |
| 702 | // wait_until panics on timeout, which is the failure signal: a |
| 703 | // coalesced-away queue never appears. |
| 704 | wait_until(|| std::fs::read_to_string(&path).is_ok_and(|f| f.contains(body))).await; |
| 705 | } |
| 706 | |
| 707 | // A clear names its own session and must not touch the other's. |
| 708 | handle.try_send(PersistRequest::ClearOfflineQueue { |
| 709 | lease: Arc::clone(&lease_a), |
| 710 | }); |
| 711 | let a = checkpoints.join("session-A.offline_queue.json"); |
| 712 | wait_until(|| !a.exists()).await; |
| 713 | assert!( |
| 714 | checkpoints.join("session-B.offline_queue.json").exists(), |
| 715 | "clearing one session must not delete another session's queued text" |
| 716 | ); |
| 717 | |
| 718 | drop(handle); |
| 719 | let _ = task.await; |
| 720 | } |
| 721 | |
| 722 | #[tokio::test] |
| 723 | async fn actor_persists_and_clears_offline_queue_requests() { |
| 724 | let tmp = tempfile::tempdir().expect("tempdir"); |
| 725 | let sessions_dir = tmp.path().join("sessions"); |
| 726 | let manager = SessionManager::new(sessions_dir.clone()).expect("manager"); |
| 727 | // The queue is keyed per session now (#5715-adjacent data-loss fix): |
| 728 | // two concurrent instances used to share one global file and the loser |
| 729 | // lost its unsent text. The request below carries session-A. |
| 730 | let queue_path = sessions_dir |
| 731 | .join("checkpoints") |
| 732 | .join("session-A.offline_queue.json"); |
| 733 | let lease = manager |
| 734 | .acquire_offline_queue_lease("session-A") |
| 735 | .expect("queue lease"); |
| 736 | let (handle, task) = spawn_persistence_actor(manager); |
| 737 | |
| 738 | let state = OfflineQueueState { |
| 739 | messages: vec![QueuedSessionMessage { |
| 740 | display: "queued from enter".to_string(), |
| 741 | skill_instruction: None, |
| 742 | skill_provenance: None, |
| 743 | }], |
| 744 | ..OfflineQueueState::default() |
| 745 | }; |
| 746 | |
| 747 | handle.try_send(PersistRequest::OfflineQueue { |
| 748 | state, |
| 749 | lease: Arc::clone(&lease), |
| 750 | }); |
| 751 | wait_until(|| { |
| 752 | std::fs::read_to_string(&queue_path) |
| 753 | .is_ok_and(|body| body.contains("queued from enter")) |
| 754 | }) |
| 755 | .await; |
| 756 | |
| 757 | handle.try_send(PersistRequest::ClearOfflineQueue { |
| 758 | lease: Arc::clone(&lease), |
| 759 | }); |
| 760 | wait_until(|| !queue_path.exists()).await; |
| 761 | handle.try_send(PersistRequest::Shutdown); |
| 762 | task.await.expect("persistence actor join"); |
| 763 | } |
| 764 | |
| 765 | #[tokio::test] |
| 766 | async fn shutdown_wait_flushes_queued_session_before_returning() { |
| 767 | let tmp = tempfile::tempdir().expect("tempdir"); |
| 768 | let sessions_dir = tmp.path().join("sessions"); |
| 769 | let manager = SessionManager::new(sessions_dir.clone()).expect("manager"); |
| 770 | let verification_manager = SessionManager::new(sessions_dir).expect("verification manager"); |
| 771 | let session = crate::session_manager::create_saved_session_with_mode( |
| 772 | &[], |
| 773 | "deepseek-v4-pro", |
| 774 | tmp.path(), |
| 775 | 0, |
| 776 | None, |
| 777 | Some("agent"), |
| 778 | ); |
| 779 | let session_id = session.metadata.id.clone(); |
| 780 | let (handle, task) = spawn_persistence_actor(manager); |
| 781 | |
| 782 | handle.try_send(PersistRequest::SessionSnapshot(session)); |
| 783 | handle.try_send(PersistRequest::Shutdown); |
| 784 | task.await.expect("persistence actor join"); |
| 785 | |
| 786 | let loaded = verification_manager |
| 787 | .load_session(&session_id) |
| 788 | .expect("shutdown must flush queued session"); |
| 789 | assert_eq!(loaded.metadata.id, session_id); |
| 790 | } |
| 791 | |
| 792 | #[tokio::test] |
| 793 | async fn shutdown_flushes_latest_snapshot_for_each_session_id() { |
| 794 | let tmp = tempfile::tempdir().expect("tempdir"); |
| 795 | let sessions_dir = tmp.path().join("sessions"); |
| 796 | let manager = SessionManager::new(sessions_dir.clone()).expect("manager"); |
| 797 | let verification_manager = SessionManager::new(sessions_dir).expect("verification manager"); |
| 798 | let mut first = crate::session_manager::create_saved_session_with_mode( |
| 799 | &[], |
| 800 | "deepseek-v4-pro", |
| 801 | tmp.path(), |
| 802 | 0, |
| 803 | None, |
| 804 | Some("agent"), |
| 805 | ); |
| 806 | first.metadata.title = "Session A".to_string(); |
| 807 | let mut second = crate::session_manager::create_saved_session_with_mode( |
| 808 | &[], |
| 809 | "deepseek-v4-pro", |
| 810 | tmp.path(), |
| 811 | 0, |
| 812 | None, |
| 813 | Some("agent"), |
| 814 | ); |
| 815 | second.metadata.title = "Session B".to_string(); |
| 816 | let first_id = first.metadata.id.clone(); |
| 817 | let second_id = second.metadata.id.clone(); |
| 818 | let (handle, task) = spawn_persistence_actor(manager); |
| 819 | |
| 820 | handle.try_send(PersistRequest::SessionSnapshot(first)); |
| 821 | handle.try_send(PersistRequest::SessionSnapshot(second)); |
| 822 | handle.try_send(PersistRequest::Shutdown); |
| 823 | task.await.expect("persistence actor join"); |
| 824 | |
| 825 | assert_eq!( |
| 826 | verification_manager |
| 827 | .load_session(&first_id) |
| 828 | .expect("session A flushed") |
| 829 | .metadata |
| 830 | .title, |
| 831 | "Session A" |
| 832 | ); |
| 833 | assert_eq!( |
| 834 | verification_manager |
| 835 | .load_session(&second_id) |
| 836 | .expect("session B flushed") |
| 837 | .metadata |
| 838 | .title, |
| 839 | "Session B" |
| 840 | ); |
| 841 | } |
| 842 | |
| 843 | #[tokio::test] |
| 844 | async fn interleaved_checkpoint_saves_and_clears_stay_per_session() { |
| 845 | let tmp = tempfile::tempdir().expect("tempdir"); |
| 846 | let sessions_dir = tmp.path().join("sessions"); |
| 847 | let manager = SessionManager::new(sessions_dir.clone()).expect("manager"); |
| 848 | let verification_manager = SessionManager::new(sessions_dir).expect("verification manager"); |
| 849 | let first = crate::session_manager::create_saved_session_with_mode( |
| 850 | &[], |
| 851 | "deepseek-v4-pro", |
| 852 | tmp.path(), |
| 853 | 0, |
| 854 | None, |
| 855 | Some("agent"), |
| 856 | ); |
| 857 | let second = crate::session_manager::create_saved_session_with_mode( |
| 858 | &[], |
| 859 | "deepseek-v4-pro", |
| 860 | tmp.path(), |
| 861 | 0, |
| 862 | None, |
| 863 | Some("agent"), |
| 864 | ); |
| 865 | let first_id = first.metadata.id.clone(); |
| 866 | let second_id = second.metadata.id.clone(); |
| 867 | let (handle, task) = spawn_persistence_actor(manager); |
| 868 | |
| 869 | // Interleave: save A, save B, clear A — all coalesced into one drain. |
| 870 | handle.try_send(PersistRequest::SaveCheckpoint { session: first }); |
| 871 | handle.try_send(PersistRequest::SaveCheckpoint { session: second }); |
| 872 | handle.try_send(PersistRequest::ClearCheckpoint { |
| 873 | session_id: first_id.clone(), |
| 874 | }); |
| 875 | handle.try_send(PersistRequest::Shutdown); |
| 876 | task.await.expect("persistence actor join"); |
| 877 | |
| 878 | assert!( |
| 879 | verification_manager |
| 880 | .load_session_checkpoint(&first_id) |
| 881 | .expect("load first checkpoint") |
| 882 | .is_none(), |
| 883 | "cleared session must have no checkpoint file" |
| 884 | ); |
| 885 | let survivor = verification_manager |
| 886 | .load_session_checkpoint(&second_id) |
| 887 | .expect("load second checkpoint") |
| 888 | .expect("second session's checkpoint must survive an unrelated clear"); |
| 889 | assert_eq!(survivor.metadata.id, second_id); |
| 890 | } |
| 891 | |
| 892 | #[tokio::test] |
| 893 | async fn flush_and_report_returns_completed_counts() { |
| 894 | let tmp = tempfile::tempdir().expect("tempdir"); |
| 895 | let sessions_dir = tmp.path().join("sessions"); |
| 896 | let manager = SessionManager::new(sessions_dir).expect("manager"); |
| 897 | let session = crate::session_manager::create_saved_session_with_mode( |
| 898 | &[], |
| 899 | "deepseek-v4-pro", |
| 900 | tmp.path(), |
| 901 | 0, |
| 902 | None, |
| 903 | Some("agent"), |
| 904 | ); |
| 905 | let (handle, task) = spawn_persistence_actor(manager); |
| 906 | |
| 907 | handle.try_send(PersistRequest::SaveCheckpoint { session }); |
| 908 | let (reply_tx, reply_rx) = tokio::sync::oneshot::channel(); |
| 909 | handle.try_send(PersistRequest::FlushAndReport { reply: reply_tx }); |
| 910 | let report = reply_rx.await.expect("flush report reply"); |
| 911 | // Whether the checkpoint was written by an earlier background cycle |
| 912 | // or by this flush, the accumulated report must count it and show no |
| 913 | // failures — and the actor keeps running afterwards. |
| 914 | assert!(report.completed >= 1, "checkpoint write must be counted"); |
| 915 | assert!(report.failures.is_empty(), "no failures expected"); |
| 916 | handle.try_send(PersistRequest::Shutdown); |
| 917 | task.await.expect("persistence actor join"); |
| 918 | } |
| 919 | |
| 920 | #[tokio::test] |
| 921 | async fn flush_and_report_propagates_write_failures() { |
| 922 | let tmp = tempfile::tempdir().expect("tempdir"); |
| 923 | let sessions_dir = tmp.path().join("sessions"); |
| 924 | let manager = SessionManager::new(sessions_dir.clone()).expect("manager"); |
| 925 | // Occupy the checkpoints directory path with a regular file so every |
| 926 | // checkpoint write deterministically fails on all platforms. |
| 927 | std::fs::write(sessions_dir.join("checkpoints"), b"not a directory") |
| 928 | .expect("block checkpoints dir"); |
| 929 | let session = crate::session_manager::create_saved_session_with_mode( |
| 930 | &[], |
| 931 | "deepseek-v4-pro", |
| 932 | tmp.path(), |
| 933 | 0, |
| 934 | None, |
| 935 | Some("agent"), |
| 936 | ); |
| 937 | let session_id = session.metadata.id.clone(); |
| 938 | let (handle, task) = spawn_persistence_actor(manager); |
| 939 | |
| 940 | handle.try_send(PersistRequest::SaveCheckpoint { session }); |
| 941 | let (reply_tx, reply_rx) = tokio::sync::oneshot::channel(); |
| 942 | handle.try_send(PersistRequest::FlushAndReport { reply: reply_tx }); |
| 943 | let report = reply_rx.await.expect("flush report reply"); |
| 944 | |
| 945 | assert!( |
| 946 | report |
| 947 | .failures |
| 948 | .iter() |
| 949 | .any(|(what, _)| what == &format!("checkpoint:{session_id}")), |
| 950 | "failed checkpoint write must be reported, got: {:?}", |
| 951 | report.failures |
| 952 | ); |
| 953 | handle.try_send(PersistRequest::Shutdown); |
| 954 | task.await.expect("persistence actor join"); |
| 955 | } |
| 956 | |
| 957 | /// Pre-write a crash-recovery checkpoint file for `session_id` directly, |
| 958 | /// so completion-commit tests can assert on its survival without needing |
| 959 | /// a prior in-flight turn. |
| 960 | fn seed_checkpoint_file( |
| 961 | sessions_dir: &std::path::Path, |
| 962 | session_id: &str, |
| 963 | ) -> std::path::PathBuf { |
| 964 | let path = sessions_dir |
| 965 | .join("checkpoints") |
| 966 | .join(format!("{session_id}.json")); |
| 967 | std::fs::create_dir_all(path.parent().expect("checkpoints parent")).expect("mkdir"); |
| 968 | std::fs::write(&path, "{}").expect("seed checkpoint file"); |
| 969 | path |
| 970 | } |
| 971 | |
| 972 | /// Deterministically fail session-file saves only: a directory at the |
| 973 | /// session's own `<id>.json` path makes `save_session` fail while the |
| 974 | /// checkpoints directory stays fully usable. |
| 975 | fn block_session_file(sessions_dir: &std::path::Path, session_id: &str) { |
| 976 | std::fs::create_dir_all(sessions_dir.join(format!("{session_id}.json"))) |
| 977 | .expect("block session file path"); |
| 978 | } |
| 979 | |
| 980 | #[tokio::test] |
| 981 | async fn completed_commit_preserves_checkpoint_when_session_save_fails() { |
| 982 | let tmp = tempfile::tempdir().expect("tempdir"); |
| 983 | let sessions_dir = tmp.path().join("sessions"); |
| 984 | let manager = SessionManager::new(sessions_dir.clone()).expect("manager"); |
| 985 | let session = crate::session_manager::create_saved_session_with_mode( |
| 986 | &[], |
| 987 | "deepseek-v4-pro", |
| 988 | tmp.path(), |
| 989 | 0, |
| 990 | None, |
| 991 | Some("agent"), |
| 992 | ); |
| 993 | let session_id = session.metadata.id.clone(); |
| 994 | let checkpoint_path = seed_checkpoint_file(&sessions_dir, &session_id); |
| 995 | block_session_file(&sessions_dir, &session_id); |
| 996 | let (handle, task) = spawn_persistence_actor(manager); |
| 997 | |
| 998 | handle.try_send(PersistRequest::CompletedCommit { session }); |
| 999 | let (reply_tx, reply_rx) = tokio::sync::oneshot::channel(); |
| 1000 | handle.try_send(PersistRequest::FlushAndReport { reply: reply_tx }); |
| 1001 | let report = reply_rx.await.expect("flush report reply"); |
| 1002 | |
| 1003 | assert!( |
| 1004 | report |
| 1005 | .failures |
| 1006 | .iter() |
| 1007 | .any(|(what, _)| what == &format!("completed-commit:{session_id}")), |
| 1008 | "failed session save must be reported, got: {:?}", |
| 1009 | report.failures |
| 1010 | ); |
| 1011 | assert!( |
| 1012 | !report |
| 1013 | .failures |
| 1014 | .iter() |
| 1015 | .any(|(what, _)| what == &format!("clear-checkpoint:{session_id}")), |
| 1016 | "the checkpoint clear must not be attempted after a failed save" |
| 1017 | ); |
| 1018 | assert!( |
| 1019 | checkpoint_path.exists(), |
| 1020 | "a failed session save must retain the crash-recovery checkpoint" |
| 1021 | ); |
| 1022 | handle.try_send(PersistRequest::Shutdown); |
| 1023 | task.await.expect("persistence actor join"); |
| 1024 | } |
| 1025 | |
| 1026 | #[tokio::test] |
| 1027 | async fn completed_commit_clears_only_after_session_save() { |
| 1028 | let tmp = tempfile::tempdir().expect("tempdir"); |
| 1029 | let sessions_dir = tmp.path().join("sessions"); |
| 1030 | let manager = SessionManager::new(sessions_dir.clone()).expect("manager"); |
| 1031 | let verification_manager = SessionManager::new(sessions_dir.clone()).expect("verify"); |
| 1032 | let session = crate::session_manager::create_saved_session_with_mode( |
| 1033 | &[], |
| 1034 | "deepseek-v4-pro", |
| 1035 | tmp.path(), |
| 1036 | 0, |
| 1037 | None, |
| 1038 | Some("agent"), |
| 1039 | ); |
| 1040 | let session_id = session.metadata.id.clone(); |
| 1041 | let checkpoint_path = seed_checkpoint_file(&sessions_dir, &session_id); |
| 1042 | let (handle, task) = spawn_persistence_actor(manager); |
| 1043 | |
| 1044 | handle.try_send(PersistRequest::CompletedCommit { session }); |
| 1045 | let (reply_tx, reply_rx) = tokio::sync::oneshot::channel(); |
| 1046 | handle.try_send(PersistRequest::FlushAndReport { reply: reply_tx }); |
| 1047 | let report = reply_rx.await.expect("flush report reply"); |
| 1048 | |
| 1049 | assert!( |
| 1050 | report.failures.is_empty(), |
| 1051 | "commit save and its clear must both succeed, got: {:?}", |
| 1052 | report.failures |
| 1053 | ); |
| 1054 | let saved = verification_manager |
| 1055 | .load_session(&session_id) |
| 1056 | .expect("completed session must be saved before its checkpoint is cleared"); |
| 1057 | assert_eq!(saved.metadata.id, session_id); |
| 1058 | assert!( |
| 1059 | !checkpoint_path.exists(), |
| 1060 | "the checkpoint is cleared only after the session save succeeded" |
| 1061 | ); |
| 1062 | handle.try_send(PersistRequest::Shutdown); |
| 1063 | task.await.expect("persistence actor join"); |
| 1064 | } |
| 1065 | |
| 1066 | #[tokio::test] |
| 1067 | async fn completed_commit_never_clears_another_sessions_checkpoint() { |
| 1068 | let tmp = tempfile::tempdir().expect("tempdir"); |
| 1069 | let sessions_dir = tmp.path().join("sessions"); |
| 1070 | let manager = SessionManager::new(sessions_dir.clone()).expect("manager"); |
| 1071 | let verification_manager = SessionManager::new(sessions_dir.clone()).expect("verify"); |
| 1072 | let committed = crate::session_manager::create_saved_session_with_mode( |
| 1073 | &[], |
| 1074 | "deepseek-v4-pro", |
| 1075 | tmp.path(), |
| 1076 | 0, |
| 1077 | None, |
| 1078 | Some("agent"), |
| 1079 | ); |
| 1080 | let inflight = crate::session_manager::create_saved_session_with_mode( |
| 1081 | &[], |
| 1082 | "deepseek-v4-pro", |
| 1083 | tmp.path(), |
| 1084 | 0, |
| 1085 | None, |
| 1086 | Some("agent"), |
| 1087 | ); |
| 1088 | let committed_id = committed.metadata.id.clone(); |
| 1089 | let inflight_id = inflight.metadata.id.clone(); |
| 1090 | let committed_checkpoint = seed_checkpoint_file(&sessions_dir, &committed_id); |
| 1091 | // The concurrent session's checkpoint must carry a loadable body, so |
| 1092 | // seed it through the real manager instead of a placeholder file. |
| 1093 | let inflight_checkpoint = verification_manager |
| 1094 | .save_checkpoint(&inflight) |
| 1095 | .expect("seed the concurrent session's checkpoint"); |
| 1096 | let (handle, task) = spawn_persistence_actor(manager); |
| 1097 | |
| 1098 | handle.try_send(PersistRequest::CompletedCommit { session: committed }); |
| 1099 | let (reply_tx, reply_rx) = tokio::sync::oneshot::channel(); |
| 1100 | handle.try_send(PersistRequest::FlushAndReport { reply: reply_tx }); |
| 1101 | let report = reply_rx.await.expect("flush report reply"); |
| 1102 | assert!(report.failures.is_empty(), "{:?}", report.failures); |
| 1103 | |
| 1104 | assert!( |
| 1105 | !committed_checkpoint.exists(), |
| 1106 | "the committed session's own checkpoint must be cleared" |
| 1107 | ); |
| 1108 | let survivor = verification_manager |
| 1109 | .load_session_checkpoint(&inflight_id) |
| 1110 | .expect("load the concurrent session's checkpoint") |
| 1111 | .expect("another session's checkpoint must never be cleared"); |
| 1112 | assert_eq!(survivor.metadata.id, inflight_id); |
| 1113 | assert!(inflight_checkpoint.exists()); |
| 1114 | handle.try_send(PersistRequest::Shutdown); |
| 1115 | task.await.expect("persistence actor join"); |
| 1116 | } |
| 1117 | |
| 1118 | #[tokio::test] |
| 1119 | async fn shutdown_preserves_inflight_checkpoint() { |
| 1120 | let tmp = tempfile::tempdir().expect("tempdir"); |
| 1121 | let sessions_dir = tmp.path().join("sessions"); |
| 1122 | let manager = SessionManager::new(sessions_dir.clone()).expect("manager"); |
| 1123 | let verification_manager = SessionManager::new(sessions_dir.clone()).expect("verify"); |
| 1124 | let session = crate::session_manager::create_saved_session_with_mode( |
| 1125 | &[], |
| 1126 | "deepseek-v4-pro", |
| 1127 | tmp.path(), |
| 1128 | 0, |
| 1129 | None, |
| 1130 | Some("agent"), |
| 1131 | ); |
| 1132 | let session_id = session.metadata.id.clone(); |
| 1133 | let checkpoint_path = seed_checkpoint_file(&sessions_dir, &session_id); |
| 1134 | let (handle, task) = spawn_persistence_actor(manager); |
| 1135 | |
| 1136 | handle.try_send(PersistRequest::SaveCheckpoint { session }); |
| 1137 | handle.try_send(PersistRequest::Shutdown); |
| 1138 | task.await.expect("persistence actor join"); |
| 1139 | |
| 1140 | assert!( |
| 1141 | checkpoint_path.exists(), |
| 1142 | "shutdown must never unconditionally clear an in-flight checkpoint" |
| 1143 | ); |
| 1144 | let recovered = verification_manager |
| 1145 | .load_session_checkpoint(&session_id) |
| 1146 | .expect("load checkpoint after shutdown") |
| 1147 | .expect("in-flight work must survive shutdown for recovery review"); |
| 1148 | assert_eq!(recovered.metadata.id, session_id); |
| 1149 | } |
| 1150 | |
| 1151 | #[tokio::test] |
| 1152 | async fn newer_inflight_checkpoint_supersedes_pending_completed_commit() { |
| 1153 | let tmp = tempfile::tempdir().expect("tempdir"); |
| 1154 | let sessions_dir = tmp.path().join("sessions"); |
| 1155 | let manager = SessionManager::new(sessions_dir.clone()).expect("manager"); |
| 1156 | let completed = crate::session_manager::create_saved_session_with_mode( |
| 1157 | &[], |
| 1158 | "deepseek-v4-pro", |
| 1159 | tmp.path(), |
| 1160 | 0, |
| 1161 | None, |
| 1162 | Some("agent"), |
| 1163 | ); |
| 1164 | let mut inflight = crate::session_manager::create_saved_session_with_mode( |
| 1165 | &[], |
| 1166 | "deepseek-v4-pro", |
| 1167 | tmp.path(), |
| 1168 | 0, |
| 1169 | None, |
| 1170 | Some("agent"), |
| 1171 | ); |
| 1172 | // The new turn reuses the same session id: a fresh in-flight |
| 1173 | // checkpoint arrives while the previous completion is still queued. |
| 1174 | inflight.metadata.id = completed.metadata.id.clone(); |
| 1175 | let session_id = completed.metadata.id.clone(); |
| 1176 | let (handle, task) = spawn_persistence_actor(manager); |
| 1177 | |
| 1178 | handle.try_send(PersistRequest::CompletedCommit { session: completed }); |
| 1179 | handle.try_send(PersistRequest::SaveCheckpoint { session: inflight }); |
| 1180 | let (reply_tx, reply_rx) = tokio::sync::oneshot::channel(); |
| 1181 | handle.try_send(PersistRequest::FlushAndReport { reply: reply_tx }); |
| 1182 | let report = reply_rx.await.expect("flush report reply"); |
| 1183 | |
| 1184 | assert!( |
| 1185 | !report |
| 1186 | .failures |
| 1187 | .iter() |
| 1188 | .any(|(what, _)| what.starts_with("clear-checkpoint:")), |
| 1189 | "the newer in-flight checkpoint must not be cleared, got: {:?}", |
| 1190 | report.failures |
| 1191 | ); |
| 1192 | let checkpoint = std::fs::read_to_string( |
| 1193 | sessions_dir |
| 1194 | .join("checkpoints") |
| 1195 | .join(format!("{session_id}.json")), |
| 1196 | ) |
| 1197 | .expect("the newer checkpoint must survive the drain"); |
| 1198 | assert!( |
| 1199 | !checkpoint.is_empty(), |
| 1200 | "the newer checkpoint body must be on disk" |
| 1201 | ); |
| 1202 | assert!( |
| 1203 | report.completed >= 1, |
| 1204 | "the newer checkpoint write must be counted, got: {report:?}" |
| 1205 | ); |
| 1206 | handle.try_send(PersistRequest::Shutdown); |
| 1207 | task.await.expect("persistence actor join"); |
| 1208 | } |
| 1209 | #[test] |
| 1210 | fn offline_queue_editor_lease_survives_until_pending_write_finishes() { |
| 1211 | let directory = tempfile::tempdir().expect("queue fixture"); |
| 1212 | let manager = SessionManager::new(directory.path().join("sessions")).expect("manager"); |
| 1213 | let lease = manager |
| 1214 | .acquire_offline_queue_lease("session-A") |
| 1215 | .expect("first editor"); |
| 1216 | let mut pending = PendingState::default(); |
| 1217 | pending.absorb(PersistRequest::OfflineQueue { |
| 1218 | state: OfflineQueueState { |
| 1219 | draft: Some(QueuedSessionMessage { |
| 1220 | display: "last edited draft".into(), |
| 1221 | skill_instruction: None, |
| 1222 | skill_provenance: None, |
| 1223 | }), |
| 1224 | ..OfflineQueueState::default() |
| 1225 | }, |
| 1226 | lease: Arc::clone(&lease), |
| 1227 | }); |
| 1228 | drop(lease); // The old window changed session before the actor ran. |
| 1229 | assert!(manager.acquire_offline_queue_lease("session-A").is_err()); |
| 1230 | let report = flush_inner(&manager, &mut pending); |
| 1231 | assert!(report.failures.is_empty(), "draft write failed: {report:?}"); |
| 1232 | assert_eq!(report.completed, 1); |
| 1233 | let _next_editor = manager |
| 1234 | .acquire_offline_queue_lease("session-A") |
| 1235 | .expect("released after write"); |
| 1236 | assert_eq!( |
| 1237 | manager |
| 1238 | .load_offline_queue_state("session-A") |
| 1239 | .unwrap() |
| 1240 | .unwrap() |
| 1241 | .draft |
| 1242 | .unwrap() |
| 1243 | .display, |
| 1244 | "last edited draft" |
| 1245 | ); |
| 1246 | } |
| 1247 | } |
| 1248 |