| 1 | use super::*; |
| 2 | use crate::automation_manager::{ |
| 3 | AutomationManager, AutomationStatus, CreateAutomationRequest, run_now_shared, |
| 4 | }; |
| 5 | use crate::runtime_threads::{RuntimeThreadManager, RuntimeThreadManagerConfig}; |
| 6 | use crate::task_manager::{TaskManager, TaskManagerConfig}; |
| 7 | |
| 8 | fn fixture_config() -> Config { |
| 9 | let mut config = Config { |
| 10 | api_key: Some("local-runtime-binding-fixture".into()), |
| 11 | base_url: Some("http://127.0.0.1:1/v1".into()), |
| 12 | ..Config::default() |
| 13 | }; |
| 14 | config.set_feature("mcp", false).unwrap(); |
| 15 | config.set_feature("subagents", false).unwrap(); |
| 16 | config |
| 17 | } |
| 18 | |
| 19 | #[tokio::test] |
| 20 | async fn runtime_store_binding_persists_on_exit_without_a_model_turn() -> anyhow::Result<()> { |
| 21 | let _environment = crate::test_support::lock_test_env(); |
| 22 | let root = tempfile::tempdir()?; |
| 23 | let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", root.path()); |
| 24 | let _runtime = crate::test_support::EnvVarGuard::remove("CODEWHALE_RUNTIME_DIR"); |
| 25 | let _legacy = crate::test_support::EnvVarGuard::remove("DEEPSEEK_RUNTIME_DIR"); |
| 26 | let explicit_store = crate::test_support::EnvVarGuard::set( |
| 27 | "CODEWHALE_RUNTIME_DIR", |
| 28 | root.path().join("original-runtime"), |
| 29 | ); |
| 30 | let mut config = fixture_config(); |
| 31 | let sessions = SessionManager::default_location()?; |
| 32 | let original = crate::session_manager::create_saved_session_with_id_and_mode( |
| 33 | "legacy-conversation".into(), |
| 34 | &[text_message("user", "retain this earlier conversation")], |
| 35 | "deepseek-v4-pro", |
| 36 | root.path(), |
| 37 | 0, |
| 38 | None, |
| 39 | None, |
| 40 | ); |
| 41 | assert!(original.metadata.runtime_store.is_none()); |
| 42 | sessions.save_session(&original)?; |
| 43 | sessions.save_checkpoint(&original)?; |
| 44 | let mut app = Box::new(create_test_app()); |
| 45 | apply_loaded_session_with_goal(&mut app, &mut config, &original, None) |
| 46 | .map_err(anyhow::Error::msg)?; |
| 47 | let task_config = TaskManagerConfig::from_runtime(&config, root.path().into(), None, Some(1)); |
| 48 | let tasks = TaskManager::start( |
| 49 | task_config.clone(), |
| 50 | config.clone(), |
| 51 | app.plugin_registry.clone(), |
| 52 | &original.metadata.id, |
| 53 | None, |
| 54 | ) |
| 55 | .await?; |
| 56 | let binding = tasks |
| 57 | .session_store_binding() |
| 58 | .expect("attached Runtime store"); |
| 59 | app.runtime_services.task_manager = Some(tasks.clone()); |
| 60 | let (handle, actor) = |
| 61 | persistence_actor::spawn_persistence_actor(SessionManager::default_location()?); |
| 62 | // Match clean exit ordering: no Engine turn, checkpoint or snapshot has |
| 63 | // been queued by this host before its TaskManager stops. |
| 64 | tasks.shutdown_and_wait().await?; |
| 65 | assert!( |
| 66 | super::super::event_loop::persist_settled_session_on_shutdown(&mut app, &handle) |
| 67 | .map_err(anyhow::Error::msg)? |
| 68 | ); |
| 69 | assert!(handle.try_send(PersistRequest::Shutdown)); |
| 70 | actor.await?; |
| 71 | let saved = sessions.load_session(&original.metadata.id)?; |
| 72 | assert_eq!(saved.metadata.runtime_store.as_ref(), Some(&binding)); |
| 73 | assert_eq!(saved.metadata.title, original.metadata.title); |
| 74 | assert_eq!(saved.messages, original.messages); |
| 75 | assert!( |
| 76 | sessions |
| 77 | .load_session_checkpoint(&original.metadata.id)? |
| 78 | .is_none() |
| 79 | ); |
| 80 | drop(app); |
| 81 | drop(tasks); |
| 82 | drop(explicit_store); |
| 83 | // Ordinary resume now reopens the same authority without an env override. |
| 84 | let resumed = TaskManager::start( |
| 85 | task_config, |
| 86 | config, |
| 87 | Arc::new(crate::plugins::PluginRegistry::empty(root.path())), |
| 88 | &saved.metadata.id, |
| 89 | saved.metadata.runtime_store.as_ref(), |
| 90 | ) |
| 91 | .await?; |
| 92 | assert_eq!(resumed.execution_scope(), binding.execution_scope); |
| 93 | resumed.shutdown_and_wait().await?; |
| 94 | Ok(()) |
| 95 | } |
| 96 | |
| 97 | #[tokio::test] |
| 98 | async fn runtime_store_binding_exit_preserves_inflight_recovery() -> anyhow::Result<()> { |
| 99 | let _environment = crate::test_support::lock_test_env(); |
| 100 | let root = tempfile::tempdir()?; |
| 101 | let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", root.path()); |
| 102 | let sessions = SessionManager::default_location()?; |
| 103 | for (loading, dispatch) in [(true, false), (false, true)] { |
| 104 | let original = crate::session_manager::create_saved_session_with_mode( |
| 105 | &[], |
| 106 | "deepseek-v4-pro", |
| 107 | root.path(), |
| 108 | 0, |
| 109 | None, |
| 110 | None, |
| 111 | ); |
| 112 | let path = sessions.save_session(&original)?; |
| 113 | let checkpoint = sessions.save_checkpoint(&original)?; |
| 114 | let saved_before = std::fs::read(&path)?; |
| 115 | let checkpoint_before = std::fs::read(&checkpoint)?; |
| 116 | let mut app = Box::new(create_test_app()); |
| 117 | app.current_session_id = Some(original.metadata.id.clone()); |
| 118 | app.is_loading = loading; |
| 119 | app.dispatch_in_flight = dispatch; |
| 120 | let (handle, actor) = |
| 121 | persistence_actor::spawn_persistence_actor(SessionManager::default_location()?); |
| 122 | assert!( |
| 123 | !super::super::event_loop::persist_settled_session_on_shutdown(&mut app, &handle) |
| 124 | .map_err(anyhow::Error::msg)? |
| 125 | ); |
| 126 | assert!(handle.try_send(PersistRequest::Shutdown)); |
| 127 | actor.await?; |
| 128 | assert_eq!(std::fs::read(path)?, saved_before); |
| 129 | assert_eq!(std::fs::read(checkpoint)?, checkpoint_before); |
| 130 | } |
| 131 | Ok(()) |
| 132 | } |
| 133 | |
| 134 | #[tokio::test] |
| 135 | async fn runtime_store_binding_survives_launch_snapshot_and_resume() -> anyhow::Result<()> { |
| 136 | let _environment = crate::test_support::lock_test_env(); |
| 137 | let root = tempfile::tempdir()?; |
| 138 | let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", root.path()); |
| 139 | let _runtime = crate::test_support::EnvVarGuard::remove("CODEWHALE_RUNTIME_DIR"); |
| 140 | let _legacy = crate::test_support::EnvVarGuard::remove("DEEPSEEK_RUNTIME_DIR"); |
| 141 | let config = fixture_config(); |
| 142 | let mut app = Box::new(create_test_app()); |
| 143 | app.workspace = root.path().into(); |
| 144 | let initial_id = super::super::event_loop::ensure_runtime_session_id(&mut app); |
| 145 | let task_config = TaskManagerConfig::from_runtime(&config, root.path().into(), None, Some(1)); |
| 146 | let tasks = TaskManager::start( |
| 147 | task_config.clone(), |
| 148 | config.clone(), |
| 149 | app.plugin_registry.clone(), |
| 150 | &initial_id, |
| 151 | None, |
| 152 | ) |
| 153 | .await?; |
| 154 | app.runtime_services.task_manager = Some(tasks.clone()); |
| 155 | let sessions = SessionManager::default_location()?; |
| 156 | // A saved initial conversation may later be deleted while another launch |
| 157 | // still refers to its Runtime store. |
| 158 | let initial = build_session_snapshot(&mut app, &sessions).map_err(anyhow::Error::msg)?; |
| 159 | sessions.save_session(&initial)?; |
| 160 | let launch = begin_launch_session(&mut app, None); |
| 161 | assert!(!launch.is_error, "{:?}", launch.message); |
| 162 | assert_ne!(app.current_session_id.as_deref(), Some(initial_id.as_str())); |
| 163 | let saved = build_session_snapshot(&mut app, &sessions).map_err(anyhow::Error::msg)?; |
| 164 | let binding = saved |
| 165 | .metadata |
| 166 | .runtime_store |
| 167 | .clone() |
| 168 | .expect("attached host binding"); |
| 169 | assert_eq!(binding.execution_scope, tasks.execution_scope()); |
| 170 | sessions.save_session(&saved)?; |
| 171 | let mut automations = AutomationManager::open(root.path().join("automations"))?; |
| 172 | automations.bind_task_manager(&tasks)?; |
| 173 | let automation = automations.create_automation(CreateAutomationRequest { |
| 174 | name: "resumed ownership fixture".into(), |
| 175 | prompt: "local fixture only".into(), |
| 176 | rrule: "FREQ=HOURLY;INTERVAL=1".into(), |
| 177 | cwds: vec![root.path().into()], |
| 178 | model: None, |
| 179 | model_provider: None, |
| 180 | model_provider_id: None, |
| 181 | mode: None, |
| 182 | allow_shell: Some(false), |
| 183 | trust_mode: Some(false), |
| 184 | auto_approve: Some(false), |
| 185 | delivery_mode: None, |
| 186 | status: Some(AutomationStatus::Paused), |
| 187 | })?; |
| 188 | tasks.shutdown_and_wait().await?; |
| 189 | drop(app); |
| 190 | drop(tasks); |
| 191 | drop(automations); |
| 192 | sessions.delete_session(&initial_id)?; |
| 193 | assert!( |
| 194 | binding.data_dir.is_dir(), |
| 195 | "transcript deletion cannot erase Runtime authority" |
| 196 | ); |
| 197 | let loaded = sessions.load_session(&saved.metadata.id)?; |
| 198 | assert_eq!(loaded.metadata.runtime_store.as_ref(), Some(&binding)); |
| 199 | let mut resumed = Box::new(create_test_app()); |
| 200 | let mut resumed_config = config.clone(); |
| 201 | apply_loaded_session_with_goal(&mut resumed, &mut resumed_config, &loaded, None) |
| 202 | .map_err(anyhow::Error::msg)?; |
| 203 | let tasks = TaskManager::start( |
| 204 | task_config.clone(), |
| 205 | config.clone(), |
| 206 | resumed.plugin_registry.clone(), |
| 207 | &loaded.metadata.id, |
| 208 | loaded.metadata.runtime_store.as_ref(), |
| 209 | ) |
| 210 | .await?; |
| 211 | assert_eq!( |
| 212 | tasks.execution_scope(), |
| 213 | automation.execution_scope.as_deref().unwrap() |
| 214 | ); |
| 215 | resumed.runtime_services.task_manager = Some(tasks.clone()); |
| 216 | let automations = Arc::new(tokio::sync::Mutex::new(AutomationManager::open( |
| 217 | root.path().join("automations"), |
| 218 | )?)); |
| 219 | // The real Run-now admission must now create its durable receipt. The |
| 220 | // configured endpoint is closed loopback and no shell/tool is authorized. |
| 221 | let run = run_now_shared(&automations, &automation.id, &tasks).await?; |
| 222 | assert!(run.task_id.is_some(), "{run:?}"); |
| 223 | assert_eq!( |
| 224 | automations |
| 225 | .lock() |
| 226 | .await |
| 227 | .list_runs(&automation.id, None)? |
| 228 | .len(), |
| 229 | 1 |
| 230 | ); |
| 231 | assert_eq!( |
| 232 | automations |
| 233 | .lock() |
| 234 | .await |
| 235 | .get_automation(&automation.id)? |
| 236 | .execution_scope, |
| 237 | automation.execution_scope |
| 238 | ); |
| 239 | tasks.shutdown_and_wait().await?; |
| 240 | drop(resumed); |
| 241 | drop(tasks); |
| 242 | // Reproduce the old resume path: deriving a store from the saved |
| 243 | // conversation id without its binding opens a foreign scope and cannot run. |
| 244 | let foreign = TaskManager::start( |
| 245 | task_config, |
| 246 | config, |
| 247 | Arc::new(crate::plugins::PluginRegistry::empty(root.path())), |
| 248 | &loaded.metadata.id, |
| 249 | None, |
| 250 | ) |
| 251 | .await?; |
| 252 | let foreign_automations = Arc::new(tokio::sync::Mutex::new(AutomationManager::open( |
| 253 | root.path().join("automations"), |
| 254 | )?)); |
| 255 | let definition_path = root |
| 256 | .path() |
| 257 | .join("automations/automations") |
| 258 | .join(format!("{}.json", automation.id)); |
| 259 | let before_foreign_run = std::fs::read(&definition_path)?; |
| 260 | let error = run_now_shared(&foreign_automations, &automation.id, &foreign) |
| 261 | .await |
| 262 | .unwrap_err(); |
| 263 | assert!( |
| 264 | error |
| 265 | .to_string() |
| 266 | .contains("another Runtime execution scope"), |
| 267 | "{error:#}" |
| 268 | ); |
| 269 | assert_eq!( |
| 270 | foreign_automations |
| 271 | .lock() |
| 272 | .await |
| 273 | .list_runs(&automation.id, None)? |
| 274 | .len(), |
| 275 | 1 |
| 276 | ); |
| 277 | assert_eq!(std::fs::read(definition_path)?, before_foreign_run); |
| 278 | let mut other_app = Box::new(create_test_app()); |
| 279 | other_app.runtime_services.task_manager = Some(foreign.clone()); |
| 280 | other_app.input = "preserve pending input".into(); |
| 281 | let old_id = other_app.current_session_id.clone(); |
| 282 | let error = apply_loaded_session_with_goal(&mut other_app, &mut resumed_config, &loaded, None) |
| 283 | .unwrap_err(); |
| 284 | // The refusal must name the route that actually works. "Resume it in a new |
| 285 | // Codewhale process" was true but unactionable: starting a new process and |
| 286 | // then picking the session from `/resume` returns here, because that is |
| 287 | // this same switch path (#6207, #6225). |
| 288 | assert!( |
| 289 | error.contains("codewhale resume"), |
| 290 | "the refusal must point at the direct-open path: {error}" |
| 291 | ); |
| 292 | assert!( |
| 293 | error.contains(&loaded.metadata.id), |
| 294 | "the refusal must name the session to open: {error}" |
| 295 | ); |
| 296 | assert_eq!(other_app.current_session_id, old_id); |
| 297 | assert_eq!(other_app.input, "preserve pending input"); |
| 298 | foreign.shutdown_and_wait().await?; |
| 299 | Ok(()) |
| 300 | } |
| 301 | |
| 302 | #[cfg(unix)] |
| 303 | #[test] |
| 304 | fn runtime_store_binding_retention_does_not_follow_session_directory_symlinks() -> anyhow::Result<()> |
| 305 | { |
| 306 | let _environment = crate::test_support::lock_test_env(); |
| 307 | let root = tempfile::tempdir()?; |
| 308 | let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", root.path()); |
| 309 | let sessions = SessionManager::default_location()?; |
| 310 | let saved = crate::session_manager::create_saved_session_with_id_and_mode( |
| 311 | "linked-session".into(), |
| 312 | &[], |
| 313 | "fixture", |
| 314 | root.path(), |
| 315 | 0, |
| 316 | None, |
| 317 | None, |
| 318 | ); |
| 319 | sessions.save_session(&saved)?; |
| 320 | let target = root.path().join("unrelated-directory"); |
| 321 | std::fs::create_dir_all(&target)?; |
| 322 | std::fs::write(target.join("keep.txt"), "preserve user data")?; |
| 323 | let link = root.path().join("sessions/linked-session"); |
| 324 | std::os::unix::fs::symlink(&target, &link)?; |
| 325 | sessions.delete_session("linked-session")?; |
| 326 | assert!(!link.exists()); |
| 327 | assert_eq!( |
| 328 | std::fs::read_to_string(target.join("keep.txt"))?, |
| 329 | "preserve user data" |
| 330 | ); |
| 331 | Ok(()) |
| 332 | } |
| 333 | |
| 334 | #[test] |
| 335 | fn runtime_store_binding_rejects_foreign_missing_or_overridden_store() -> anyhow::Result<()> { |
| 336 | let _environment = crate::test_support::lock_test_env(); |
| 337 | let root = tempfile::tempdir()?; |
| 338 | let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", root.path()); |
| 339 | let _runtime = crate::test_support::EnvVarGuard::remove("CODEWHALE_RUNTIME_DIR"); |
| 340 | let _legacy = crate::test_support::EnvVarGuard::remove("DEEPSEEK_RUNTIME_DIR"); |
| 341 | let config = fixture_config(); |
| 342 | let cfg = RuntimeThreadManagerConfig::for_session(root.path().join("tasks"), "original"); |
| 343 | let runtime = RuntimeThreadManager::open(config.clone(), root.path().into(), cfg.clone())?; |
| 344 | let binding = runtime.session_store_binding(); |
| 345 | drop(runtime); |
| 346 | let state_path = binding.data_dir.join("state.json"); |
| 347 | let before = std::fs::read(&state_path)?; |
| 348 | let mut wrong = binding.clone(); |
| 349 | wrong.execution_scope = "0".repeat(64); |
| 350 | let open = |binding: &crate::runtime_threads::RuntimeStoreBinding| { |
| 351 | RuntimeThreadManager::open_for_session( |
| 352 | config.clone(), |
| 353 | root.path().into(), |
| 354 | cfg.clone(), |
| 355 | Arc::new(crate::plugins::PluginRegistry::empty(root.path())), |
| 356 | Some(binding), |
| 357 | ) |
| 358 | }; |
| 359 | assert!( |
| 360 | open(&wrong) |
| 361 | .err() |
| 362 | .unwrap() |
| 363 | .to_string() |
| 364 | .contains("ownership does not match") |
| 365 | ); |
| 366 | assert_eq!(std::fs::read(&state_path)?, before); |
| 367 | wrong.data_dir = root.path().join("missing-store"); |
| 368 | assert!(open(&wrong).is_err()); |
| 369 | assert!( |
| 370 | !wrong.data_dir.exists(), |
| 371 | "saved binding cannot create a replacement authority" |
| 372 | ); |
| 373 | let _override = crate::test_support::EnvVarGuard::set( |
| 374 | "CODEWHALE_RUNTIME_DIR", |
| 375 | root.path().join("foreign-override"), |
| 376 | ); |
| 377 | assert!( |
| 378 | open(&binding) |
| 379 | .err() |
| 380 | .unwrap() |
| 381 | .to_string() |
| 382 | .contains("override conflicts") |
| 383 | ); |
| 384 | Ok(()) |
| 385 | } |
| 386 | |
| 387 | #[test] |
| 388 | fn missing_runtime_store_recovers_without_reusing_authority_or_resurrecting_stale_binding() |
| 389 | -> anyhow::Result<()> { |
| 390 | let _environment = crate::test_support::lock_test_env(); |
| 391 | let root = tempfile::tempdir()?; |
| 392 | let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", root.path()); |
| 393 | let _runtime = crate::test_support::EnvVarGuard::remove("CODEWHALE_RUNTIME_DIR"); |
| 394 | let _legacy = crate::test_support::EnvVarGuard::remove("DEEPSEEK_RUNTIME_DIR"); |
| 395 | let sessions = SessionManager::default_location()?; |
| 396 | let mut saved = crate::session_manager::create_saved_session_with_id_and_mode( |
| 397 | "interrupted".into(), |
| 398 | &[text_message("user", "retain my work")], |
| 399 | "deepseek-v4-pro", |
| 400 | root.path(), |
| 401 | 0, |
| 402 | None, |
| 403 | None, |
| 404 | ); |
| 405 | let missing = crate::runtime_threads::RuntimeStoreBinding { |
| 406 | data_dir: root.path().join("sessions/previous/runtime"), |
| 407 | execution_scope: "0".repeat(64), |
| 408 | }; |
| 409 | saved.metadata.runtime_store = Some(missing.clone()); |
| 410 | sessions.save_session(&saved)?; |
| 411 | let stale = saved.clone(); |
| 412 | let manager = RuntimeThreadManager::open_for_session( |
| 413 | fixture_config(), |
| 414 | root.path().into(), |
| 415 | RuntimeThreadManagerConfig::for_session(root.path().join("tasks"), "interrupted"), |
| 416 | Arc::new(crate::plugins::PluginRegistry::empty(root.path())), |
| 417 | Some(&missing), |
| 418 | )?; |
| 419 | let recovered = manager.session_store_binding(); |
| 420 | assert_ne!(recovered.execution_scope, missing.execution_scope); |
| 421 | assert_ne!(recovered.data_dir, missing.data_dir); |
| 422 | assert!(!missing.data_dir.exists()); |
| 423 | assert!( |
| 424 | RuntimeThreadManager::open_for_session( |
| 425 | fixture_config(), |
| 426 | root.path().into(), |
| 427 | RuntimeThreadManagerConfig::for_session(root.path().join("tasks"), "interrupted"), |
| 428 | Arc::new(crate::plugins::PluginRegistry::empty(root.path())), |
| 429 | Some(&missing), |
| 430 | ) |
| 431 | .is_err(), |
| 432 | "concurrent recovery cannot mint a second owner" |
| 433 | ); |
| 434 | // Losing the process before the repaired binding is saved must leave a |
| 435 | // retryable, session-scoped store, not an orphan or a second authority. |
| 436 | drop(manager); |
| 437 | let manager = RuntimeThreadManager::open_for_session( |
| 438 | fixture_config(), |
| 439 | root.path().into(), |
| 440 | RuntimeThreadManagerConfig::for_session(root.path().join("tasks"), "interrupted"), |
| 441 | Arc::new(crate::plugins::PluginRegistry::empty(root.path())), |
| 442 | Some(&missing), |
| 443 | )?; |
| 444 | assert_eq!(manager.session_store_binding(), recovered); |
| 445 | let other_recovery = RuntimeThreadManager::open_for_session( |
| 446 | fixture_config(), |
| 447 | root.path().into(), |
| 448 | RuntimeThreadManagerConfig::for_session(root.path().join("tasks"), "other-interrupted"), |
| 449 | Arc::new(crate::plugins::PluginRegistry::empty(root.path())), |
| 450 | Some(&missing), |
| 451 | )?; |
| 452 | assert_ne!( |
| 453 | other_recovery.session_store_binding().data_dir, |
| 454 | recovered.data_dir |
| 455 | ); |
| 456 | assert_ne!( |
| 457 | other_recovery.session_store_binding().execution_scope, |
| 458 | recovered.execution_scope |
| 459 | ); |
| 460 | saved.metadata.runtime_store = Some(recovered.clone()); |
| 461 | sessions.save_session(&saved)?; |
| 462 | sessions.save_session(&stale)?; |
| 463 | sessions.save_checkpoint(&stale)?; |
| 464 | let durable = sessions.load_session("interrupted")?; |
| 465 | assert_eq!(durable.metadata.runtime_store, Some(recovered.clone())); |
| 466 | assert_eq!(durable.messages, stale.messages); |
| 467 | let competing = RuntimeThreadManager::open_for_session( |
| 468 | fixture_config(), |
| 469 | root.path().into(), |
| 470 | RuntimeThreadManagerConfig::for_session(root.path().join("tasks"), "competing"), |
| 471 | Arc::new(crate::plugins::PluginRegistry::empty(root.path())), |
| 472 | None, |
| 473 | )?; |
| 474 | let mut competing_snapshot = stale.clone(); |
| 475 | competing_snapshot.metadata.runtime_store = Some(competing.session_store_binding()); |
| 476 | assert!(sessions.save_session(&competing_snapshot).is_err()); |
| 477 | assert!(sessions.save_checkpoint(&competing_snapshot).is_err()); |
| 478 | assert_eq!( |
| 479 | sessions.load_session("interrupted")?.metadata.runtime_store, |
| 480 | Some(recovered), |
| 481 | "another valid owner cannot overwrite the completed recovery" |
| 482 | ); |
| 483 | Ok(()) |
| 484 | } |
| 485 | |
| 486 | #[tokio::test] |
| 487 | async fn picker_recovers_missing_store_into_the_idle_host_and_persists_before_returning() |
| 488 | -> anyhow::Result<()> { |
| 489 | let _environment = crate::test_support::lock_test_env(); |
| 490 | let root = tempfile::tempdir()?; |
| 491 | let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", root.path()); |
| 492 | let _runtime = crate::test_support::EnvVarGuard::remove("CODEWHALE_RUNTIME_DIR"); |
| 493 | let _legacy = crate::test_support::EnvVarGuard::remove("DEEPSEEK_RUNTIME_DIR"); |
| 494 | let sessions = SessionManager::default_location()?; |
| 495 | let mut config = fixture_config(); |
| 496 | let mut saved = crate::session_manager::create_saved_session_with_id_and_mode( |
| 497 | "picker-interrupted".into(), |
| 498 | &[text_message("user", "retain my work")], |
| 499 | "deepseek-v4-pro", |
| 500 | root.path(), |
| 501 | 0, |
| 502 | None, |
| 503 | None, |
| 504 | ); |
| 505 | saved.metadata.runtime_store = Some(crate::runtime_threads::RuntimeStoreBinding { |
| 506 | data_dir: root.path().join("sessions/previous/runtime"), |
| 507 | execution_scope: "0".repeat(64), |
| 508 | }); |
| 509 | sessions.save_session(&saved)?; |
| 510 | let mut app = Box::new(create_test_app()); |
| 511 | let tasks = TaskManager::start( |
| 512 | TaskManagerConfig::from_runtime(&config, root.path().into(), None, Some(1)), |
| 513 | config.clone(), |
| 514 | app.plugin_registry.clone(), |
| 515 | "picker-current", |
| 516 | None, |
| 517 | ) |
| 518 | .await?; |
| 519 | let binding = tasks.session_store_binding().expect("current host"); |
| 520 | app.runtime_services.task_manager = Some(tasks.clone()); |
| 521 | app.current_session_id = Some("picker-current".into()); |
| 522 | app.api_messages_mut() |
| 523 | .push(text_message("user", "current conversation")); |
| 524 | let current_messages = app.api_messages.clone(); |
| 525 | let plan_state = app.plan_state.clone(); |
| 526 | let held = plan_state |
| 527 | .try_lock() |
| 528 | .expect("hold Work state during recovery"); |
| 529 | assert!(apply_loaded_session_with_goal(&mut app, &mut config, &saved, None).is_err()); |
| 530 | assert_eq!(app.current_session_id.as_deref(), Some("picker-current")); |
| 531 | assert_eq!(app.api_messages, current_messages); |
| 532 | assert_eq!( |
| 533 | sessions |
| 534 | .load_session("picker-interrupted")? |
| 535 | .metadata |
| 536 | .runtime_store |
| 537 | .as_ref(), |
| 538 | Some(&binding), |
| 539 | "binding repair survives a contended UI restore" |
| 540 | ); |
| 541 | drop(held); |
| 542 | apply_loaded_session_with_goal(&mut app, &mut config, &saved, None) |
| 543 | .map_err(anyhow::Error::msg)?; |
| 544 | assert_eq!( |
| 545 | app.current_session_id.as_deref(), |
| 546 | Some("picker-interrupted") |
| 547 | ); |
| 548 | let durable = sessions.load_session("picker-interrupted")?; |
| 549 | assert_eq!(durable.metadata.runtime_store.as_ref(), Some(&binding)); |
| 550 | assert_eq!(durable.messages, saved.messages); |
| 551 | assert_eq!( |
| 552 | app.current_session_metadata.as_ref().unwrap().runtime_store, |
| 553 | Some(binding) |
| 554 | ); |
| 555 | sessions.save_session(&saved)?; |
| 556 | assert_eq!( |
| 557 | sessions |
| 558 | .load_session("picker-interrupted")? |
| 559 | .metadata |
| 560 | .runtime_store, |
| 561 | durable.metadata.runtime_store |
| 562 | ); |
| 563 | tasks.shutdown_and_wait().await?; |
| 564 | Ok(()) |
| 565 | } |
| 566 | |
| 567 | /// #6207: a store that exists but is empty, unheld, and scope-free holds |
| 568 | /// nothing a session switch could abandon. A force-quit leaves exactly that |
| 569 | /// shape — the directory is on disk, ownerless, with zero events — and |
| 570 | /// refusing it left the session unopenable while protecting nothing. |
| 571 | #[test] |
| 572 | fn adoptable_empty_store_reports_nothing_to_abandon() -> anyhow::Result<()> { |
| 573 | let _environment = crate::test_support::lock_test_env(); |
| 574 | let root = tempfile::tempdir()?; |
| 575 | let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", root.path()); |
| 576 | let _runtime = crate::test_support::EnvVarGuard::remove("CODEWHALE_RUNTIME_DIR"); |
| 577 | let _legacy = crate::test_support::EnvVarGuard::remove("DEEPSEEK_RUNTIME_DIR"); |
| 578 | |
| 579 | let store_dir = root.path().join("sessions/interrupted/runtime"); |
| 580 | // Open a real store, so the layout under test is the product's rather than |
| 581 | // the test's idea of it. |
| 582 | drop(crate::runtime_threads::RuntimeThreadStore::open( |
| 583 | store_dir.clone(), |
| 584 | )?); |
| 585 | |
| 586 | let binding = crate::runtime_threads::RuntimeStoreBinding { |
| 587 | data_dir: store_dir.clone(), |
| 588 | execution_scope: "0".repeat(64), |
| 589 | }; |
| 590 | assert!( |
| 591 | !binding.is_missing_session_store()?, |
| 592 | "the store exists, so the old predicate cannot recover it" |
| 593 | ); |
| 594 | assert!( |
| 595 | binding.has_no_durable_work()?, |
| 596 | "a freshly opened store holds nothing to abandon" |
| 597 | ); |
| 598 | assert!( |
| 599 | !binding.has_live_holder()?, |
| 600 | "nobody holds the freshly opened store" |
| 601 | ); |
| 602 | assert!( |
| 603 | !binding.has_scope_pinned_automation()?, |
| 604 | "no automations exist under the fixture home" |
| 605 | ); |
| 606 | assert!( |
| 607 | binding.is_adoptable_empty_store()?, |
| 608 | "empty, unheld, scope-free: adoptable" |
| 609 | ); |
| 610 | |
| 611 | // Each work directory must be load-bearing on its own. If `open` gains a |
| 612 | // directory that RUNTIME_STORE_WORK_DIRS misses, this is the assertion that |
| 613 | // notices, instead of the miss silently widening what a switch will adopt. |
| 614 | for dir in [ |
| 615 | "threads", |
| 616 | "turns", |
| 617 | "items", |
| 618 | "events", |
| 619 | "goals", |
| 620 | "agent-mail", |
| 621 | "turn-operations", |
| 622 | ] { |
| 623 | let marker = store_dir.join(dir).join("work.json"); |
| 624 | std::fs::write(&marker, "{}")?; |
| 625 | assert!( |
| 626 | !binding.has_no_durable_work()?, |
| 627 | "{dir} holds work; the store must not be adopted" |
| 628 | ); |
| 629 | assert!( |
| 630 | !binding.is_adoptable_empty_store()?, |
| 631 | "{dir} blocks the adopt" |
| 632 | ); |
| 633 | std::fs::remove_file(&marker)?; |
| 634 | } |
| 635 | assert!( |
| 636 | binding.is_adoptable_empty_store()?, |
| 637 | "markers removed: adoptable again" |
| 638 | ); |
| 639 | Ok(()) |
| 640 | } |
| 641 | |
| 642 | /// #6207: the race that reverted the first fix — a live foreign TaskManager |
| 643 | /// holds the store while its disk state is still empty, so emptiness alone |
| 644 | /// cannot tell abandonment from a holder that has not flushed yet. The |
| 645 | /// process-owner lock is held for the manager's lifetime, which is what |
| 646 | /// distinguishes the two. |
| 647 | #[test] |
| 648 | fn adoptable_empty_store_refuses_a_live_holder() -> anyhow::Result<()> { |
| 649 | let _environment = crate::test_support::lock_test_env(); |
| 650 | let root = tempfile::tempdir()?; |
| 651 | let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", root.path()); |
| 652 | let _runtime = crate::test_support::EnvVarGuard::remove("CODEWHALE_RUNTIME_DIR"); |
| 653 | let _legacy = crate::test_support::EnvVarGuard::remove("DEEPSEEK_RUNTIME_DIR"); |
| 654 | |
| 655 | let store_dir = root.path().join("sessions/held/runtime"); |
| 656 | drop(crate::runtime_threads::RuntimeThreadStore::open( |
| 657 | store_dir.clone(), |
| 658 | )?); |
| 659 | let binding = crate::runtime_threads::RuntimeStoreBinding { |
| 660 | data_dir: store_dir.clone(), |
| 661 | execution_scope: "0".repeat(64), |
| 662 | }; |
| 663 | assert!(binding.is_adoptable_empty_store()?); |
| 664 | |
| 665 | let _held = crate::runtime_threads::RuntimeProcessOwnerLock::acquire(&store_dir)?; |
| 666 | assert!( |
| 667 | binding.has_live_holder()?, |
| 668 | "the held owner lock reads as held" |
| 669 | ); |
| 670 | assert!( |
| 671 | !binding.is_adoptable_empty_store()?, |
| 672 | "a held store must refuse even while its disk state is empty" |
| 673 | ); |
| 674 | drop(_held); |
| 675 | assert!( |
| 676 | !binding.has_live_holder()?, |
| 677 | "releasing the lock releases the hold" |
| 678 | ); |
| 679 | assert!( |
| 680 | binding.is_adoptable_empty_store()?, |
| 681 | "unheld again: adoptable" |
| 682 | ); |
| 683 | Ok(()) |
| 684 | } |
| 685 | |
| 686 | /// #6207: scope-pinned automations are recorded outside the store |
| 687 | /// directories, so an otherwise empty store with one bound to its scope |
| 688 | /// still refuses — adopting it would orphan their scheduled work. |
| 689 | #[test] |
| 690 | fn adoptable_empty_store_refuses_a_scope_pinned_automation() -> anyhow::Result<()> { |
| 691 | let _environment = crate::test_support::lock_test_env(); |
| 692 | let root = tempfile::tempdir()?; |
| 693 | let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", root.path()); |
| 694 | let _runtime = crate::test_support::EnvVarGuard::remove("CODEWHALE_RUNTIME_DIR"); |
| 695 | let _legacy = crate::test_support::EnvVarGuard::remove("DEEPSEEK_RUNTIME_DIR"); |
| 696 | |
| 697 | let store_dir = root.path().join("sessions/pinned/runtime"); |
| 698 | drop(crate::runtime_threads::RuntimeThreadStore::open( |
| 699 | store_dir.clone(), |
| 700 | )?); |
| 701 | let scope = "ab".repeat(32); |
| 702 | let binding = crate::runtime_threads::RuntimeStoreBinding { |
| 703 | data_dir: store_dir.clone(), |
| 704 | execution_scope: scope.clone(), |
| 705 | }; |
| 706 | assert!(binding.is_adoptable_empty_store()?); |
| 707 | |
| 708 | let automations = AutomationManager::open(root.path().join("automations"))?; |
| 709 | let created = automations.create_automation(CreateAutomationRequest { |
| 710 | name: "scope fixture".into(), |
| 711 | prompt: "local fixture only".into(), |
| 712 | rrule: "FREQ=HOURLY;INTERVAL=1".into(), |
| 713 | cwds: vec![root.path().into()], |
| 714 | model: None, |
| 715 | model_provider: None, |
| 716 | model_provider_id: None, |
| 717 | mode: None, |
| 718 | allow_shell: Some(false), |
| 719 | trust_mode: Some(false), |
| 720 | auto_approve: Some(false), |
| 721 | delivery_mode: None, |
| 722 | status: Some(AutomationStatus::Paused), |
| 723 | })?; |
| 724 | automations.edit_automation(&created.id, |record| { |
| 725 | let mut record = record.ok_or_else(|| anyhow::anyhow!("fresh automation must exist"))?; |
| 726 | record.execution_scope = Some(scope.clone()); |
| 727 | Ok(Some(record)) |
| 728 | })?; |
| 729 | assert!( |
| 730 | binding.has_scope_pinned_automation()?, |
| 731 | "the bound automation is visible from the binding's scope" |
| 732 | ); |
| 733 | assert!( |
| 734 | !binding.is_adoptable_empty_store()?, |
| 735 | "a scope-pinned automation blocks the adopt" |
| 736 | ); |
| 737 | Ok(()) |
| 738 | } |
| 739 | |
| 740 | /// #6207 end to end: the picker adopts an existing-but-empty unheld store |
| 741 | /// into the idle host and persists the repaired binding, the way the |
| 742 | /// missing-store path already does. |
| 743 | #[tokio::test] |
| 744 | async fn picker_adopts_existing_empty_unheld_store() -> anyhow::Result<()> { |
| 745 | let _environment = crate::test_support::lock_test_env(); |
| 746 | let root = tempfile::tempdir()?; |
| 747 | let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", root.path()); |
| 748 | let _runtime = crate::test_support::EnvVarGuard::remove("CODEWHALE_RUNTIME_DIR"); |
| 749 | let _legacy = crate::test_support::EnvVarGuard::remove("DEEPSEEK_RUNTIME_DIR"); |
| 750 | let sessions = SessionManager::default_location()?; |
| 751 | let mut config = fixture_config(); |
| 752 | |
| 753 | let store_dir = root.path().join("sessions/previous/runtime"); |
| 754 | drop(crate::runtime_threads::RuntimeThreadStore::open( |
| 755 | store_dir.clone(), |
| 756 | )?); |
| 757 | let mut saved = crate::session_manager::create_saved_session_with_id_and_mode( |
| 758 | "picker-adoptable".into(), |
| 759 | &[text_message("user", "retain my work")], |
| 760 | "deepseek-v4-pro", |
| 761 | root.path(), |
| 762 | 0, |
| 763 | None, |
| 764 | None, |
| 765 | ); |
| 766 | saved.metadata.runtime_store = Some(crate::runtime_threads::RuntimeStoreBinding { |
| 767 | data_dir: store_dir, |
| 768 | execution_scope: "0".repeat(64), |
| 769 | }); |
| 770 | sessions.save_session(&saved)?; |
| 771 | |
| 772 | let mut app = Box::new(create_test_app()); |
| 773 | let tasks = TaskManager::start( |
| 774 | TaskManagerConfig::from_runtime(&config, root.path().into(), None, Some(1)), |
| 775 | config.clone(), |
| 776 | app.plugin_registry.clone(), |
| 777 | "picker-current", |
| 778 | None, |
| 779 | ) |
| 780 | .await?; |
| 781 | let binding = tasks.session_store_binding().expect("current host"); |
| 782 | app.runtime_services.task_manager = Some(tasks.clone()); |
| 783 | app.current_session_id = Some("picker-current".into()); |
| 784 | |
| 785 | apply_loaded_session_with_goal(&mut app, &mut config, &saved, None) |
| 786 | .map_err(anyhow::Error::msg)?; |
| 787 | assert_eq!(app.current_session_id.as_deref(), Some("picker-adoptable")); |
| 788 | let durable = sessions.load_session("picker-adoptable")?; |
| 789 | assert_eq!(durable.metadata.runtime_store.as_ref(), Some(&binding)); |
| 790 | assert_eq!(durable.messages, saved.messages); |
| 791 | tasks.shutdown_and_wait().await?; |
| 792 | Ok(()) |
| 793 | } |
| 794 |