| 1 | use super::*; |
| 2 | use crate::task_manager::{TaskManager, TaskManagerConfig}; |
| 3 | |
| 4 | fn fixture_config() -> Config { |
| 5 | let mut config = Config { |
| 6 | api_key: Some("local-fixture-key".into()), |
| 7 | base_url: Some("http://127.0.0.1:1/v1".into()), |
| 8 | ..Config::default() |
| 9 | }; |
| 10 | config.set_feature("mcp", false).unwrap(); |
| 11 | config.set_feature("subagents", false).unwrap(); |
| 12 | config |
| 13 | } |
| 14 | |
| 15 | #[tokio::test] |
| 16 | async fn idle_runtime_engine_cycles_are_drained_before_same_scope_reopen() -> Result<()> { |
| 17 | let _env = crate::test_support::lock_test_env(); |
| 18 | let root = tempfile::tempdir()?; |
| 19 | let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", root.path()); |
| 20 | let cfg = fixture_config(); |
| 21 | let runtime_config = test_manager_config(root.path().join("runtime")); |
| 22 | let runtime = Arc::new(RuntimeThreadManager::open( |
| 23 | cfg.clone(), |
| 24 | root.path().into(), |
| 25 | runtime_config.clone(), |
| 26 | )?); |
| 27 | let tasks = TaskManager::start_with_runtime_manager( |
| 28 | TaskManagerConfig::from_runtime(&cfg, root.path().into(), None, Some(1)), |
| 29 | cfg.clone(), |
| 30 | runtime.clone(), |
| 31 | ) |
| 32 | .await?; |
| 33 | let thread = runtime |
| 34 | .create_thread(CreateThreadRequest::default()) |
| 35 | .await?; |
| 36 | // Load the real production Engine with its actual strong Runtime services. |
| 37 | // No turn is submitted, so constructing this Engine makes no provider call. |
| 38 | let engine = runtime.get_engine(&thread.id).await?; |
| 39 | engine.get_session_snapshot().await?; |
| 40 | runtime.spawn_goal_continuation(thread.id.clone(), 3_600); |
| 41 | let weak = Arc::downgrade(&tasks); |
| 42 | tasks.shutdown_and_wait().await?; |
| 43 | assert!(runtime.active.lock().await.engines.is_empty()); |
| 44 | assert!(runtime.get_engine(&thread.id).await.is_err()); |
| 45 | assert!( |
| 46 | RuntimeThreadManager::open(cfg.clone(), root.path().into(), runtime_config.clone()) |
| 47 | .is_err(), |
| 48 | "retained Runtime handle still owns its scope" |
| 49 | ); |
| 50 | drop(engine); |
| 51 | drop(tasks); |
| 52 | drop(runtime); |
| 53 | assert!( |
| 54 | weak.upgrade().is_none(), |
| 55 | "idle Engine service cycle must have ended" |
| 56 | ); |
| 57 | let reopened = RuntimeThreadManager::open(cfg, root.path().into(), runtime_config)?; |
| 58 | reopened.shutdown_and_wait().await?; |
| 59 | Ok(()) |
| 60 | } |
| 61 | |
| 62 | #[tokio::test] |
| 63 | async fn shutdown_waits_actual_engine_exit_and_terminal_monitor_before_releasing_scope() |
| 64 | -> Result<()> { |
| 65 | use crate::core::engine::Engine; |
| 66 | use crate::llm_client::mock::{MockLlmClient, canned}; |
| 67 | let _env = crate::test_support::lock_test_env(); |
| 68 | let root = tempfile::tempdir()?; |
| 69 | let _home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", root.path()); |
| 70 | let cfg = fixture_config(); |
| 71 | let manager_cfg = test_manager_config(root.path().join("runtime")); |
| 72 | let runtime = Arc::new(RuntimeThreadManager::open( |
| 73 | cfg.clone(), |
| 74 | root.path().into(), |
| 75 | manager_cfg.clone(), |
| 76 | )?); |
| 77 | let tasks = TaskManager::start_with_runtime_manager( |
| 78 | TaskManagerConfig::from_runtime(&cfg, root.path().into(), None, Some(1)), |
| 79 | cfg.clone(), |
| 80 | runtime.clone(), |
| 81 | ) |
| 82 | .await?; |
| 83 | let thread = runtime |
| 84 | .create_thread(CreateThreadRequest::default()) |
| 85 | .await?; |
| 86 | let model = Arc::new(MockLlmClient::new(vec![canned::simple_text_turn( |
| 87 | "owned fixture", |
| 88 | )])); |
| 89 | let (engine, handle) = Engine::new_with_model_client( |
| 90 | EngineConfig { |
| 91 | workspace: root.path().into(), |
| 92 | model: thread.model.clone(), |
| 93 | subagents_enabled: false, |
| 94 | snapshots_enabled: false, |
| 95 | memory_enabled: false, |
| 96 | terminal_chrome_enabled: false, |
| 97 | runtime_services: crate::tools::spec::RuntimeToolServices { |
| 98 | task_manager: Some(tasks.clone()), |
| 99 | active_thread_id: Some(thread.id.clone()), |
| 100 | dynamic_tool_executor: Some(Arc::new(runtime.as_ref().clone())), |
| 101 | ..Default::default() |
| 102 | }, |
| 103 | ..EngineConfig::default() |
| 104 | }, |
| 105 | &cfg, |
| 106 | model.clone(), |
| 107 | ); |
| 108 | runtime |
| 109 | .install_test_engine(&thread.id, handle.clone()) |
| 110 | .await?; |
| 111 | let (release, blocked) = oneshot::channel(); |
| 112 | let worker = tokio::spawn(async move { |
| 113 | blocked.await.unwrap(); |
| 114 | engine.run().await; |
| 115 | }); |
| 116 | runtime |
| 117 | .engine_workers |
| 118 | .lock() |
| 119 | .push((handle.clone(), retained_completion(worker))); |
| 120 | let turn = runtime |
| 121 | .start_turn( |
| 122 | &thread.id, |
| 123 | StartTurnRequest { |
| 124 | prompt: "fixture admission".into(), |
| 125 | ..Default::default() |
| 126 | }, |
| 127 | ) |
| 128 | .await?; |
| 129 | let drain = tokio::spawn({ |
| 130 | let tasks = tasks.clone(); |
| 131 | async move { tasks.shutdown_and_wait().await } |
| 132 | }); |
| 133 | tokio::time::timeout(Duration::from_secs(5), runtime.cancel_token.cancelled()).await?; |
| 134 | assert!( |
| 135 | !drain.is_finished(), |
| 136 | "a cancellation request cannot prove Engine exit" |
| 137 | ); |
| 138 | assert!( |
| 139 | RuntimeThreadManager::open(cfg.clone(), root.path().into(), manager_cfg.clone()).is_err() |
| 140 | ); |
| 141 | assert!( |
| 142 | runtime |
| 143 | .start_turn( |
| 144 | &thread.id, |
| 145 | StartTurnRequest { |
| 146 | prompt: "late admission".into(), |
| 147 | ..Default::default() |
| 148 | } |
| 149 | ) |
| 150 | .await |
| 151 | .is_err() |
| 152 | ); |
| 153 | let completion = runtime.engine_workers.lock()[0].1.clone(); |
| 154 | tokio::time::timeout(Duration::from_secs(5), async { |
| 155 | while completion.try_lock().is_ok() { |
| 156 | sleep(Duration::from_millis(5)).await; |
| 157 | } |
| 158 | }) |
| 159 | .await?; |
| 160 | drain.abort(); |
| 161 | assert!(drain.await.unwrap_err().is_cancelled()); |
| 162 | let retry = tokio::spawn({ |
| 163 | let tasks = tasks.clone(); |
| 164 | async move { tasks.shutdown_and_wait().await } |
| 165 | }); |
| 166 | sleep(Duration::from_millis(25)).await; |
| 167 | assert!( |
| 168 | !retry.is_finished(), |
| 169 | "retry must still own and await the original Engine join" |
| 170 | ); |
| 171 | release.send(()).unwrap(); |
| 172 | tokio::time::timeout(Duration::from_secs(15), retry).await???; |
| 173 | drop(completion); |
| 174 | assert!(runtime.store.load_turn(&turn.id)?.status != RuntimeTurnStatus::InProgress); |
| 175 | drop(handle); |
| 176 | drop(tasks); |
| 177 | drop(runtime); |
| 178 | let reopened = RuntimeThreadManager::open(cfg, root.path().into(), manager_cfg)?; |
| 179 | reopened.shutdown_and_wait().await?; |
| 180 | Ok(()) |
| 181 | } |
| 182 | |
| 183 | #[tokio::test] |
| 184 | async fn shutdown_drains_accepted_user_input_receipt_after_caller_disconnects() -> Result<()> { |
| 185 | let root = tempfile::tempdir()?; |
| 186 | let runtime = Arc::new(test_manager(root.path().join("runtime"))?); |
| 187 | let thread = runtime |
| 188 | .create_thread(CreateThreadRequest::default()) |
| 189 | .await?; |
| 190 | let harness = mock_engine_handle(); |
| 191 | runtime |
| 192 | .install_test_engine(&thread.id, harness.handle.clone()) |
| 193 | .await?; |
| 194 | runtime.register_pending_user_input( |
| 195 | &thread.id, |
| 196 | PendingUserInputRequest { |
| 197 | id: "input_drain".into(), |
| 198 | turn_id: "turn_drain".into(), |
| 199 | request: crate::tools::user_input::UserInputRequest { |
| 200 | questions: Vec::new(), |
| 201 | }, |
| 202 | }, |
| 203 | ); |
| 204 | let hold_receipt = runtime.event_emit.lock().await; |
| 205 | let submission = tokio::spawn({ |
| 206 | let runtime = runtime.clone(); |
| 207 | let thread_id = thread.id.clone(); |
| 208 | async move { |
| 209 | runtime |
| 210 | .submit_user_input( |
| 211 | &thread_id, |
| 212 | "input_drain", |
| 213 | crate::tools::user_input::UserInputResponse { |
| 214 | answers: Vec::new(), |
| 215 | }, |
| 216 | ) |
| 217 | .await |
| 218 | } |
| 219 | }); |
| 220 | tokio::time::timeout(Duration::from_secs(5), async { |
| 221 | while runtime.turn_monitors.lock().is_empty() { |
| 222 | sleep(Duration::from_millis(5)).await; |
| 223 | } |
| 224 | }) |
| 225 | .await?; |
| 226 | submission.abort(); |
| 227 | assert!(submission.await.unwrap_err().is_cancelled()); |
| 228 | let drain = tokio::spawn({ |
| 229 | let runtime = runtime.clone(); |
| 230 | async move { runtime.shutdown_and_wait().await } |
| 231 | }); |
| 232 | tokio::time::timeout(Duration::from_secs(5), runtime.cancel_token.cancelled()).await?; |
| 233 | sleep(Duration::from_millis(25)).await; |
| 234 | assert!( |
| 235 | !drain.is_finished(), |
| 236 | "accepted detached receipt is still part of shutdown" |
| 237 | ); |
| 238 | drop(hold_receipt); |
| 239 | tokio::time::timeout(Duration::from_secs(5), drain).await???; |
| 240 | let events = runtime.events_since(&thread.id, None)?; |
| 241 | assert_eq!( |
| 242 | events |
| 243 | .iter() |
| 244 | .filter(|event| event.event == "user_input.answered") |
| 245 | .count(), |
| 246 | 1 |
| 247 | ); |
| 248 | assert!(runtime.pending_user_inputs.lock().is_empty()); |
| 249 | Ok(()) |
| 250 | } |
| 251 | |
| 252 | #[tokio::test] |
| 253 | async fn shutdown_fences_recovery_readers_without_publishing_late_receipts() -> Result<()> { |
| 254 | let root = tempfile::tempdir()?; |
| 255 | let runtime = Arc::new(test_manager(root.path().join("runtime"))?); |
| 256 | let thread = runtime |
| 257 | .create_thread(CreateThreadRequest::default()) |
| 258 | .await?; |
| 259 | let turn = sample_turn(&thread.id, "turn_recovery_drain", RuntimeTurnStatus::Failed); |
| 260 | runtime.store.save_turn(&turn)?; |
| 261 | runtime.queue_recovery_receipt(RecoveredTurnReceipt { |
| 262 | turn, |
| 263 | unresolved_dynamic_tools: Vec::new(), |
| 264 | }); |
| 265 | let hold = runtime.recovery_flush.lock().await; |
| 266 | let reader = tokio::spawn({ |
| 267 | let runtime = runtime.clone(); |
| 268 | let id = thread.id.clone(); |
| 269 | async move { runtime.get_thread(&id).await } |
| 270 | }); |
| 271 | tokio::task::yield_now().await; |
| 272 | let shutdown = tokio::spawn({ |
| 273 | let runtime = runtime.clone(); |
| 274 | async move { runtime.shutdown_and_wait().await } |
| 275 | }); |
| 276 | tokio::time::timeout(Duration::from_secs(5), runtime.cancel_token.cancelled()).await?; |
| 277 | assert!( |
| 278 | !shutdown.is_finished(), |
| 279 | "shutdown must fence pending recovery producers" |
| 280 | ); |
| 281 | drop(hold); |
| 282 | assert!( |
| 283 | tokio::time::timeout(Duration::from_secs(5), reader) |
| 284 | .await?? |
| 285 | .is_err() |
| 286 | ); |
| 287 | tokio::time::timeout(Duration::from_secs(5), shutdown).await???; |
| 288 | assert!( |
| 289 | runtime.recovery_receipts.lock().contains_key(&thread.id), |
| 290 | "unadmitted recovery remains queued for the next owner" |
| 291 | ); |
| 292 | assert!( |
| 293 | runtime |
| 294 | .events_since(&thread.id, None)? |
| 295 | .iter() |
| 296 | .all(|event| event.event != "turn.completed") |
| 297 | ); |
| 298 | assert!(runtime.get_thread_detail(&thread.id).await.is_err()); |
| 299 | Ok(()) |
| 300 | } |
| 301 | |
| 302 | #[tokio::test] |
| 303 | async fn shutdown_drains_an_already_started_recovery_flush() -> Result<()> { |
| 304 | let root = tempfile::tempdir()?; |
| 305 | let runtime = Arc::new(test_manager(root.path().join("runtime"))?); |
| 306 | let thread = runtime |
| 307 | .create_thread(CreateThreadRequest::default()) |
| 308 | .await?; |
| 309 | let turn = sample_turn( |
| 310 | &thread.id, |
| 311 | "turn_started_recovery", |
| 312 | RuntimeTurnStatus::Failed, |
| 313 | ); |
| 314 | runtime.store.save_turn(&turn)?; |
| 315 | runtime.queue_recovery_receipt(RecoveredTurnReceipt { |
| 316 | turn, |
| 317 | unresolved_dynamic_tools: Vec::new(), |
| 318 | }); |
| 319 | let hold = runtime.event_emit.lock().await; |
| 320 | let reader = tokio::spawn({ |
| 321 | let runtime = runtime.clone(); |
| 322 | let id = thread.id.clone(); |
| 323 | async move { runtime.get_thread(&id).await } |
| 324 | }); |
| 325 | tokio::time::timeout(Duration::from_secs(5), async { |
| 326 | while runtime.recovery_flush.try_lock().is_ok() { |
| 327 | sleep(Duration::from_millis(5)).await; |
| 328 | } |
| 329 | }) |
| 330 | .await?; |
| 331 | let shutdown = tokio::spawn({ |
| 332 | let runtime = runtime.clone(); |
| 333 | async move { runtime.shutdown_and_wait().await } |
| 334 | }); |
| 335 | tokio::time::timeout(Duration::from_secs(5), runtime.cancel_token.cancelled()).await?; |
| 336 | sleep(Duration::from_millis(25)).await; |
| 337 | assert!( |
| 338 | !shutdown.is_finished(), |
| 339 | "accepted recovery write must settle before drain succeeds" |
| 340 | ); |
| 341 | drop(hold); |
| 342 | tokio::time::timeout(Duration::from_secs(5), reader).await???; |
| 343 | tokio::time::timeout(Duration::from_secs(5), shutdown).await???; |
| 344 | assert!(!runtime.recovery_receipts.lock().contains_key(&thread.id)); |
| 345 | assert_eq!( |
| 346 | runtime |
| 347 | .events_since(&thread.id, None)? |
| 348 | .iter() |
| 349 | .filter(|event| event.event == "turn.completed") |
| 350 | .count(), |
| 351 | 1 |
| 352 | ); |
| 353 | Ok(()) |
| 354 | } |
| 355 |