tests.rs - CodeWhale

tests.rs

1	use super::*;
2	use crate::core::engine::{MockApprovalEvent, mock_engine_handle};
3	use crate::core::events::{Event as EngineEvent, TurnOutcomeStatus};
4	use std::time::{Duration, Instant};
5	use tokio::sync::{mpsc, oneshot};
6	use tokio::time::sleep;
7	use uuid::Uuid;
8
9	fn test_runtime_dir() -> PathBuf {
10	std::env::temp_dir().join(format!("deepseek-runtime-threads-{}", Uuid::new_v4()))
11	}
12
13	const EVENT_PROCESS_ROLE_ENV: &str = "CODEWHALE_TEST_EVENT_PROCESS_ROLE";
14	const EVENT_PROCESS_ROOT_ENV: &str = "CODEWHALE_TEST_EVENT_PROCESS_ROOT";
15	const EVENT_PROCESS_THREAD_ENV: &str = "CODEWHALE_TEST_EVENT_PROCESS_THREAD";
16	const EVENT_PROCESS_SIGNAL_ENV: &str = "CODEWHALE_TEST_EVENT_PROCESS_SIGNAL";
17	const EVENT_PROCESS_START_ENV: &str = "CODEWHALE_TEST_EVENT_PROCESS_START";
18	const EVENT_PROCESS_WORKER_ENV: &str = "CODEWHALE_TEST_EVENT_PROCESS_WORKER";
19	const EVENT_PROCESS_COUNT_ENV: &str = "CODEWHALE_TEST_EVENT_PROCESS_COUNT";
20	const EVENT_PROCESS_HELPER: &str = "runtime_threads::tests::runtime_event_process_child_helper";
21
22	#[test]
23	#[ignore = "spawned by real cross-process Runtime event tests"]
24	fn runtime_event_process_child_helper() {
25	let Ok(role) = std::env::var(EVENT_PROCESS_ROLE_ENV) else {
26	return;
27	};
28	let root = PathBuf::from(
29	std::env::var_os(EVENT_PROCESS_ROOT_ENV).expect("event child needs a store root"),
30	);
31	let thread_id = std::env::var(EVENT_PROCESS_THREAD_ENV).expect("event child needs a thread id");
32	let signal = PathBuf::from(
33	std::env::var_os(EVENT_PROCESS_SIGNAL_ENV).expect("event child needs a signal path"),
34	);
35	let store = RuntimeThreadStore::open(root).expect("event child opens Runtime store");
36
37	match role.as_str() {
38	"writer" => {
39	let start = PathBuf::from(
40	std::env::var_os(EVENT_PROCESS_START_ENV)
41	.expect("event writer needs a start barrier"),
42	);
43	let worker =
44	std::env::var(EVENT_PROCESS_WORKER_ENV).expect("event writer needs a worker id");
45	let count = std::env::var(EVENT_PROCESS_COUNT_ENV)
46	.expect("event writer needs a count")
47	.parse::<u64>()
48	.expect("event writer count must be numeric");
49	std::fs::write(&signal, b"ready").expect("announce ready event writer");
50	wait_for_runtime_event_test_file(&start, "writer start barrier");
51	let runtime = tokio::runtime::Builder::new_current_thread()
52	.enable_all()
53	.build()
54	.expect("build event writer runtime");
55	runtime.block_on(async {
56	for index in 0..count {
57	store
58	.append_event(
59	&thread_id,
60	None,
61	None,
62	"process.event",
63	json!({ "worker": worker, "index": index }),
64	)
65	.await
66	.expect("append cross-process Runtime event");
67	}
68	});
69	}
70	"holder" => {
71	store
72	.with_event_transaction(Duration::from_secs(5), \|\| {
73	std::fs::write(&signal, b"locked").expect("announce held event lock");
74	std::thread::sleep(Duration::from_secs(60));
75	Ok(())
76	})
77	.expect("hold Runtime event transaction");
78	}
79	"torn" => {
80	store
81	.with_event_transaction(Duration::from_secs(5), \|\| {
82	let mut state = load_runtime_store_state(&store.state_path)?;
83	let seq = state.next_seq;
84	state.next_seq = seq.checked_add(1).context("reserve torn event sequence")?;
85	write_json_atomic(&store.state_path, &state)?;
86	let path = store.events_path(&thread_id)?;
87	let mut file =
88	open_runtime_store_file(&path, "torn Runtime event fixture", \|options\| {
89	options.create(true).append(true);
90	})?;
91	let record = RuntimeEventRecord {
92	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
93	seq,
94	timestamp: Utc::now(),
95	thread_id: thread_id.clone(),
96	turn_id: None,
97	item_id: None,
98	event: "torn.process.event".to_string(),
99	payload: json!({}),
100	};
101	file.write_all(&serde_json::to_vec(&record)?)?;
102	file.flush()?;
103	file.sync_all()?;
104	std::fs::write(&signal, b"torn").expect("announce torn event append");
105	std::thread::sleep(Duration::from_secs(60));
106	Ok(())
107	})
108	.expect("hold torn Runtime event transaction");
109	}
110	"phantom" => {
111	store
112	.with_event_transaction(Duration::from_secs(5), \|\| {
113	let path = store.events_path(&thread_id)?;
114	let mut append = open_runtime_store_file(
115	&path,
116	"phantom Runtime event fixture",
117	\|options\| {
118	options.create(true).append(true);
119	},
120	)?;
121	let rollback = open_runtime_store_file(
122	&path,
123	"phantom Runtime event rollback",
124	\|options\| {
125	options.write(true);
126	},
127	)?;
128	validate_same_runtime_store_file_handles(&append, &rollback, &path)?;
129	let original_len = append.metadata()?.len();
130	serde_json::to_writer(
131	&mut append,
132	&RuntimeEventRecord {
133	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
134	seq: 999,
135	timestamp: Utc::now(),
136	thread_id: thread_id.clone(),
137	turn_id: None,
138	item_id: None,
139	event: "phantom.process.event".to_string(),
140	payload: json!({}),
141	},
142	)?;
143	append.write_all(b"\n")?;
144	append.flush()?;
145	std::fs::write(&signal, b"visible").expect("announce rollback candidate");
146	std::thread::sleep(Duration::from_millis(300));
147	rollback_failed_event_append_handle(&rollback, original_len)
148	})
149	.expect("roll back phantom Runtime event");
150	}
151	other => panic!("unknown Runtime event child role {other}"),
152	}
153	}
154
155	fn wait_for_runtime_event_test_file(path: &Path, label: &str) {
156	let deadline = Instant::now() + Duration::from_secs(30);
157	while !path.exists() {
158	assert!(
159	Instant::now() < deadline,
160	"timed out waiting for {label} at {}",
161	path.display()
162	);
163	std::thread::sleep(Duration::from_millis(5));
164	}
165	}
166
167	fn spawn_runtime_event_child(
168	role: &str,
169	root: &Path,
170	thread_id: &str,
171	signal: &Path,
172	) -> RuntimeEventChildGuard {
173	let mut command = std::process::Command::new(
174	std::env::current_exe().expect("current test executable for Runtime event child"),
175	);
176	command
177	.arg(EVENT_PROCESS_HELPER)
178	.args(["--exact", "--ignored", "--test-threads", "1"])
179	.env(EVENT_PROCESS_ROLE_ENV, role)
180	.env(EVENT_PROCESS_ROOT_ENV, root)
181	.env(EVENT_PROCESS_THREAD_ENV, thread_id)
182	.env(EVENT_PROCESS_SIGNAL_ENV, signal)
183	.env(EVENT_PROCESS_START_ENV, root.join("process-writers.start"))
184	.env(EVENT_PROCESS_WORKER_ENV, signal.as_os_str())
185	.env(EVENT_PROCESS_COUNT_ENV, "8")
186	.stdin(std::process::Stdio::null())
187	.stdout(std::process::Stdio::null())
188	.stderr(std::process::Stdio::null());
189	RuntimeEventChildGuard::new(command.spawn().expect("spawn Runtime event child"))
190	}
191
192	struct RuntimeEventChildGuard(Option<std::process::Child>);
193
194	impl RuntimeEventChildGuard {
195	const fn new(child: std::process::Child) -> Self {
196	Self(Some(child))
197	}
198
199	fn wait_success(mut self, label: &str) {
200	use wait_timeout::ChildExt as _;
201
202	let outcome = self
203	.0
204	.as_mut()
205	.expect("Runtime event child is present")
206	.wait_timeout(Duration::from_secs(30));
207	match outcome {
208	Ok(Some(status)) => {
209	self.0.take();
210	assert!(status.success(), "{label} failed with {status}");
211	}
212	Ok(None) => panic!("timed out waiting for {label}"),
213	Err(error) => panic!("failed waiting for {label}: {error}"),
214	}
215	}
216
217	fn kill_and_reap(mut self) -> std::io::Result<std::process::ExitStatus> {
218	let child = self.0.as_mut().expect("Runtime event child is present");
219	let kill = child.kill();
220	let status = child.wait();
221	if status.is_ok() {
222	self.0.take();
223	}
224	kill.and(status)
225	}
226	}
227
228	impl Drop for RuntimeEventChildGuard {
229	fn drop(&mut self) {
230	if let Some(child) = self.0.as_mut() {
231	let _ = child.kill();
232	let _ = child.wait();
233	}
234	}
235	}
236
237	fn test_manager_config(data_dir: PathBuf) -> RuntimeThreadManagerConfig {
238	RuntimeThreadManagerConfig {
239	task_data_dir: data_dir.clone(),
240	data_dir,
241	max_active_threads: 4,
242	}
243	}
244
245	fn test_manager(data_dir: PathBuf) -> Result<RuntimeThreadManager> {
246	RuntimeThreadManager::open(
247	Config::default(),
248	PathBuf::from("."),
249	test_manager_config(data_dir),
250	)
251	}
252
253	struct ApprovalTimeoutGuard {
254	previous_ms: u64,
255	}
256
257	impl Drop for ApprovalTimeoutGuard {
258	fn drop(&mut self) {
259	set_test_approval_decision_timeout_ms(self.previous_ms);
260	}
261	}
262
263	fn test_approval_timeout_ms(ms: u64) -> ApprovalTimeoutGuard {
264	ApprovalTimeoutGuard {
265	previous_ms: set_test_approval_decision_timeout_ms(ms),
266	}
267	}
268
269	struct DynamicToolTimeoutGuard {
270	previous_ms: u64,
271	}
272
273	impl Drop for DynamicToolTimeoutGuard {
274	fn drop(&mut self) {
275	set_test_dynamic_tool_result_timeout_ms(self.previous_ms);
276	}
277	}
278
279	fn test_dynamic_tool_timeout_ms(ms: u64) -> DynamicToolTimeoutGuard {
280	DynamicToolTimeoutGuard {
281	previous_ms: set_test_dynamic_tool_result_timeout_ms(ms),
282	}
283	}
284
285	struct EventAppendFaultGuard {
286	restore: Option<EventAppendTestFaultRestore>,
287	}
288
289	impl EventAppendFaultGuard {
290	fn arm(thread_id: &str, fault: EventAppendTestFault) -> Self {
291	Self::arm_repeated(thread_id, fault, 1)
292	}
293
294	fn arm_repeated(thread_id: &str, fault: EventAppendTestFault, count: usize) -> Self {
295	Self {
296	restore: Some(set_test_event_append_fault(thread_id, fault, count)),
297	}
298	}
299	}
300
301	impl Drop for EventAppendFaultGuard {
302	fn drop(&mut self) {
303	if let Some(restore) = self.restore.take() {
304	restore_test_event_append_fault(restore);
305	}
306	}
307	}
308
309	fn sample_thread(thread_id: &str) -> ThreadRecord {
310	let now = Utc::now();
311	ThreadRecord {
312	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
313	id: thread_id.to_string(),
314	created_at: now,
315	updated_at: now,
316	model: DEFAULT_TEXT_MODEL.to_string(),
317	model_provider: None,
318	model_provider_id: None,
319	workspace: PathBuf::from("."),
320	mode: AppMode::Agent.as_setting().to_string(),
321	permission_posture: Some("ask".to_string()),
322	allow_shell: false,
323	trust_mode: false,
324	auto_approve: false,
325	latest_turn_id: None,
326	latest_response_bookmark: None,
327	archived: false,
328	system_prompt: None,
329	task_id: None,
330	title: None,
331	session_id: None,
332	}
333	}
334
335	fn sample_turn(thread_id: &str, turn_id: &str, status: RuntimeTurnStatus) -> TurnRecord {
336	let now = Utc::now();
337	TurnRecord {
338	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
339	id: turn_id.to_string(),
340	thread_id: thread_id.to_string(),
341	status,
342	input_summary: "sample".to_string(),
343	created_at: now,
344	started_at: Some(now),
345	ended_at: None,
346	duration_ms: None,
347	usage: None,
348	permission_posture: None,
349	effective_provider: None,
350	effective_provider_id: None,
351	effective_billing_surface: None,
352	effective_endpoint_fingerprint: None,
353	effective_billing_mode: None,
354	effective_dispatched_at: None,
355	effective_model: None,
356	routed_usage: Vec::new(),
357	routed_usage_source_ids: Vec::new(),
358	routed_usage_dropped_records: 0,
359	error: None,
360	item_ids: Vec::new(),
361	steer_count: 0,
362	}
363	}
364
365	fn set_test_turn_route(
366	turn: &mut TurnRecord,
367	provider: ApiProvider,
368	provider_identity: &str,
369	model: &str,
370	billing_surface: Option<&str>,
371	billing_mode: crate::cost_status::RouteBillingMode,
372	) {
373	turn.persist_effective_route(&crate::cost_status::EffectiveRouteEnvelope {
374	provider,
375	provider_identity: provider_identity.to_string(),
376	model: model.to_string(),
377	billing_surface: billing_surface.map(str::to_string),
378	endpoint_fingerprint: None,
379	billing_mode,
380	dispatched_at: turn.created_at,
381	});
382	}
383
384	#[test]
385	fn runtime_compaction_uses_provider_route_context() {
386	let limits = codewhale_config::route::RouteLimits {
387	context_tokens: Some(272_000),
388	input_tokens: None,
389	output_tokens: None,
390	};
391	let config = runtime_compaction_config(
392	ApiProvider::OpenaiCodex,
393	"gpt-5.5",
394	Some(limits),
395	false,
396	false,
397	80.0,
398	);
399
400	assert!(config.enabled);
401	// The threshold is 80% of the route's spendable input budget after
402	// output reservation and headroom, not 80% of the raw context window.
403	assert_eq!(config.token_threshold, 213_504);
404	assert_eq!(config.effective_context_window, Some(272_000));
405	}
406
407	#[test]
408	fn legacy_turn_record_has_no_invented_route_provenance() {
409	let turn = sample_turn("thr_legacy", "turn_legacy", RuntimeTurnStatus::Completed);
410	let mut value = serde_json::to_value(turn).expect("serialize turn");
411	let object = value.as_object_mut().expect("turn object");
412	object.remove("effective_provider");
413	object.remove("effective_provider_id");
414	object.remove("effective_billing_surface");
415	object.remove("effective_endpoint_fingerprint");
416	object.remove("effective_billing_mode");
417	object.remove("effective_dispatched_at");
418	object.remove("effective_model");
419	object.remove("routed_usage");
420
421	let restored: TurnRecord = serde_json::from_value(value).expect("deserialize legacy turn");
422	assert_eq!(restored.effective_provider, None);
423	assert_eq!(restored.effective_billing_surface, None);
424	assert_eq!(restored.effective_billing_mode, None);
425	assert_eq!(restored.effective_dispatched_at, None);
426	assert_eq!(restored.effective_model, None);
427	assert!(restored.routed_usage.is_empty());
428	}
429
430	#[tokio::test]
431	async fn named_custom_thread_identity_round_trips_and_fails_closed_when_removed() -> Result<()> {
432	let mut custom = std::collections::HashMap::new();
433	custom.insert(
434	"lm-studio".to_string(),
435	crate::config::ProviderConfig {
436	kind: Some("openai-compatible".to_string()),
437	base_url: Some("http://127.0.0.1:1234/v1".to_string()),
438	model: Some("local-default".to_string()),
439	..crate::config::ProviderConfig::default()
440	},
441	);
442	let config = Config {
443	provider: Some("lm-studio".to_string()),
444	providers: Some(crate::config::ProvidersConfig {
445	custom,
446	..crate::config::ProvidersConfig::default()
447	}),
448	..Config::default()
449	};
450	let manager = RuntimeThreadManager::open(
451	config.clone(),
452	PathBuf::from("."),
453	test_manager_config(test_runtime_dir()),
454	)?;
455
456	let thread = manager
457	.create_thread(CreateThreadRequest {
458	model: Some("local-code-model".to_string()),
459	model_provider: Some("lm-studio".to_string()),
460	..CreateThreadRequest::default()
461	})
462	.await?;
463	let persisted = manager.get_thread(&thread.id).await?;
464	assert_eq!(persisted.model_provider.as_deref(), Some("custom"));
465	assert_eq!(persisted.model_provider_id.as_deref(), Some("lm-studio"));
466	let serialized = serde_json::to_string(&persisted)?;
467	assert!(serialized.contains("\"model_provider\":\"custom\""));
468	assert!(serialized.contains("\"model_provider_id\":\"lm-studio\""));
469	assert!(!serialized.contains("127.0.0.1:1234"));
470
471	let route = manager.resolved_route_for_thread(&config, &persisted)?;
472	assert_eq!(route.identity.provider, ApiProvider::Custom);
473	assert_eq!(route.identity.key, "lm-studio");
474	assert_eq!(route.model, "local-code-model");
475	assert_eq!(route.config.deepseek_base_url(), "http://127.0.0.1:1234/v1");
476
477	let err = manager
478	.resolved_route_for_thread(&Config::default(), &persisted)
479	.expect_err("removed provider must fail closed");
480	let message = err.to_string();
481	assert!(message.contains("[providers.lm-studio]"), "{message}");
482	assert!(message.contains("will not fall back"), "{message}");
483
484	let mut legacy_value = serde_json::to_value(&persisted)?;
485	legacy_value
486	.as_object_mut()
487	.expect("thread object")
488	.remove("model_provider");
489	legacy_value
490	.as_object_mut()
491	.expect("thread object")
492	.remove("model_provider_id");
493	let legacy: ThreadRecord = serde_json::from_value(legacy_value)?;
494	assert_eq!(legacy.model_provider, None);
495	Ok(())
496	}
497
498	#[test]
499	fn legacy_literal_custom_thread_resume_requires_and_keeps_root_route() -> Result<()> {
500	let config = Config {
501	provider: Some("custom".to_string()),
502	base_url: Some("http://127.0.0.1:18180/v1".to_string()),
503	default_text_model: Some("legacy-default-model".to_string()),
504	..Config::default()
505	};
506	let manager = RuntimeThreadManager::open(
507	config.clone(),
508	PathBuf::from("."),
509	test_manager_config(test_runtime_dir()),
510	)?;
511	let mut persisted = sample_thread("thr_legacy_custom");
512	persisted.model = "legacy-saved-model".to_string();
513	persisted.model_provider = Some("custom".to_string());
514	let restored: ThreadRecord = serde_json::from_str(&serde_json::to_string(&persisted)?)?;
515
516	let route = manager.resolved_route_for_thread(&config, &restored)?;
517	assert_eq!(route.identity.provider, ApiProvider::Custom);
518	assert_eq!(route.identity.key, "custom");
519	assert_eq!(route.model, "legacy-saved-model");
520	assert_eq!(
521	route.config.deepseek_base_url(),
522	"http://127.0.0.1:18180/v1"
523	);
524	assert!(
525	route
526	.config
527	.providers
528	.as_ref()
529	.is_none_or(\|providers\| !providers.custom.contains_key("custom")),
530	"route resolution must not synthesize an ambiguous [providers.custom] table"
531	);
532	assert_eq!(
533	route
534	.config
535	.resolve_provider_identity("custom")
536	.map_err(anyhow::Error::msg)?,
537	crate::config::ProviderIdentity {
538	provider: ApiProvider::Custom,
539	key: "custom".to_string(),
540	exact_id: None,
541	}
542	);
543	let repeated = manager.resolved_route_for_thread(&route.config, &restored)?;
544	assert_eq!(repeated.identity.key, "custom");
545	assert_eq!(repeated.model, "legacy-saved-model");
546	assert_eq!(
547	repeated.config.deepseek_base_url(),
548	"http://127.0.0.1:18180/v1"
549	);
550
551	let named_config = {
552	let mut custom = std::collections::HashMap::new();
553	custom.insert(
554	"lm-studio".to_string(),
555	crate::config::ProviderConfig {
556	kind: Some("openai-compatible".to_string()),
557	base_url: Some("http://127.0.0.1:18181/v1".to_string()),
558	model: Some("named-model".to_string()),
559	..crate::config::ProviderConfig::default()
560	},
561	);
562	Config {
563	provider: Some("lm-studio".to_string()),
564	providers: Some(crate::config::ProvidersConfig {
565	custom,
566	..crate::config::ProvidersConfig::default()
567	}),
568	..Config::default()
569	}
570	};
571	let error = manager
572	.resolved_route_for_thread(&named_config, &restored)
573	.expect_err("id-less root record must not migrate to a named table")
574	.to_string();
575	assert!(error.contains("root-level"), "{error}");
576	assert!(error.contains("will not guess or fall back"), "{error}");
577
578	Ok(())
579	}
580
581	#[tokio::test]
582	async fn root_custom_thread_and_turn_writers_omit_exact_id() -> Result<()> {
583	let config = Config {
584	provider: Some("custom".to_string()),
585	base_url: Some("http://127.0.0.1:18180/v1".to_string()),
586	default_text_model: Some("legacy-root-model".to_string()),
587	..Config::default()
588	};
589	let manager = RuntimeThreadManager::open(
590	config,
591	PathBuf::from("."),
592	test_manager_config(test_runtime_dir()),
593	)?;
594	let thread = manager
595	.create_thread(CreateThreadRequest {
596	model: Some("legacy-root-model".to_string()),
597	..CreateThreadRequest::default()
598	})
599	.await?;
600	assert_eq!(thread.model_provider.as_deref(), Some("custom"));
601	assert_eq!(thread.model_provider_id, None);
602	assert!(!serde_json::to_string(&thread)?.contains("model_provider_id"));
603
604	let mut harness = install_mock_engine(&manager, &thread.id).await;
605	let turn = manager
606	.start_turn(
607	&thread.id,
608	StartTurnRequest {
609	prompt: "keep the root route".to_string(),
610	..StartTurnRequest::default()
611	},
612	)
613	.await?;
614	assert_eq!(turn.effective_provider.as_deref(), Some("custom"));
615	assert_eq!(turn.effective_provider_id, None);
616	assert!(!serde_json::to_string(&turn)?.contains("effective_provider_id"));
617	match harness.rx_op.recv().await {
618	Some(Op::SendMessage { route, .. }) => {
619	assert_eq!(route.identity.key, "custom");
620	assert_eq!(route.identity.exact_id, None);
621	assert_eq!(
622	route.config.deepseek_base_url(),
623	"http://127.0.0.1:18180/v1"
624	);
625	}
626	other => panic!("expected root custom send, got {other:?}"),
627	}
628	Ok(())
629	}
630
631	#[tokio::test]
632	async fn real_turn_client_preflight_failure_writes_no_in_progress_record() -> Result<()> {
633	let mut custom = std::collections::HashMap::new();
634	custom.insert(
635	"preflight-failure".to_string(),
636	crate::config::ProviderConfig {
637	kind: Some("openai-compatible".to_string()),
638	base_url: Some("https://preflight.invalid/v1".to_string()),
639	model: Some("preflight-model".to_string()),
640	api_key: Some("test-key".to_string()),
641	// Client construction rejects this independently of ambient auth,
642	// keeping the async regression hermetic without a global env lock.
643	insecure_skip_tls_verify: Some(true),
644	..crate::config::ProviderConfig::default()
645	},
646	);
647	let manager = RuntimeThreadManager::open(
648	Config {
649	provider: Some("preflight-failure".to_string()),
650	providers: Some(crate::config::ProvidersConfig {
651	custom,
652	..crate::config::ProvidersConfig::default()
653	}),
654	..Config::default()
655	},
656	PathBuf::from("."),
657	test_manager_config(test_runtime_dir()),
658	)?;
659	let thread = manager
660	.create_thread(CreateThreadRequest::default())
661	.await?;
662
663	let error = manager
664	.start_turn(
665	&thread.id,
666	StartTurnRequest {
667	prompt: "must not become a zombie turn".to_string(),
668	..StartTurnRequest::default()
669	},
670	)
671	.await
672	.expect_err("missing credentials must fail before turn persistence")
673	.to_string();
674
675	assert!(
676	error.contains("TLS certificate verification cannot be disabled"),
677	"{error}"
678	);
679	assert!(manager.store.list_turns_for_thread(&thread.id)?.is_empty());
680	assert_eq!(manager.get_thread(&thread.id).await?.latest_turn_id, None);
681	Ok(())
682	}
683
684	#[tokio::test]
685	async fn closed_turn_mailbox_rolls_back_durable_records_and_active_claim() -> Result<()> {
686	let manager = test_manager(test_runtime_dir())?;
687	let thread = manager
688	.create_thread(CreateThreadRequest::default())
689	.await?;
690	let harness = install_mock_engine(&manager, &thread.id).await;
691	let before_active = {
692	let active = manager.active.lock().await;
693	let state = active.engines.get(&thread.id).expect("installed engine");
694	(
695	state.active_turn.as_ref().map(\|turn\| turn.turn_id.clone()),
696	state.route_identity.clone(),
697	state.route_model.clone(),
698	active.lru.clone(),
699	)
700	};
701	let before_thread = serde_json::to_value(manager.get_thread(&thread.id).await?)?;
702	let before_events = serde_json::to_value(manager.events_since(&thread.id, None)?)?;
703	drop(harness.rx_op);
704
705	let error = manager
706	.start_turn(
707	&thread.id,
708	StartTurnRequest {
709	prompt: "mailbox is already closed".to_string(),
710	..StartTurnRequest::default()
711	},
712	)
713	.await
714	.expect_err("closed mailbox must reject the turn")
715	.to_string();
716	assert!(error.contains("Failed to start turn"), "{error}");
717
718	assert!(manager.store.list_turns_for_thread(&thread.id)?.is_empty());
719	assert_eq!(
720	serde_json::to_value(manager.get_thread(&thread.id).await?)?,
721	before_thread
722	);
723	assert_eq!(
724	serde_json::to_value(manager.events_since(&thread.id, None)?)?,
725	before_events
726	);
727	assert_eq!(
728	std::fs::read_dir(&manager.store.items_dir)?.count(),
729	0,
730	"failed send must remove the optimistic user item"
731	);
732	let after_active = {
733	let active = manager.active.lock().await;
734	let state = active.engines.get(&thread.id).expect("installed engine");
735	(
736	state.active_turn.as_ref().map(\|turn\| turn.turn_id.clone()),
737	state.route_identity.clone(),
738	state.route_model.clone(),
739	active.lru.clone(),
740	)
741	};
742	assert_eq!(after_active, before_active);
743	Ok(())
744	}
745
746	#[tokio::test]
747	async fn cancellation_while_waiting_for_mailbox_capacity_claims_nothing() -> Result<()> {
748	let manager = test_manager(test_runtime_dir())?;
749	let thread = manager
750	.create_thread(CreateThreadRequest::default())
751	.await?;
752	let mut harness = install_mock_engine(&manager, &thread.id).await;
753
754	for _ in 0..32 {
755	harness.handle.try_send(Op::ListSubAgents)?;
756	}
757	let start_sender_count = harness.handle.tx_op.strong_count();
758	let start_manager = manager.clone();
759	let start_thread_id = thread.id.clone();
760	let start_task = tokio::spawn(async move {
761	start_manager
762	.start_turn(
763	&start_thread_id,
764	StartTurnRequest {
765	prompt: "cancel before mailbox capacity".to_string(),
766	..StartTurnRequest::default()
767	},
768	)
769	.await
770	});
771	wait_for_sender_strong_count(&harness.handle.tx_op, start_sender_count + 2).await?;
772	assert!(
773	!start_task.is_finished(),
774	"start should be waiting for capacity"
775	);
776	assert!(manager.store.list_turns_for_thread(&thread.id)?.is_empty());
777	assert_eq!(manager.get_thread(&thread.id).await?.latest_turn_id, None);
778	assert_eq!(manager.active_turn_flags(&thread.id, "missing").await, None);
779	start_task.abort();
780	let _ = start_task.await;
781	for _ in 0..32 {
782	assert!(matches!(
783	harness.rx_op.recv().await,
784	Some(Op::ListSubAgents)
785	));
786	}
787
788	for _ in 0..32 {
789	harness.handle.try_send(Op::ListSubAgents)?;
790	}
791	let compact_sender_count = harness.handle.tx_op.strong_count();
792	let compact_manager = manager.clone();
793	let compact_thread_id = thread.id.clone();
794	let compact_task = tokio::spawn(async move {
795	compact_manager
796	.compact_thread(&compact_thread_id, CompactThreadRequest::default())
797	.await
798	});
799	wait_for_sender_strong_count(&harness.handle.tx_op, compact_sender_count + 2).await?;
800	assert!(
801	!compact_task.is_finished(),
802	"compaction should be waiting for capacity"
803	);
804	assert!(manager.store.list_turns_for_thread(&thread.id)?.is_empty());
805	assert_eq!(manager.get_thread(&thread.id).await?.latest_turn_id, None);
806	compact_task.abort();
807	let _ = compact_task.await;
808	for _ in 0..32 {
809	assert!(matches!(
810	harness.rx_op.recv().await,
811	Some(Op::ListSubAgents)
812	));
813	}
814	Ok(())
815	}
816
817	#[tokio::test]
818	async fn caller_cancellation_after_engine_acceptance_keeps_owned_turn_lifecycle() -> Result<()> {
819	let manager = test_manager(test_runtime_dir())?;
820	let thread = manager
821	.create_thread(CreateThreadRequest::default())
822	.await?;
823	let mut harness = install_mock_engine(&manager, &thread.id).await;
824
825	// Block the first start-event append so the public future remains
826	// cancellable after the operation has entered the engine mailbox.
827	let mut event_lock = fd_lock::RwLock::new(manager.store.open_event_lock()?);
828	let event_state_guard = event_lock.try_write()?;
829	let start_manager = manager.clone();
830	let thread_id = thread.id.clone();
831	let start_task = tokio::spawn(async move {
832	start_manager
833	.start_turn(
834	&thread_id,
835	StartTurnRequest {
836	prompt: "the lifecycle outlives its caller".to_string(),
837	..StartTurnRequest::default()
838	},
839	)
840	.await
841	});
842	assert!(matches!(
843	tokio::time::timeout(Duration::from_secs(2), harness.rx_op.recv()).await?,
844	Some(Op::SendMessage { .. })
845	));
846	let turns = manager.store.list_turns_for_thread(&thread.id)?;
847	assert_eq!(turns.len(), 1);
848	let turn_id = turns[0].id.clone();
849	assert_eq!(turns[0].status, RuntimeTurnStatus::InProgress);
850	assert_eq!(turns[0].item_ids.len(), 1);
851	assert_eq!(
852	manager.store.load_item(&turns[0].item_ids[0])?.turn_id,
853	turn_id
854	);
855	assert!(
856	manager
857	.active_turn_flags(&thread.id, &turn_id)
858	.await
859	.is_some()
860	);
861
862	start_task.abort();
863	let _ = start_task.await;
864	drop(event_state_guard);
865
866	harness
867	.tx_event
868	.send(EngineEvent::MessageStarted { index: 0 })
869	.await?;
870	harness
871	.tx_event
872	.send(EngineEvent::MessageDelta {
873	index: 0,
874	content: "owned monitor is live".to_string(),
875	})
876	.await?;
877	harness
878	.tx_event
879	.send(EngineEvent::MessageComplete { index: 0 })
880	.await?;
881	harness
882	.tx_event
883	.send(EngineEvent::TurnComplete {
884	usage: Usage::default(),
885	status: TurnOutcomeStatus::Completed,
886	error: None,
887	tool_catalog: None,
888	base_url: None,
889	})
890	.await?;
891	let terminal = wait_for_terminal_turn(&manager, &turn_id, Duration::from_secs(2)).await?;
892	assert_eq!(terminal.status, RuntimeTurnStatus::Completed);
893	assert_eq!(manager.active_turn_flags(&thread.id, &turn_id).await, None);
894
895	let lifecycle: Vec<String> = manager
896	.events_since(&thread.id, None)?
897	.iter()
898	.filter(\|event\| event.turn_id.as_deref() == Some(turn_id.as_str()))
899	.map(\|event\| event.event.clone())
900	.collect();
901	assert_eq!(
902	&lifecycle[..3],
903	&["turn.started", "item.started", "item.completed"]
904	);
905	assert_eq!(lifecycle.last().map(String::as_str), Some("turn.completed"));
906	Ok(())
907	}
908
909	#[tokio::test]
910	async fn thread_updates_while_start_waits_for_capacity_survive_latest_turn_write() -> Result<()> {
911	let manager = test_manager(test_runtime_dir())?;
912	let thread = manager
913	.create_thread(CreateThreadRequest::default())
914	.await?;
915	let mut harness = install_mock_engine(&manager, &thread.id).await;
916	for _ in 0..32 {
917	harness.handle.try_send(Op::ListSubAgents)?;
918	}
919	let sender_count = harness.handle.tx_op.strong_count();
920
921	let start_manager = manager.clone();
922	let thread_id = thread.id.clone();
923	let start_task = tokio::spawn(async move {
924	start_manager
925	.start_turn(
926	&thread_id,
927	StartTurnRequest {
928	prompt: "preserve concurrent metadata".to_string(),
929	..StartTurnRequest::default()
930	},
931	)
932	.await
933	});
934	wait_for_sender_strong_count(&harness.handle.tx_op, sender_count + 2).await?;
935	assert!(!start_task.is_finished());
936
937	manager
938	.update_thread(
939	&thread.id,
940	UpdateThreadRequest {
941	title: Some("new title while queued".to_string()),
942	..UpdateThreadRequest::default()
943	},
944	)
945	.await?;
946	assert!(matches!(
947	harness.rx_op.recv().await,
948	Some(Op::ListSubAgents)
949	));
950	let turn = tokio::time::timeout(Duration::from_secs(2), start_task).await???;
951	let mut saw_send = false;
952	for _ in 0..32 {
953	if matches!(harness.rx_op.recv().await, Some(Op::SendMessage { .. })) {
954	saw_send = true;
955	break;
956	}
957	}
958	assert!(
959	saw_send,
960	"accepted send must remain behind refresh operations"
961	);
962
963	harness
964	.tx_event
965	.send(EngineEvent::MessageStarted { index: 0 })
966	.await?;
967	harness
968	.tx_event
969	.send(EngineEvent::MessageDelta {
970	index: 0,
971	content: "metadata retained".to_string(),
972	})
973	.await?;
974	harness
975	.tx_event
976	.send(EngineEvent::MessageComplete { index: 0 })
977	.await?;
978	harness
979	.tx_event
980	.send(EngineEvent::TurnComplete {
981	usage: Usage::default(),
982	status: TurnOutcomeStatus::Completed,
983	error: None,
984	tool_catalog: None,
985	base_url: None,
986	})
987	.await?;
988	let terminal = wait_for_terminal_turn(&manager, &turn.id, Duration::from_secs(2)).await?;
989	assert_eq!(terminal.status, RuntimeTurnStatus::Completed);
990	assert_eq!(turn.item_ids.len(), 1);
991	assert!(
992	terminal.item_ids.contains(&turn.item_ids[0]),
993	"the accepted user item must survive later assistant-item writes"
994	);
995	assert_eq!(
996	manager.store.load_turn(&turn.id)?.item_ids,
997	terminal.item_ids
998	);
999	let updated = manager.get_thread(&thread.id).await?;
1000	assert_eq!(updated.title.as_deref(), Some("new title while queued"));
1001	assert_eq!(updated.latest_turn_id.as_deref(), Some(turn.id.as_str()));
1002	Ok(())
1003	}
1004
1005	#[tokio::test]
1006	async fn execution_update_while_start_waits_rejects_stale_operation() -> Result<()> {
1007	let manager = test_manager(test_runtime_dir())?;
1008	let thread = manager
1009	.create_thread(CreateThreadRequest::default())
1010	.await?;
1011	let mut harness = install_mock_engine(&manager, &thread.id).await;
1012	for _ in 0..32 {
1013	harness.handle.try_send(Op::ListSubAgents)?;
1014	}
1015	let sender_count = harness.handle.tx_op.strong_count();
1016	let start_manager = manager.clone();
1017	let thread_id = thread.id.clone();
1018	let start_task = tokio::spawn(async move {
1019	start_manager
1020	.start_turn(
1021	&thread_id,
1022	StartTurnRequest {
1023	prompt: "must not use stale mode".to_string(),
1024	..StartTurnRequest::default()
1025	},
1026	)
1027	.await
1028	});
1029	wait_for_sender_strong_count(&harness.handle.tx_op, sender_count + 2).await?;
1030
1031	manager
1032	.update_thread(
1033	&thread.id,
1034	UpdateThreadRequest {
1035	mode: Some(AppMode::Plan.as_setting().to_string()),
1036	..UpdateThreadRequest::default()
1037	},
1038	)
1039	.await?;
1040	assert!(matches!(
1041	harness.rx_op.recv().await,
1042	Some(Op::ListSubAgents)
1043	));
1044	let error = tokio::time::timeout(Duration::from_secs(2), start_task)
1045	.await??
1046	.expect_err("stale operation must fail")
1047	.to_string();
1048	assert!(error.contains("execution settings changed"), "{error}");
1049	for _ in 0..31 {
1050	assert!(matches!(
1051	harness.rx_op.recv().await,
1052	Some(Op::ListSubAgents)
1053	));
1054	}
1055	assert!(harness.rx_op.try_recv().is_err());
1056	assert!(manager.store.list_turns_for_thread(&thread.id)?.is_empty());
1057	let updated = manager.get_thread(&thread.id).await?;
1058	assert_eq!(updated.mode, AppMode::Plan.as_setting());
1059	assert_eq!(updated.latest_turn_id, None);
1060	assert_eq!(manager.active_turn_flags(&thread.id, "missing").await, None);
1061	Ok(())
1062	}
1063
1064	#[tokio::test]
1065	async fn compact_lifecycle_outlives_caller_and_preserves_concurrent_thread_updates() -> Result<()> {
1066	let manager = test_manager(test_runtime_dir())?;
1067	let thread = manager
1068	.create_thread(CreateThreadRequest::default())
1069	.await?;
1070	let mut harness = install_mock_engine(&manager, &thread.id).await;
1071	for _ in 0..32 {
1072	harness.handle.try_send(Op::ListSubAgents)?;
1073	}
1074	let sender_count = harness.handle.tx_op.strong_count();
1075
1076	let compact_manager = manager.clone();
1077	let thread_id = thread.id.clone();
1078	let compact_task = tokio::spawn(async move {
1079	compact_manager
1080	.compact_thread(&thread_id, CompactThreadRequest::default())
1081	.await
1082	});
1083	wait_for_sender_strong_count(&harness.handle.tx_op, sender_count + 2).await?;
1084	assert!(!compact_task.is_finished());
1085	manager
1086	.update_thread(
1087	&thread.id,
1088	UpdateThreadRequest {
1089	title: Some("title before compact claim".to_string()),
1090	..UpdateThreadRequest::default()
1091	},
1092	)
1093	.await?;
1094	// Once capacity is released, block the acknowledgement events so the
1095	// API future can be dropped after the engine accepted the operation.
1096	let mut event_lock = fd_lock::RwLock::new(manager.store.open_event_lock()?);
1097	let event_state_guard = event_lock.try_write()?;
1098	let mut saw_compact = false;
1099	for _ in 0..33 {
1100	if matches!(
1101	tokio::time::timeout(Duration::from_secs(2), harness.rx_op.recv()).await?,
1102	Some(Op::CompactContext { .. })
1103	) {
1104	saw_compact = true;
1105	break;
1106	}
1107	}
1108	assert!(
1109	saw_compact,
1110	"manual compaction must enter the engine mailbox"
1111	);
1112	let turns = manager.store.list_turns_for_thread(&thread.id)?;
1113	assert_eq!(turns.len(), 1);
1114	let turn_id = turns[0].id.clone();
1115	compact_task.abort();
1116	let _ = compact_task.await;
1117	drop(event_state_guard);
1118
1119	harness
1120	.tx_event
1121	.send(EngineEvent::CompactionStarted {
1122	id: "manual_owned".to_string(),
1123	auto: false,
1124	message: "compaction started".to_string(),
1125	})
1126	.await?;
1127	harness
1128	.tx_event
1129	.send(EngineEvent::CompactionCompleted {
1130	id: "manual_owned".to_string(),
1131	auto: false,
1132	message: "compaction completed".to_string(),
1133	messages_before: Some(4),
1134	messages_after: Some(2),
1135	summary_prompt: None,
1136	})
1137	.await?;
1138	harness
1139	.tx_event
1140	.send(EngineEvent::TurnComplete {
1141	usage: Usage::default(),
1142	status: TurnOutcomeStatus::Completed,
1143	error: None,
1144	tool_catalog: None,
1145	base_url: None,
1146	})
1147	.await?;
1148	let terminal = wait_for_terminal_turn(&manager, &turn_id, Duration::from_secs(2)).await?;
1149	assert_eq!(terminal.status, RuntimeTurnStatus::Completed);
1150	assert_eq!(manager.active_turn_flags(&thread.id, &turn_id).await, None);
1151	let updated = manager.get_thread(&thread.id).await?;
1152	assert_eq!(updated.title.as_deref(), Some("title before compact claim"));
1153	assert_eq!(updated.latest_turn_id.as_deref(), Some(turn_id.as_str()));
1154	Ok(())
1155	}
1156
1157	#[tokio::test]
1158	async fn concurrent_turn_starts_leave_one_claim_and_one_consistent_durable_turn() -> Result<()> {
1159	let manager = test_manager(test_runtime_dir())?;
1160	let thread = manager
1161	.create_thread(CreateThreadRequest::default())
1162	.await?;
1163	let mut harness = install_mock_engine(&manager, &thread.id).await;
1164	let first = manager.start_turn(
1165	&thread.id,
1166	StartTurnRequest {
1167	prompt: "first concurrent turn".to_string(),
1168	..StartTurnRequest::default()
1169	},
1170	);
1171	let second = manager.start_turn(
1172	&thread.id,
1173	StartTurnRequest {
1174	prompt: "second concurrent turn".to_string(),
1175	..StartTurnRequest::default()
1176	},
1177	);
1178
1179	let (first, second) = tokio::join!(first, second);
1180	let (turn, rejection) = match (first, second) {
1181	(Ok(turn), Err(error)) \| (Err(error), Ok(turn)) => (turn, error),
1182	(first, second) => {
1183	panic!("expected one accepted turn and one rejection: {first:?} {second:?}")
1184	}
1185	};
1186	assert!(
1187	rejection.to_string().contains("already has an active turn"),
1188	"{rejection}"
1189	);
1190	let turns = manager.store.list_turns_for_thread(&thread.id)?;
1191	assert_eq!(turns.len(), 1);
1192	assert_eq!(turns[0].id, turn.id);
1193	assert_eq!(
1194	manager.get_thread(&thread.id).await?.latest_turn_id,
1195	Some(turn.id.clone())
1196	);
1197	assert_eq!(
1198	manager.active_turn_flags(&thread.id, &turn.id).await,
1199	Some((false, false))
1200	);
1201	match harness.rx_op.recv().await {
1202	Some(Op::SendMessage { compaction, .. }) => {
1203	assert_eq!(
1204	compaction.runtime_cost_owner.as_deref(),
1205	Some(turn.id.as_str())
1206	);
1207	}
1208	other => panic!("expected SendMessage op, got {other:?}"),
1209	}
1210	assert!(harness.rx_op.try_recv().is_err());
1211	Ok(())
1212	}
1213
1214	#[test]
1215	fn legacy_custom_thread_stays_on_root_when_literal_table_coexists() -> Result<()> {
1216	let mut custom = std::collections::HashMap::new();
1217	custom.insert(
1218	"custom".to_string(),
1219	crate::config::ProviderConfig {
1220	kind: Some("openai-compatible".to_string()),
1221	base_url: Some("http://127.0.0.1:18182/v1".to_string()),
1222	model: Some("table-model".to_string()),
1223	..crate::config::ProviderConfig::default()
1224	},
1225	);
1226	let config = Config {
1227	provider: Some("custom".to_string()),
1228	base_url: Some("http://127.0.0.1:18181/v1".to_string()),
1229	default_text_model: Some("legacy-root-model".to_string()),
1230	providers: Some(crate::config::ProvidersConfig {
1231	custom,
1232	..crate::config::ProvidersConfig::default()
1233	}),
1234	..Config::default()
1235	};
1236	let manager = RuntimeThreadManager::open(
1237	config.clone(),
1238	PathBuf::from("."),
1239	test_manager_config(test_runtime_dir()),
1240	)?;
1241	let mut legacy = sample_thread("thr_ambiguous_legacy_custom");
1242	legacy.model = "legacy-saved-model".to_string();
1243	legacy.model_provider = Some("custom".to_string());
1244	legacy.model_provider_id = None;
1245
1246	let root = manager.resolved_route_for_thread(&config, &legacy)?;
1247	assert_eq!(root.identity.provider, ApiProvider::Custom);
1248	assert_eq!(root.identity.key, "custom");
1249	assert_eq!(root.identity.exact_id, None);
1250	assert_eq!(root.config.deepseek_base_url(), "http://127.0.0.1:18181/v1");
1251
1252	legacy.model_provider_id = Some("custom".to_string());
1253	let exact = manager.resolved_route_for_thread(&config, &legacy)?;
1254	assert_eq!(exact.identity.provider, ApiProvider::Custom);
1255	assert_eq!(exact.identity.key, "custom");
1256	assert_eq!(exact.identity.exact_id.as_deref(), Some("custom"));
1257	assert_eq!(
1258	exact.config.deepseek_base_url(),
1259	"http://127.0.0.1:18182/v1"
1260	);
1261	let root_only = Config {
1262	provider: Some("custom".to_string()),
1263	base_url: Some("http://127.0.0.1:18181/v1".to_string()),
1264	default_text_model: Some("legacy-root-model".to_string()),
1265	..Config::default()
1266	};
1267	let error = manager
1268	.resolved_route_for_thread(&root_only, &legacy)
1269	.expect_err("exact literal table thread must not fall back to root")
1270	.to_string();
1271	assert!(error.contains("[providers.custom]"), "{error}");
1272	assert!(error.contains("will not fall back"), "{error}");
1273	Ok(())
1274	}
1275
1276	#[tokio::test]
1277	async fn empty_imported_custom_id_fails_closed_when_root_and_table_coexist() -> Result<()> {
1278	let mut custom = std::collections::HashMap::new();
1279	custom.insert(
1280	"custom".to_string(),
1281	crate::config::ProviderConfig {
1282	kind: Some("openai-compatible".to_string()),
1283	base_url: Some("http://127.0.0.1:18182/v1".to_string()),
1284	model: Some("table-model".to_string()),
1285	..crate::config::ProviderConfig::default()
1286	},
1287	);
1288	let config = Config {
1289	provider: Some("custom".to_string()),
1290	base_url: Some("http://127.0.0.1:18181/v1".to_string()),
1291	default_text_model: Some("legacy-root-model".to_string()),
1292	providers: Some(crate::config::ProvidersConfig {
1293	custom,
1294	..crate::config::ProvidersConfig::default()
1295	}),
1296	..Config::default()
1297	};
1298	let manager = RuntimeThreadManager::open(
1299	config.clone(),
1300	PathBuf::from("."),
1301	test_manager_config(test_runtime_dir()),
1302	)?;
1303
1304	let mut imported = sample_thread("thr_empty_custom_id");
1305	imported.model_provider = Some("custom".to_string());
1306	imported.model_provider_id = Some(" ".to_string());
1307	let error = manager
1308	.resolved_route_for_thread(&config, &imported)
1309	.expect_err("malformed imported identity must not acquire the root route")
1310	.to_string();
1311	assert!(error.contains("empty exact provider id"), "{error}");
1312
1313	let before = manager.store.list_threads()?.len();
1314	let request_error = manager
1315	.create_thread(CreateThreadRequest {
1316	model_provider: Some("custom".to_string()),
1317	model_provider_id: Some(String::new()),
1318	..CreateThreadRequest::default()
1319	})
1320	.await
1321	.expect_err("malformed create request must fail before persistence")
1322	.to_string();
1323	assert!(
1324	request_error.contains("empty exact provider id"),
1325	"{request_error}"
1326	);
1327	assert_eq!(manager.store.list_threads()?.len(), before);
1328	Ok(())
1329	}
1330
1331	#[tokio::test]
1332	async fn thread_records_and_create_requests_preserve_provider_kind_id_pairing() -> Result<()> {
1333	let mut custom = std::collections::HashMap::new();
1334	custom.insert(
1335	"openai".to_string(),
1336	crate::config::ProviderConfig {
1337	kind: Some("openai-compatible".to_string()),
1338	base_url: Some("http://127.0.0.1:18183/v1".to_string()),
1339	model: Some("custom-openai-model".to_string()),
1340	..crate::config::ProviderConfig::default()
1341	},
1342	);
1343	let config = Config {
1344	provider: Some("openai".to_string()),
1345	providers: Some(crate::config::ProvidersConfig {
1346	custom,
1347	..crate::config::ProvidersConfig::default()
1348	}),
1349	..Config::default()
1350	};
1351	let manager = RuntimeThreadManager::open(
1352	config.clone(),
1353	PathBuf::from("."),
1354	test_manager_config(test_runtime_dir()),
1355	)?;
1356
1357	for provider_id in [None, Some("openai".to_string())] {
1358	let mut built_in = sample_thread("thr_builtin_openai_collision");
1359	built_in.model_provider = Some("openai".to_string());
1360	built_in.model_provider_id = provider_id;
1361	let error = manager
1362	.resolved_route_for_thread(&config, &built_in)
1363	.expect_err("built-in thread must not route through same-key custom endpoint")
1364	.to_string();
1365	assert!(error.contains("requires built-in 'openai'"), "{error}");
1366	assert!(error.contains("shadows"), "{error}");
1367	}
1368
1369	let mut exact_custom = sample_thread("thr_custom_openai_collision");
1370	exact_custom.model = "custom-openai-model".to_string();
1371	exact_custom.model_provider = Some("custom".to_string());
1372	exact_custom.model_provider_id = Some("openai".to_string());
1373	let route = manager.resolved_route_for_thread(&config, &exact_custom)?;
1374	assert_eq!(route.identity.provider, ApiProvider::Custom);
1375	assert_eq!(route.identity.key, "openai");
1376	assert_eq!(
1377	route.config.deepseek_base_url(),
1378	"http://127.0.0.1:18183/v1"
1379	);
1380
1381	let mut auto_thread = exact_custom.clone();
1382	auto_thread.id = "thr_auto_openai_collision".to_string();
1383	auto_thread.model = "auto".to_string();
1384	manager.store.save_thread(&auto_thread)?;
1385	let mut restored_turn = sample_turn(
1386	&auto_thread.id,
1387	"turn_openai_collision",
1388	RuntimeTurnStatus::Completed,
1389	);
1390	restored_turn.effective_provider = Some("openai".to_string());
1391	restored_turn.effective_provider_id = None;
1392	restored_turn.effective_model = Some("custom-openai-model".to_string());
1393	manager.store.save_turn(&restored_turn)?;
1394	let turn_error = manager
1395	.resolved_route_for_thread(&config, &auto_thread)
1396	.expect_err("restored built-in turn must not be captured by custom endpoint")
1397	.to_string();
1398	assert!(
1399	turn_error.contains("requires built-in 'openai'"),
1400	"{turn_error}"
1401	);
1402
1403	restored_turn.effective_provider = Some("custom".to_string());
1404	restored_turn.effective_provider_id = Some("openai".to_string());
1405	manager.store.save_turn(&restored_turn)?;
1406	let restored_custom = manager.resolved_route_for_thread(&config, &auto_thread)?;
1407	assert_eq!(restored_custom.identity.provider, ApiProvider::Custom);
1408	assert_eq!(restored_custom.identity.key, "openai");
1409	assert_eq!(restored_custom.model, "custom-openai-model");
1410
1411	let request_error = manager
1412	.create_thread(CreateThreadRequest {
1413	model_provider: Some("openai".to_string()),
1414	model_provider_id: Some("openai".to_string()),
1415	..CreateThreadRequest::default()
1416	})
1417	.await
1418	.expect_err("built-in request must fail closed under exact custom shadow")
1419	.to_string();
1420	assert!(
1421	request_error.contains("requires built-in 'openai'"),
1422	"{request_error}"
1423	);
1424
1425	let created = manager
1426	.create_thread(CreateThreadRequest {
1427	model_provider: Some("custom".to_string()),
1428	model_provider_id: Some("openai".to_string()),
1429	..CreateThreadRequest::default()
1430	})
1431	.await?;
1432	assert_eq!(created.model_provider.as_deref(), Some("custom"));
1433	assert_eq!(created.model_provider_id.as_deref(), Some("openai"));
1434	assert_eq!(created.model, "custom-openai-model");
1435	Ok(())
1436	}
1437
1438	#[tokio::test]
1439	async fn config_reload_updates_next_turn_route_without_mutating_engine_route() -> Result<()> {
1440	let mut custom = std::collections::HashMap::new();
1441	custom.insert(
1442	"lm-studio".to_string(),
1443	crate::config::ProviderConfig {
1444	kind: Some("openai-compatible".to_string()),
1445	base_url: Some("http://127.0.0.1:18181/v1".to_string()),
1446	model: Some("local-model".to_string()),
1447	api_key: Some("old-local-test-key".to_string()),
1448	..crate::config::ProviderConfig::default()
1449	},
1450	);
1451	let config = Config {
1452	provider: Some("lm-studio".to_string()),
1453	providers: Some(crate::config::ProvidersConfig {
1454	custom,
1455	..crate::config::ProvidersConfig::default()
1456	}),
1457	..Config::default()
1458	};
1459	let manager = RuntimeThreadManager::open(
1460	config.clone(),
1461	PathBuf::from("."),
1462	test_manager_config(test_runtime_dir()),
1463	)?;
1464	let thread = manager
1465	.create_thread(CreateThreadRequest {
1466	model: Some("local-model".to_string()),
1467	model_provider: Some("lm-studio".to_string()),
1468	..CreateThreadRequest::default()
1469	})
1470	.await?;
1471	let mut harness = install_mock_engine(&manager, &thread.id).await;
1472
1473	let mut reloaded = config;
1474	let provider = reloaded
1475	.providers
1476	.as_mut()
1477	.and_then(\|providers\| providers.custom.get_mut("lm-studio"))
1478	.expect("named custom provider");
1479	provider.base_url = Some("http://127.0.0.1:18182/v1".to_string());
1480	provider.api_key = Some("new-local-test-key".to_string());
1481	manager.reload_config(reloaded).await?;
1482
1483	let refreshed = manager.resolved_route_for_thread(&manager.read_config(), &thread)?;
1484	assert_eq!(refreshed.identity.key, "lm-studio");
1485	assert_eq!(
1486	refreshed.config.deepseek_base_url(),
1487	"http://127.0.0.1:18182/v1"
1488	);
1489	for _ in 0..3 {
1490	let op = harness.rx_op.recv().await.expect("runtime control op");
1491	assert!(
1492	matches!(
1493	op,
1494	Op::SetCompaction { .. }
1495	\| Op::SetStreamChunkTimeout { .. }
1496	\| Op::SetSubagentRuntimeConfig { .. }
1497	),
1498	"reload must not mutate an engine provider route: {op:?}"
1499	);
1500	}
1501	let compact_turn = manager
1502	.compact_thread(
1503	&thread.id,
1504	CompactThreadRequest {
1505	reason: Some("verify refreshed route".to_string()),
1506	},
1507	)
1508	.await?;
1509	assert_eq!(compact_turn.effective_provider.as_deref(), Some("custom"));
1510	assert_eq!(
1511	compact_turn.effective_provider_id.as_deref(),
1512	Some("lm-studio")
1513	);
1514	assert_eq!(compact_turn.effective_model.as_deref(), Some("local-model"));
1515	match harness.rx_op.recv().await {
1516	Some(Op::CompactContext { route, compaction }) => {
1517	assert_eq!(route.identity.key, "lm-studio");
1518	assert_eq!(
1519	route.config.deepseek_base_url(),
1520	"http://127.0.0.1:18182/v1"
1521	);
1522	assert_eq!(compaction.model, "local-model");
1523	assert_eq!(
1524	compaction.effective_context_window,
1525	Some(crate::route_budget::route_context_window_tokens(
1526	ApiProvider::Custom,
1527	"local-model",
1528	crate::route_budget::known_route_limits(route.candidate.limits()),
1529	))
1530	);
1531	}
1532	other => panic!("expected typed compact route, got {other:?}"),
1533	}
1534
1535	Ok(())
1536	}
1537
1538	#[tokio::test]
1539	async fn config_sync_reports_removed_named_custom_route_and_keeps_mailbox_clean() -> Result<()> {
1540	let mut custom = std::collections::HashMap::new();
1541	custom.insert(
1542	"lm-studio".to_string(),
1543	crate::config::ProviderConfig {
1544	kind: Some("openai-compatible".to_string()),
1545	base_url: Some("http://127.0.0.1:18181/v1".to_string()),
1546	model: Some("local-model".to_string()),
1547	api_key: Some("local-test-key".to_string()),
1548	..crate::config::ProviderConfig::default()
1549	},
1550	);
1551	let config = Config {
1552	provider: Some("lm-studio".to_string()),
1553	providers: Some(crate::config::ProvidersConfig {
1554	custom,
1555	..crate::config::ProvidersConfig::default()
1556	}),
1557	..Config::default()
1558	};
1559	let manager = RuntimeThreadManager::open(
1560	config,
1561	PathBuf::from("."),
1562	test_manager_config(test_runtime_dir()),
1563	)?;
1564	let thread = manager
1565	.create_thread(CreateThreadRequest {
1566	model: Some("local-model".to_string()),
1567	model_provider: Some("lm-studio".to_string()),
1568	..CreateThreadRequest::default()
1569	})
1570	.await?;
1571	let mut harness = install_mock_engine(&manager, &thread.id).await;
1572
1573	let err = manager
1574	.reload_config(Config::default())
1575	.await
1576	.expect_err("removed named custom route must fail config reload");
1577
1578	let message = err.to_string();
1579	assert!(message.contains(&thread.id), "{message}");
1580	assert!(message.contains("lm-studio"), "{message}");
1581	assert!(harness.rx_op.try_recv().is_err());
1582	Ok(())
1583	}
1584
1585	#[tokio::test]
1586	async fn create_thread_uses_requested_named_custom_provider_default_model() -> Result<()> {
1587	let mut custom = std::collections::HashMap::new();
1588	for (name, base_url, model) in [
1589	("custom-a", "http://127.0.0.1:18181/v1", "model-a"),
1590	("custom-b", "http://127.0.0.1:18182/v1", "model-b"),
1591	] {
1592	custom.insert(
1593	name.to_string(),
1594	crate::config::ProviderConfig {
1595	kind: Some("openai-compatible".to_string()),
1596	base_url: Some(base_url.to_string()),
1597	model: Some(model.to_string()),
1598	..Default::default()
1599	},
1600	);
1601	}
1602	let config = Config {
1603	provider: Some("custom-b".to_string()),
1604	providers: Some(crate::config::ProvidersConfig {
1605	custom,
1606	..Default::default()
1607	}),
1608	..Default::default()
1609	};
1610	let manager = RuntimeThreadManager::open(
1611	config.clone(),
1612	PathBuf::from("."),
1613	test_manager_config(test_runtime_dir()),
1614	)?;
1615
1616	let thread = manager
1617	.create_thread(CreateThreadRequest {
1618	model_provider: Some("custom-a".to_string()),
1619	..Default::default()
1620	})
1621	.await?;
1622
1623	assert_eq!(thread.model_provider.as_deref(), Some("custom"));
1624	assert_eq!(thread.model_provider_id.as_deref(), Some("custom-a"));
1625	assert_eq!(thread.model, "model-a");
1626	let route = manager.resolved_route_for_thread(&config, &thread)?;
1627	assert_eq!(route.identity.key, "custom-a");
1628	assert_eq!(
1629	route.config.deepseek_base_url(),
1630	"http://127.0.0.1:18181/v1"
1631	);
1632	Ok(())
1633	}
1634
1635	#[tokio::test]
1636	async fn create_thread_uses_requested_non_current_builtin_default_model() -> Result<()> {
1637	let config = Config {
1638	provider: Some("openrouter".to_string()),
1639	default_text_model: Some(DEFAULT_TEXT_MODEL.to_string()),
1640	..Default::default()
1641	};
1642	let manager = RuntimeThreadManager::open(
1643	config,
1644	PathBuf::from("."),
1645	test_manager_config(test_runtime_dir()),
1646	)?;
1647
1648	let thread = manager
1649	.create_thread(CreateThreadRequest {
1650	model_provider: Some("zai".to_string()),
1651	..Default::default()
1652	})
1653	.await?;
1654
1655	assert_eq!(thread.model_provider.as_deref(), Some("zai"));
1656	assert_eq!(thread.model, crate::config::DEFAULT_ZAI_MODEL);
1657	Ok(())
1658	}
1659
1660	#[tokio::test]
1661	async fn simultaneous_named_custom_auto_threads_keep_exact_routes() -> Result<()> {
1662	let mut custom = std::collections::HashMap::new();
1663	for (name, base_url, model) in [
1664	("custom-a", "http://127.0.0.1:18181/v1", "model-a"),
1665	("custom-b", "http://127.0.0.1:18182/v1", "model-b"),
1666	] {
1667	custom.insert(
1668	name.to_string(),
1669	crate::config::ProviderConfig {
1670	kind: Some("openai-compatible".to_string()),
1671	base_url: Some(base_url.to_string()),
1672	model: Some(model.to_string()),
1673	..Default::default()
1674	},
1675	);
1676	}
1677	let manager = RuntimeThreadManager::open(
1678	Config {
1679	provider: Some("custom-b".to_string()),
1680	providers: Some(crate::config::ProvidersConfig {
1681	custom,
1682	..Default::default()
1683	}),
1684	..Default::default()
1685	},
1686	PathBuf::from("."),
1687	test_manager_config(test_runtime_dir()),
1688	)?;
1689	let thread_a = manager
1690	.create_thread(CreateThreadRequest {
1691	model: Some("auto".to_string()),
1692	model_provider: Some("custom-a".to_string()),
1693	..Default::default()
1694	})
1695	.await?;
1696	let thread_b = manager
1697	.create_thread(CreateThreadRequest {
1698	model: Some("auto".to_string()),
1699	model_provider: Some("custom-b".to_string()),
1700	..Default::default()
1701	})
1702	.await?;
1703	let mut harness_a = install_mock_engine(&manager, &thread_a.id).await;
1704	let mut harness_b = install_mock_engine(&manager, &thread_b.id).await;
1705
1706	let request_a = manager.start_turn(
1707	&thread_a.id,
1708	StartTurnRequest {
1709	prompt: "route A".to_string(),
1710	..Default::default()
1711	},
1712	);
1713	let request_b = manager.start_turn(
1714	&thread_b.id,
1715	StartTurnRequest {
1716	prompt: "route B".to_string(),
1717	..Default::default()
1718	},
1719	);
1720	let (turn_a, turn_b) = tokio::join!(request_a, request_b);
1721	let turn_a = turn_a?;
1722	let turn_b = turn_b?;
1723
1724	assert_eq!(turn_a.effective_provider.as_deref(), Some("custom"));
1725	assert_eq!(turn_a.effective_provider_id.as_deref(), Some("custom-a"));
1726	assert_eq!(turn_a.effective_model.as_deref(), Some("model-a"));
1727	assert_eq!(turn_b.effective_provider.as_deref(), Some("custom"));
1728	assert_eq!(turn_b.effective_provider_id.as_deref(), Some("custom-b"));
1729	assert_eq!(turn_b.effective_model.as_deref(), Some("model-b"));
1730	match harness_a.rx_op.recv().await {
1731	Some(Op::SendMessage { route, .. }) => {
1732	assert_eq!(route.identity.provider, ApiProvider::Custom);
1733	assert_eq!(route.identity.key, "custom-a");
1734	assert_eq!(route.model, "model-a");
1735	}
1736	other => panic!("expected custom A send, got {other:?}"),
1737	}
1738	match harness_b.rx_op.recv().await {
1739	Some(Op::SendMessage { route, .. }) => {
1740	assert_eq!(route.identity.provider, ApiProvider::Custom);
1741	assert_eq!(route.identity.key, "custom-b");
1742	assert_eq!(route.model, "model-b");
1743	}
1744	other => panic!("expected custom B send, got {other:?}"),
1745	}
1746	Ok(())
1747	}
1748
1749	#[test]
1750	fn turn_record_persists_billing_surface_without_raw_endpoint() {
1751	let mut turn = sample_turn("thr_surface", "turn_surface", RuntimeTurnStatus::Completed);
1752	let fingerprint = "a".repeat(64);
1753	turn.persist_effective_route(&crate::cost_status::EffectiveRouteEnvelope {
1754	provider: ApiProvider::Stepfun,
1755	provider_identity: "stepfun-primary".to_string(),
1756	model: "step-3.7-flash".to_string(),
1757	billing_surface: Some(crate::pricing::STEPFUN_PAYG_BILLING_SURFACE.to_string()),
1758	endpoint_fingerprint: Some(fingerprint.clone()),
1759	billing_mode: crate::cost_status::RouteBillingMode::Metered,
1760	dispatched_at: turn.created_at,
1761	});
1762
1763	let value = serde_json::to_value(turn).expect("serialize turn");
1764	assert_eq!(
1765	value["effective_billing_surface"],
1766	crate::pricing::STEPFUN_PAYG_BILLING_SURFACE
1767	);
1768	assert_eq!(value["effective_billing_mode"], "metered");
1769	assert_eq!(value["effective_endpoint_fingerprint"], fingerprint);
1770	assert!(value["effective_dispatched_at"].is_string());
1771	assert!(value.get("base_url").is_none());
1772	assert!(value.get("effective_base_url").is_none());
1773	}
1774
1775	#[test]
1776	fn serialized_turn_record_redacts_all_route_and_source_fields() {
1777	let mut value = serde_json::to_value(sample_turn(
1778	"thr_secret_route",
1779	"turn_secret_route",
1780	RuntimeTurnStatus::Completed,
1781	))
1782	.expect("serialize clean fixture");
1783	value["effective_provider"] = serde_json::json!("Authorization: Bearer provider-secret");
1784	value["effective_provider_id"] = serde_json::json!("CUSTOM_API_KEY=sk-provider-secret");
1785	value["effective_model"] = serde_json::json!("../.ssh/model-secret");
1786	value["effective_billing_surface"] =
1787	serde_json::json!("https://alice:password@example.test/v1?token=secret#fragment");
1788	value["effective_endpoint_fingerprint"] = serde_json::json!("secret-endpoint");
1789	value["routed_usage_source_ids"] = serde_json::json!(["raw-provider-response-secret"]);
1790	value["routed_usage"] = serde_json::json!([{
1791	"route": {
1792	"provider": "custom",
1793	"provider_identity": "TOKEN=ghp_child_secret",
1794	"model": r"relative\secret\model",
1795	"billing_surface": "https://host.test/v1?key=child-secret",
1796	"endpoint_fingerprint": "child-endpoint-secret",
1797	"billing_mode": "metered",
1798	"dispatched_at": Utc::now(),
1799	},
1800	"usage": Usage::default(),
1801	}]);
1802	let turn: TurnRecord = serde_json::from_value(value).expect("deserialize hostile fixture");
1803
1804	let serialized = serde_json::to_string(&turn).expect("serialize turn record");
1805	for secret in [
1806	"provider-secret",
1807	"sk-provider-secret",
1808	".ssh",
1809	"alice",
1810	"password",
1811	"token=secret",
1812	"raw-provider-response-secret",
1813	"ghp_child_secret",
1814	"child-secret",
1815	"child-endpoint-secret",
1816	] {
1817	assert!(
1818	!serialized.contains(secret),
1819	"serialized turn leaked {secret:?}: {serialized}"
1820	);
1821	}
1822	}
1823
1824	#[tokio::test]
1825	async fn aggregate_usage_keeps_codex_tokens_without_api_dollar_pricing() -> Result<()> {
1826	let manager = test_manager(test_runtime_dir())?;
1827	let mut thread = sample_thread("thr_mixed_routes");
1828	thread.model = "auto".to_string();
1829	manager.store.save_thread(&thread)?;
1830
1831	let usage = Usage {
1832	input_tokens: 10_000,
1833	output_tokens: 1_000,
1834	..Usage::default()
1835	};
1836	let mut deepseek = sample_turn(&thread.id, "turn_deepseek", RuntimeTurnStatus::Completed);
1837	deepseek.usage = Some(usage.clone());
1838	set_test_turn_route(
1839	&mut deepseek,
1840	ApiProvider::Deepseek,
1841	ApiProvider::Deepseek.as_str(),
1842	"deepseek-v4-flash",
1843	// A real dispatch persists the classified surface. Leaving it absent
1844	// here would price the row at official DeepSeek rates on nothing but
1845	// the provider name, which is what the route audit now refuses.
1846	Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1847	crate::cost_status::RouteBillingMode::Metered,
1848	);
1849	manager.store.save_turn(&deepseek)?;
1850
1851	let mut codex = sample_turn(&thread.id, "turn_codex", RuntimeTurnStatus::Completed);
1852	codex.usage = Some(usage);
1853	set_test_turn_route(
1854	&mut codex,
1855	ApiProvider::OpenaiCodex,
1856	ApiProvider::OpenaiCodex.as_str(),
1857	"gpt-5.5",
1858	Some(crate::pricing::OAUTH_SUBSCRIPTION_BILLING_SURFACE),
1859	crate::cost_status::RouteBillingMode::Subscription,
1860	);
1861	manager.store.save_turn(&codex)?;
1862
1863	let report = manager
1864	.aggregate_usage(None, None, UsageGroupBy::Provider)
1865	.await?;
1866	assert_eq!(report.totals.turns, 2);
1867	assert_eq!(report.totals.input_tokens, 20_000);
1868	assert!(report.totals.cost_usd > 0.0);
1869
1870	let deepseek_bucket = report
1871	.buckets
1872	.iter()
1873	.find(\|bucket\| bucket.key == ApiProvider::Deepseek.as_str())
1874	.expect("DeepSeek bucket");
1875	let codex_bucket = report
1876	.buckets
1877	.iter()
1878	.find(\|bucket\| bucket.key == ApiProvider::OpenaiCodex.as_str())
1879	.expect("Codex bucket");
1880	assert!(deepseek_bucket.cost_usd > 0.0);
1881	assert_eq!(codex_bucket.cost_usd, 0.0);
1882	assert_eq!(deepseek_bucket.priced_turns, 1);
1883	assert_eq!(deepseek_bucket.unpriced_turns, 0);
1884	assert_eq!(codex_bucket.priced_turns, 0);
1885	assert_eq!(codex_bucket.nonmetered_turns, 1);
1886	assert_eq!(codex_bucket.unpriced_turns, 0);
1887	assert!(codex_bucket.unpriced_reasons.is_empty());
1888	assert_eq!(report.totals.priced_turns, 1);
1889	assert_eq!(report.totals.nonmetered_turns, 1);
1890	assert_eq!(report.totals.unpriced_turns, 0);
1891	assert!(report.totals.cost_complete);
1892	assert_eq!(codex_bucket.input_tokens, 10_000);
1893	assert_eq!(report.totals.cost_usd, deepseek_bucket.cost_usd);
1894	Ok(())
1895	}
1896
1897	/// An aggregate in which nothing could be priced is unavailable, not zero.
1898	///
1899	/// `cost_usd` is a `f64` and an all-unknown run leaves it at `0.0`, so the
1900	/// only thing standing between a reader and a fabricated "$0.00 spent" is that
1901	/// the aggregate also reports zero priced turns and incomplete coverage. Those
1902	/// qualifiers are the contract; this pins them at the aggregate level, which is
1903	/// where per-turn honesty is most easily lost by summing.
1904	#[tokio::test]
1905	async fn aggregate_usage_reports_an_all_unknown_run_as_unavailable_not_zero() -> Result<()> {
1906	let manager = test_manager(test_runtime_dir())?;
1907	let thread = sample_thread("thr_all_unknown");
1908	manager.store.save_thread(&thread)?;
1909	let usage = Usage {
1910	input_tokens: 10_000,
1911	output_tokens: 1_000,
1912	..Usage::default()
1913	};
1914
1915	for (index, surface) in [
1916	// Endpoint classified but unplaceable.
1917	Some(crate::pricing::UNCLASSIFIED_BILLING_SURFACE),
1918	// No endpoint classification at all: a provider name is not evidence
1919	// that the official endpoint served the turn.
1920	None,
1921	]
1922	.into_iter()
1923	.enumerate()
1924	{
1925	let mut turn = sample_turn(
1926	&thread.id,
1927	&format!("turn_unknown_{index}"),
1928	RuntimeTurnStatus::Completed,
1929	);
1930	turn.usage = Some(usage.clone());
1931	set_test_turn_route(
1932	&mut turn,
1933	ApiProvider::Openai,
1934	ApiProvider::Openai.as_str(),
1935	"gpt-5.5",
1936	surface,
1937	crate::cost_status::RouteBillingMode::Metered,
1938	);
1939	manager.store.save_turn(&turn)?;
1940	}
1941
1942	let report = manager
1943	.aggregate_usage(None, None, UsageGroupBy::Thread)
1944	.await?;
1945
1946	assert_eq!(report.totals.turns, 2);
1947	// Tokens are known and are reported. Only the money is unknown; an unknown
1948	// price must not suppress usage that the provider did report.
1949	assert_eq!(report.totals.input_tokens, 20_000);
1950	assert_eq!(report.totals.priced_turns, 0);
1951	assert_eq!(report.totals.unpriced_turns, 2);
1952	assert_eq!(report.totals.nonmetered_turns, 0);
1953	assert!(
1954	!report.totals.cost_complete,
1955	"an all-unknown run must never be a complete total"
1956	);
1957	assert_eq!(
1958	report.totals.cost_usd, 0.0,
1959	"no spend may be invented for an unpriced run"
1960	);
1961	assert!(
1962	!report.totals.unpriced_reasons.is_empty(),
1963	"an unpriced aggregate must say why"
1964	);
1965	for bucket in &report.buckets {
1966	assert_eq!(bucket.priced_turns, 0);
1967	assert!(
1968	bucket.unpriced_turns > 0,
1969	"a bucket cannot report a complete zero either"
1970	);
1971	}
1972	Ok(())
1973	}
1974
1975	#[tokio::test]
1976	async fn aggregate_usage_marks_unknown_cost_as_subtotal_and_keeps_cache_writes() -> Result<()> {
1977	let manager = test_manager(test_runtime_dir())?;
1978	let thread = sample_thread("thr_cost_coverage");
1979	manager.store.save_thread(&thread)?;
1980	let usage = Usage {
1981	input_tokens: 10_000,
1982	output_tokens: 1_000,
1983	prompt_cache_write_tokens: Some(321),
1984	..Usage::default()
1985	};
1986
1987	let mut priced = sample_turn(&thread.id, "turn_priced", RuntimeTurnStatus::Completed);
1988	priced.usage = Some(usage.clone());
1989	set_test_turn_route(
1990	&mut priced,
1991	ApiProvider::Deepseek,
1992	ApiProvider::Deepseek.as_str(),
1993	"deepseek-v4-flash",
1994	Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
1995	crate::cost_status::RouteBillingMode::Metered,
1996	);
1997	manager.store.save_turn(&priced)?;
1998
1999	let mut unknown = sample_turn(&thread.id, "turn_unknown", RuntimeTurnStatus::Completed);
2000	unknown.usage = Some(usage);
2001	set_test_turn_route(
2002	&mut unknown,
2003	ApiProvider::Openai,
2004	ApiProvider::Openai.as_str(),
2005	"gpt-5.5",
2006	Some(crate::pricing::UNCLASSIFIED_BILLING_SURFACE),
2007	crate::cost_status::RouteBillingMode::Unknown,
2008	);
2009	manager.store.save_turn(&unknown)?;
2010
2011	let report = manager
2012	.aggregate_usage(None, None, UsageGroupBy::Thread)
2013	.await?;
2014	assert!(report.totals.cost_usd > 0.0, "priced subtotal is retained");
2015	assert_eq!(report.totals.priced_turns, 1);
2016	assert_eq!(report.totals.unpriced_turns, 1);
2017	assert!(!report.totals.cost_complete);
2018	assert_eq!(report.totals.nonmetered_turns, 0);
2019	assert_eq!(report.totals.cache_write_tokens, 642);
2020	assert!(
2021	report
2022	.totals
2023	.unpriced_reasons
2024	.contains("unknown_billing_basis")
2025	);
2026	Ok(())
2027	}
2028
2029	#[tokio::test]
2030	async fn aggregate_usage_prices_slow_predispatch_turn_at_dispatch_boundary() -> Result<()> {
2031	let manager = test_manager(test_runtime_dir())?;
2032	let mut thread = sample_thread("thr_historical_pricing");
2033	thread.model = "claude-sonnet-5".to_string();
2034	manager.store.save_thread(&thread)?;
2035
2036	let usage = Usage {
2037	input_tokens: 1_000_000,
2038	output_tokens: 0,
2039	..Usage::default()
2040	};
2041	for (turn_id, created_at, dispatched_at) in [
2042	("turn_intro", "2026-08-31T23:59:58Z", "2026-08-31T23:59:59Z"),
2043	(
2044	"turn_slow_snapshot",
2045	"2026-08-31T23:59:59Z",
2046	"2026-09-01T00:00:00Z",
2047	),
2048	] {
2049	let mut turn = sample_turn(&thread.id, turn_id, RuntimeTurnStatus::Completed);
2050	turn.created_at = created_at.parse().expect("recorded turn time");
2051	turn.started_at = Some(turn.created_at);
2052	turn.usage = Some(usage.clone());
2053	set_test_turn_route(
2054	&mut turn,
2055	ApiProvider::Anthropic,
2056	ApiProvider::Anthropic.as_str(),
2057	"claude-sonnet-5",
2058	Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
2059	crate::cost_status::RouteBillingMode::Metered,
2060	);
2061	turn.effective_dispatched_at = Some(dispatched_at.parse().expect("dispatch time"));
2062	manager.store.save_turn(&turn)?;
2063	}
2064
2065	let report = manager
2066	.aggregate_usage(None, None, UsageGroupBy::Model)
2067	.await?;
2068
2069	assert_eq!(report.totals.turns, 2);
2070	assert!((report.totals.cost_usd - 5.0).abs() < f64::EPSILON);
2071	assert_eq!(report.buckets.len(), 1);
2072	assert!((report.buckets[0].cost_usd - 5.0).abs() < f64::EPSILON);
2073	Ok(())
2074	}
2075
2076	#[tokio::test]
2077	async fn aggregate_usage_prices_only_stepfun_payg_surface() -> Result<()> {
2078	let manager = test_manager(test_runtime_dir())?;
2079	let mut thread = sample_thread("thr_stepfun_surfaces");
2080	thread.model = "step-3.7-flash".to_string();
2081	manager.store.save_thread(&thread)?;
2082
2083	let usage = Usage {
2084	input_tokens: 1_000_000,
2085	output_tokens: 500_000,
2086	prompt_cache_hit_tokens: Some(250_000),
2087	..Usage::default()
2088	};
2089	for (turn_id, surface) in [
2090	(
2091	"turn_stepfun_payg",
2092	crate::pricing::STEPFUN_PAYG_BILLING_SURFACE,
2093	),
2094	(
2095	"turn_stepfun_plan",
2096	crate::pricing::STEPFUN_PLAN_BILLING_SURFACE,
2097	),
2098	] {
2099	let mut turn = sample_turn(&thread.id, turn_id, RuntimeTurnStatus::Completed);
2100	turn.usage = Some(usage.clone());
2101	set_test_turn_route(
2102	&mut turn,
2103	ApiProvider::Stepfun,
2104	ApiProvider::Stepfun.as_str(),
2105	"step-3.7-flash",
2106	Some(surface),
2107	if surface == crate::pricing::STEPFUN_PLAN_BILLING_SURFACE {
2108	crate::cost_status::RouteBillingMode::Subscription
2109	} else {
2110	crate::cost_status::RouteBillingMode::Metered
2111	},
2112	);
2113	manager.store.save_turn(&turn)?;
2114	}
2115
2116	let report = manager
2117	.aggregate_usage(None, None, UsageGroupBy::Provider)
2118	.await?;
2119	assert_eq!(report.totals.turns, 2);
2120	assert!((report.totals.cost_usd - 0.735).abs() < 1e-12);
2121	Ok(())
2122	}
2123
2124	#[tokio::test]
2125	async fn aggregate_usage_includes_exclusive_child_calls_and_zero_usage_receipts() -> Result<()> {
2126	let manager = test_manager(test_runtime_dir())?;
2127	let thread = sample_thread("thr_child_usage");
2128	manager.store.save_thread(&thread)?;
2129
2130	let mut turn = sample_turn(&thread.id, "turn_parent", RuntimeTurnStatus::Completed);
2131	turn.usage = Some(Usage {
2132	input_tokens: 100,
2133	output_tokens: 20,
2134	prompt_cache_hit_tokens: Some(40),
2135	prompt_cache_miss_tokens: Some(50),
2136	prompt_cache_write_tokens: Some(10),
2137	reasoning_tokens: Some(5),
2138	..Usage::default()
2139	});
2140	set_test_turn_route(
2141	&mut turn,
2142	ApiProvider::Deepseek,
2143	"deepseek-parent",
2144	"deepseek-v4-flash",
2145	Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
2146	crate::cost_status::RouteBillingMode::Metered,
2147	);
2148	turn.routed_usage
2149	.push(crate::cost_status::EffectiveRouteUsage {
2150	route: crate::cost_status::EffectiveRouteEnvelope {
2151	provider: ApiProvider::Deepseek,
2152	provider_identity: "deepseek-child".to_string(),
2153	model: "deepseek-v4-flash".to_string(),
2154	// A child call carries the surface its own dispatch classified;
2155	// the route audit will not price a metered route without one.
2156	billing_surface: Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE.to_string()),
2157	endpoint_fingerprint: None,
2158	billing_mode: crate::cost_status::RouteBillingMode::Metered,
2159	dispatched_at: turn.created_at,
2160	},
2161	usage: Usage {
2162	input_tokens: 30,
2163	output_tokens: 7,
2164	prompt_cache_hit_tokens: Some(10),
2165	prompt_cache_miss_tokens: Some(20),
2166	reasoning_tokens: Some(6),
2167	..Usage::default()
2168	},
2169	});
2170	turn.routed_usage
2171	.push(crate::cost_status::EffectiveRouteUsage {
2172	route: crate::cost_status::EffectiveRouteEnvelope {
2173	provider: ApiProvider::OpenaiCodex,
2174	provider_identity: "codex-oauth".to_string(),
2175	model: "gpt-5.5".to_string(),
2176	billing_surface: Some(
2177	crate::pricing::OAUTH_SUBSCRIPTION_BILLING_SURFACE.to_string(),
2178	),
2179	endpoint_fingerprint: None,
2180	billing_mode: crate::cost_status::RouteBillingMode::Subscription,
2181	dispatched_at: turn.created_at,
2182	},
2183	usage: Usage::default(),
2184	});
2185	manager.store.save_turn(&turn)?;
2186
2187	let report = manager
2188	.aggregate_usage(None, None, UsageGroupBy::Thread)
2189	.await?;
2190	assert_eq!(report.totals.turns, 3, "parent plus two child calls");
2191	assert_eq!(report.totals.input_tokens, 70);
2192	assert_eq!(report.totals.cached_tokens, 50);
2193	assert_eq!(report.totals.cache_write_tokens, 10);
2194	assert_eq!(report.totals.output_tokens, 27);
2195	assert_eq!(report.totals.reasoning_tokens, 11);
2196	assert_eq!(report.totals.priced_turns, 2);
2197	assert_eq!(report.totals.nonmetered_turns, 1);
2198	assert_eq!(report.totals.unpriced_turns, 0);
2199	assert!(report.totals.cost_complete);
2200	assert_eq!(report.totals.route_receipts.len(), 3);
2201	Ok(())
2202	}
2203
2204	#[tokio::test]
2205	async fn aggregate_usage_filters_each_call_by_its_dispatch_timestamp() -> Result<()> {
2206	let manager = test_manager(test_runtime_dir())?;
2207	let thread = sample_thread("thr_dispatch_window");
2208	manager.store.save_thread(&thread)?;
2209	let window = Utc::now();
2210	let mut turn = sample_turn(
2211	&thread.id,
2212	"turn_dispatch_window",
2213	RuntimeTurnStatus::Completed,
2214	);
2215	turn.created_at = window - chrono::Duration::days(2);
2216	turn.usage = Some(Usage {
2217	input_tokens: 100,
2218	..Usage::default()
2219	});
2220	set_test_turn_route(
2221	&mut turn,
2222	ApiProvider::Deepseek,
2223	"deepseek-parent",
2224	"deepseek-v4-flash",
2225	Some(crate::pricing::FIRST_PARTY_PAYG_BILLING_SURFACE),
2226	crate::cost_status::RouteBillingMode::Metered,
2227	);
2228	turn.routed_usage
2229	.push(crate::cost_status::EffectiveRouteUsage {
2230	route: crate::cost_status::EffectiveRouteEnvelope {
2231	provider: ApiProvider::Deepseek,
2232	provider_identity: "deepseek-child".to_string(),
2233	model: "deepseek-v4-flash".to_string(),
2234	billing_surface: None,
2235	endpoint_fingerprint: None,
2236	billing_mode: crate::cost_status::RouteBillingMode::Metered,
2237	dispatched_at: window,
2238	},
2239	usage: Usage {
2240	input_tokens: 33,
2241	..Usage::default()
2242	},
2243	});
2244	manager.store.save_turn(&turn)?;
2245
2246	let report = manager
2247	.aggregate_usage(
2248	Some(window - chrono::Duration::minutes(1)),
2249	Some(window + chrono::Duration::minutes(1)),
2250	UsageGroupBy::Thread,
2251	)
2252	.await?;
2253	assert_eq!(report.totals.turns, 1);
2254	assert_eq!(report.totals.input_tokens, 33);
2255	Ok(())
2256	}
2257
2258	#[tokio::test]
2259	async fn aggregate_usage_marks_bounded_journal_truncation_incomplete() -> Result<()> {
2260	let manager = test_manager(test_runtime_dir())?;
2261	let thread = sample_thread("thr_truncated_usage");
2262	manager.store.save_thread(&thread)?;
2263	let mut turn = sample_turn(
2264	&thread.id,
2265	"turn_truncated_usage",
2266	RuntimeTurnStatus::Completed,
2267	);
2268	turn.routed_usage_dropped_records = 2;
2269	manager.store.save_turn(&turn)?;
2270
2271	let report = manager
2272	.aggregate_usage(None, None, UsageGroupBy::Thread)
2273	.await?;
2274	assert_eq!(report.totals.dropped_usage_records, 2);
2275	assert_eq!(report.totals.unpriced_turns, 2);
2276	assert_eq!(report.totals.turns, 2);
2277	assert!(!report.totals.cost_complete);
2278	assert!(
2279	report
2280	.totals
2281	.unpriced_reasons
2282	.contains("runtime_usage_journal_truncated")
2283	);
2284	Ok(())
2285	}
2286
2287	#[test]
2288	fn runtime_usage_sink_survives_parent_terminal_and_restart_exactly_once() -> Result<()> {
2289	let _cost_scope = crate::cost_status::test_scope();
2290	let data_dir = test_runtime_dir();
2291	let manager = test_manager(data_dir.clone())?;
2292	let thread = sample_thread("thr_compaction_sink");
2293	manager.store.save_thread(&thread)?;
2294	let turn = sample_turn(
2295	&thread.id,
2296	"turn_compaction_sink",
2297	RuntimeTurnStatus::InProgress,
2298	);
2299	manager.store.save_turn(&turn)?;
2300	manager.register_runtime_usage_sink(&turn.id);
2301	let lease = crate::cost_status::acquire_runtime_usage_lease(&turn.id)
2302	.expect("active runtime owner lease");
2303	crate::cost_status::finish_runtime_usage_owner(&turn.id);
2304
2305	let route = crate::cost_status::EffectiveRouteEnvelope::capture(
2306	None,
2307	ApiProvider::Deepseek,
2308	"deepseek-compaction",
2309	"deepseek-v4-flash",
2310	Some(ApiProvider::Deepseek.default_base_url()),
2311	Utc::now(),
2312	);
2313	crate::cost_status::report_effective_route_for_runtime(
2314	crate::cost_status::scope_token(),
2315	Some(&turn.id),
2316	"compaction:response:test",
2317	&route,
2318	&Usage {
2319	input_tokens: 17,
2320	output_tokens: 3,
2321	..Usage::default()
2322	},
2323	);
2324	crate::cost_status::report_effective_route_for_runtime(
2325	crate::cost_status::scope_token(),
2326	Some(&turn.id),
2327	"compaction:response:test",
2328	&route,
2329	&Usage {
2330	input_tokens: 17,
2331	output_tokens: 3,
2332	..Usage::default()
2333	},
2334	);
2335	drop(lease);
2336
2337	let reopened = test_manager(data_dir.clone())?;
2338	reopened.register_runtime_usage_sink(&turn.id);
2339	crate::cost_status::report_effective_route_for_runtime(
2340	crate::cost_status::scope_token(),
2341	Some(&turn.id),
2342	"compaction:response:test",
2343	&route,
2344	&Usage {
2345	input_tokens: 17,
2346	output_tokens: 3,
2347	..Usage::default()
2348	},
2349	);
2350	crate::cost_status::finish_runtime_usage_owner(&turn.id);
2351	let reopened = test_manager(data_dir)?;
2352	let persisted = reopened.store.load_turn(&turn.id)?;
2353	assert_eq!(persisted.routed_usage.len(), 1);
2354	assert_eq!(persisted.routed_usage[0].usage.input_tokens, 17);
2355	assert!(
2356	crate::cost_status::take_runtime_usage(&turn.id)
2357	.records
2358	.is_empty(),
2359	"successful synchronous persistence must not duplicate into fallback"
2360	);
2361	assert!(
2362	crate::cost_status::drain().is_empty(),
2363	"runtime-owned usage must not duplicate into the TUI pool"
2364	);
2365	Ok(())
2366	}
2367
2368	#[test]
2369	fn routed_usage_append_is_bounded_and_idempotent_for_every_delivery_path() {
2370	let mut turn = sample_turn(
2371	"thr_all_paths",
2372	"turn_all_paths",
2373	RuntimeTurnStatus::InProgress,
2374	);
2375	let route = crate::cost_status::EffectiveRouteEnvelope::capture(
2376	None,
2377	ApiProvider::Deepseek,
2378	"deepseek-primary",
2379	"deepseek-v4-flash",
2380	Some(ApiProvider::Deepseek.default_base_url()),
2381	Utc::now(),
2382	);
2383	for index in 0..90 {
2384	let path = match index % 3 {
2385	0 => "sink",
2386	1 => "mailbox",
2387	_ => "fallback",
2388	};
2389	assert!(append_routed_usage_record(
2390	&mut turn,
2391	&format!("{path}:response:{index}"),
2392	crate::cost_status::EffectiveRouteUsage {
2393	route: route.clone(),
2394	usage: Usage {
2395	input_tokens: index,
2396	..Usage::default()
2397	},
2398	},
2399	));
2400	}
2401
2402	assert_eq!(turn.routed_usage.len(), MAX_ROUTED_USAGE_RECORDS_PER_TURN);
2403	assert_eq!(turn.routed_usage_dropped_records, 26);
2404	assert_eq!(turn.routed_usage_source_ids.len(), 90);
2405	let before = turn.clone();
2406	assert!(!append_routed_usage_record(
2407	&mut turn,
2408	"sink:response:0",
2409	crate::cost_status::EffectiveRouteUsage {
2410	route,
2411	usage: Usage {
2412	input_tokens: 999,
2413	..Usage::default()
2414	},
2415	},
2416	));
2417	assert_eq!(turn.routed_usage, before.routed_usage);
2418	assert_eq!(
2419	turn.routed_usage_dropped_records,
2420	before.routed_usage_dropped_records
2421	);
2422	}
2423
2424	#[tokio::test]
2425	async fn aggregate_usage_fails_closed_for_legacy_reconstructed_route() -> Result<()> {
2426	let manager = test_manager(test_runtime_dir())?;
2427	let thread = sample_thread("thr_legacy_cost");
2428	manager.store.save_thread(&thread)?;
2429	let mut turn = sample_turn(&thread.id, "turn_legacy_cost", RuntimeTurnStatus::Completed);
2430	turn.usage = Some(Usage {
2431	input_tokens: 1_000,
2432	output_tokens: 10,
2433	..Usage::default()
2434	});
2435	// These mutable-era fields are intentionally insufficient: no exact
2436	// identity, billing mode, or dispatch timestamp was persisted.
2437	turn.effective_provider = Some(ApiProvider::Deepseek.as_str().to_string());
2438	turn.effective_model = Some("deepseek-v4-flash".to_string());
2439	manager.store.save_turn(&turn)?;
2440
2441	let report = manager
2442	.aggregate_usage(None, None, UsageGroupBy::Thread)
2443	.await?;
2444	assert_eq!(report.totals.cost_usd, 0.0);
2445	assert_eq!(report.totals.unpriced_turns, 1);
2446	assert!(
2447	report
2448	.totals
2449	.unpriced_reasons
2450	.contains("unknown_provider_route")
2451	);
2452	assert!(report.totals.route_receipts.is_empty());
2453	Ok(())
2454	}
2455
2456	#[test]
2457	fn runtime_usage_accumulators_saturate_tokens_and_currency() {
2458	let mut total = f64::MAX;
2459	saturating_add_usd(&mut total, f64::MAX);
2460	assert_eq!(total, f64::MAX);
2461
2462	let thread = sample_thread("thr_saturating");
2463	let turn = sample_turn(&thread.id, "turn_saturating", RuntimeTurnStatus::Completed);
2464	let mut totals = UsageTotals {
2465	input_tokens: u64::MAX,
2466	output_tokens: u64::MAX,
2467	cached_tokens: u64::MAX,
2468	reasoning_tokens: u64::MAX,
2469	reasoning_replay_tokens: u64::MAX,
2470	cache_write_tokens: u64::MAX,
2471	..UsageTotals::default()
2472	};
2473	let mut buckets = std::collections::BTreeMap::new();
2474	accumulate_runtime_usage_record(
2475	&mut totals,
2476	&mut buckets,
2477	UsageGroupBy::Thread,
2478	None,
2479	&Usage {
2480	input_tokens: 1,
2481	output_tokens: 1,
2482	prompt_cache_hit_tokens: Some(1),
2483	prompt_cache_write_tokens: Some(1),
2484	reasoning_tokens: Some(1),
2485	reasoning_replay_tokens: Some(1),
2486	..Usage::default()
2487	},
2488	&turn,
2489	&thread,
2490	);
2491	assert_eq!(totals.input_tokens, u64::MAX);
2492	assert_eq!(totals.output_tokens, u64::MAX);
2493	assert_eq!(totals.cached_tokens, u64::MAX);
2494	assert_eq!(totals.reasoning_tokens, u64::MAX);
2495	assert_eq!(totals.reasoning_replay_tokens, u64::MAX);
2496	assert_eq!(totals.cache_write_tokens, u64::MAX);
2497	}
2498
2499	fn sample_item(turn_id: &str, item_id: &str, status: TurnItemLifecycleStatus) -> TurnItemRecord {
2500	TurnItemRecord {
2501	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
2502	id: item_id.to_string(),
2503	turn_id: turn_id.to_string(),
2504	kind: TurnItemKind::Status,
2505	status,
2506	summary: "sample item".to_string(),
2507	detail: None,
2508	metadata: None,
2509	artifact_refs: Vec::new(),
2510	started_at: Some(Utc::now()),
2511	ended_at: None,
2512	}
2513	}
2514
2515	async fn install_mock_engine(
2516	manager: &RuntimeThreadManager,
2517	thread_id: &str,
2518	) -> crate::core::engine::MockEngineHandle {
2519	let harness = mock_engine_handle();
2520	manager
2521	.install_test_engine(thread_id, harness.handle.clone())
2522	.await
2523	.expect("install mock engine");
2524	harness
2525	}
2526
2527	async fn wait_for_sender_strong_count<T>(
2528	sender: &tokio::sync::mpsc::Sender<T>,
2529	minimum: usize,
2530	) -> Result<()> {
2531	tokio::time::timeout(Duration::from_secs(2), async {
2532	while sender.strong_count() < minimum {
2533	tokio::task::yield_now().await;
2534	}
2535	})
2536	.await
2537	.map_err(\|_\| anyhow!("Timed out waiting for mailbox reservation"))?;
2538	Ok(())
2539	}
2540
2541	async fn wait_for_terminal_turn(
2542	manager: &RuntimeThreadManager,
2543	turn_id: &str,
2544	timeout: Duration,
2545	) -> Result<TurnRecord> {
2546	let deadline = Instant::now() + timeout;
2547	loop {
2548	let turn = manager.store.load_turn(turn_id)?;
2549	let terminal = matches!(
2550	turn.status,
2551	RuntimeTurnStatus::Completed
2552	\| RuntimeTurnStatus::Failed
2553	\| RuntimeTurnStatus::Interrupted
2554	\| RuntimeTurnStatus::Canceled
2555	);
2556	if terminal {
2557	let receipt_is_durable =
2558	manager
2559	.events_since(&turn.thread_id, None)?
2560	.iter()
2561	.any(\|event\| {
2562	event.event == "turn.completed" && event.turn_id.as_deref() == Some(turn_id)
2563	});
2564	let claim_is_clear = manager
2565	.active_turn_flags(&turn.thread_id, turn_id)
2566	.await
2567	.is_none();
2568	if receipt_is_durable && claim_is_clear {
2569	return Ok(turn);
2570	}
2571	}
2572	if Instant::now() >= deadline {
2573	bail!("Timed out waiting for turn {turn_id}");
2574	}
2575	sleep(Duration::from_millis(20)).await;
2576	}
2577	}
2578
2579	#[test]
2580	fn store_load_thread_rejects_newer_schema_version() {
2581	let dir = test_runtime_dir();
2582	let store = RuntimeThreadStore::open(dir.clone()).expect("open store");
2583
2584	// Construct a thread record persisted with a future schema version.
2585	let mut thread = sample_thread("thr_future");
2586	thread.schema_version = CURRENT_RUNTIME_SCHEMA_VERSION + 1;
2587
2588	// Bypass save_thread (which would respect our local schema_version)
2589	// by writing the JSON directly so we can simulate a future writer.
2590	let path = store.threads_dir.join(format!("{}.json", thread.id));
2591	std::fs::create_dir_all(path.parent().unwrap()).expect("mkdirs");
2592	let payload = serde_json::to_string(&thread).expect("serialize thread");
2593	std::fs::write(&path, payload).expect("write thread");
2594
2595	let err = store
2596	.load_thread(&thread.id)
2597	.expect_err("load_thread must reject newer schema");
2598	let msg = format!("{err:#}");
2599	assert!(msg.contains("newer than supported"), "got: {msg}");
2600
2601	// Cleanup so we don't leak across tests.
2602	let _ = std::fs::remove_dir_all(dir);
2603	}
2604
2605	#[test]
2606	fn runtime_event_sequences_serialize_across_real_processes() -> Result<()> {
2607	let dir = test_runtime_dir();
2608	let store = RuntimeThreadStore::open(dir.clone())?;
2609	let thread_id = "thr_process_writers";
2610	let start = dir.join("process-writers.start");
2611	let writer_count = 4_u64;
2612	let events_per_writer = 8_u64;
2613	let mut children = Vec::new();
2614
2615	for worker in 0..writer_count {
2616	let signal = dir.join(format!("process-writer-{worker}.ready"));
2617	children.push((
2618	spawn_runtime_event_child("writer", &dir, thread_id, &signal),
2619	signal,
2620	));
2621	}
2622	for (_, signal) in &children {
2623	wait_for_runtime_event_test_file(signal, "event writer readiness");
2624	}
2625	// The children already opened their stores before this barrier. The old
2626	// store-local sequence cache therefore allocated duplicates deterministically.
2627	std::fs::write(&start, b"go")?;
2628	for (child, _) in children {
2629	child.wait_success("cross-process event writer");
2630	}
2631
2632	let events = store.events_since(thread_id, None)?;
2633	let expected = writer_count * events_per_writer;
2634	assert_eq!(u64::try_from(events.len())?, expected);
2635	assert_eq!(
2636	events.iter().map(\|event\| event.seq).collect::<Vec<_>>(),
2637	(1..=expected).collect::<Vec<_>>()
2638	);
2639	let identities = events
2640	.iter()
2641	.map(\|event\| {
2642	(
2643	event.payload["worker"].as_str().unwrap().to_string(),
2644	event.payload["index"].as_u64().unwrap(),
2645	)
2646	})
2647	.collect::<HashSet<_>>();
2648	assert_eq!(u64::try_from(identities.len())?, expected);
2649	let runtime = tokio::runtime::Builder::new_current_thread()
2650	.enable_all()
2651	.build()?;
2652	assert_eq!(runtime.block_on(store.current_seq())?, expected);
2653	assert_eq!(
2654	load_runtime_store_state(&store.state_path)?.next_seq,
2655	expected + 1
2656	);
2657
2658	std::fs::remove_dir_all(dir)?;
2659	Ok(())
2660	}
2661
2662	#[test]
2663	fn runtime_event_lock_timeout_is_non_mutating_and_holder_death_releases_lock() -> Result<()> {
2664	let dir = test_runtime_dir();
2665	let store = RuntimeThreadStore::open(dir.clone())?;
2666	let thread_id = "thr_process_holder";
2667	let signal = dir.join("process-holder.ready");
2668	let child = spawn_runtime_event_child("holder", &dir, thread_id, &signal);
2669	wait_for_runtime_event_test_file(&signal, "held Runtime event lock");
2670
2671	let state_before = std::fs::read(&store.state_path)?;
2672	let event_path = store.events_path(thread_id)?;
2673	let started = Instant::now();
2674	let error = store
2675	.append_event_transaction(
2676	thread_id.to_string(),
2677	None,
2678	None,
2679	"must.timeout".to_string(),
2680	json!({}),
2681	Duration::from_millis(75),
2682	)
2683	.expect_err("contended Runtime event transaction must time out");
2684	let elapsed = started.elapsed();
2685	assert!(error.downcast_ref::<RuntimeEventLockTimeout>().is_some());
2686	assert!(
2687	elapsed >= Duration::from_millis(75),
2688	"returned too early: {elapsed:?}"
2689	);
2690	assert!(
2691	elapsed < Duration::from_secs(2),
2692	"timeout was unbounded: {elapsed:?}"
2693	);
2694	assert_eq!(std::fs::read(&store.state_path)?, state_before);
2695	assert!(!event_path.exists());
2696
2697	let status = child.kill_and_reap()?;
2698	assert!(
2699	!status.success(),
2700	"killed lock holder unexpectedly succeeded"
2701	);
2702	let runtime = tokio::runtime::Builder::new_current_thread()
2703	.enable_all()
2704	.build()?;
2705	let committed = runtime.block_on(store.append_event(
2706	thread_id,
2707	None,
2708	None,
2709	"after.holder.death",
2710	json!({}),
2711	))?;
2712	assert_eq!(committed.seq, 1);
2713
2714	std::fs::remove_dir_all(dir)?;
2715	Ok(())
2716	}
2717
2718	#[test]
2719	fn killed_torn_append_is_repaired_and_preserves_reserved_gap() -> Result<()> {
2720	let dir = test_runtime_dir();
2721	let store = RuntimeThreadStore::open(dir.clone())?;
2722	let thread_id = "thr_process_torn";
2723	let signal = dir.join("process-torn.ready");
2724	let child = spawn_runtime_event_child("torn", &dir, thread_id, &signal);
2725	wait_for_runtime_event_test_file(&signal, "durable torn Runtime event tail");
2726	let status = child.kill_and_reap()?;
2727	assert!(
2728	!status.success(),
2729	"killed torn writer unexpectedly succeeded"
2730	);
2731
2732	let runtime = tokio::runtime::Builder::new_current_thread()
2733	.enable_all()
2734	.build()?;
2735	let committed = runtime.block_on(store.append_event(
2736	thread_id,
2737	None,
2738	None,
2739	"after.torn.death",
2740	json!({}),
2741	))?;
2742	assert_eq!(committed.seq, 2);
2743	assert_eq!(runtime.block_on(store.current_seq())?, 2);
2744	let events = store.events_since(thread_id, None)?;
2745	assert_eq!(events.len(), 1);
2746	assert_eq!(events[0].seq, 2);
2747	let raw = std::fs::read(store.events_path(thread_id)?)?;
2748	assert_eq!(raw.last(), Some(&b'\n'));
2749	for line in raw
2750	.split(\|byte\| *byte == b'\n')
2751	.filter(\|line\| !line.is_empty())
2752	{
2753	serde_json::from_slice::<RuntimeEventRecord>(line)?;
2754	}
2755
2756	std::fs::remove_dir_all(dir)?;
2757	Ok(())
2758	}
2759
2760	#[test]
2761	fn event_reader_never_observes_cross_process_rollback_candidate() -> Result<()> {
2762	let dir = test_runtime_dir();
2763	let store = RuntimeThreadStore::open(dir.clone())?;
2764	let thread_id = "thr_process_phantom";
2765	let signal = dir.join("process-phantom.ready");
2766	let child = spawn_runtime_event_child("phantom", &dir, thread_id, &signal);
2767	wait_for_runtime_event_test_file(&signal, "visible rollback candidate");
2768
2769	let started = Instant::now();
2770	assert!(store.events_since(thread_id, None)?.is_empty());
2771	assert!(
2772	started.elapsed() >= Duration::from_millis(200),
2773	"reader did not wait for rollback disposition"
2774	);
2775	child.wait_success("phantom rollback child");
2776	assert!(store.events_since(thread_id, None)?.is_empty());
2777
2778	std::fs::remove_dir_all(dir)?;
2779	Ok(())
2780	}
2781
2782	#[tokio::test(flavor = "current_thread")]
2783	async fn event_history_lock_wait_does_not_block_tokio_worker() -> Result<()> {
2784	let dir = test_runtime_dir();
2785	let manager = test_manager(dir.clone())?;
2786	let thread_id = "thr_async_history_wait";
2787	let signal = dir.join("async-history-holder.ready");
2788	let child = spawn_runtime_event_child("holder", &dir, thread_id, &signal);
2789	wait_for_runtime_event_test_file(&signal, "async history lock holder");
2790
2791	let history = manager.events_since_async(thread_id, None);
2792	tokio::pin!(history);
2793	tokio::select! {
2794	result = &mut history => panic!("history lock wait returned early: {result:?}"),
2795	() = sleep(Duration::from_millis(50)) => {}
2796	}
2797	let status = child.kill_and_reap()?;
2798	assert!(!status.success(), "killed history lock holder succeeded");
2799	assert!(history.await?.is_empty());
2800
2801	std::fs::remove_dir_all(dir)?;
2802	Ok(())
2803	}
2804
2805	#[test]
2806	fn runtime_event_sequence_exhaustion_fails_before_mutation() -> Result<()> {
2807	let dir = test_runtime_dir();
2808	let store = RuntimeThreadStore::open(dir.clone())?;
2809	store.with_event_transaction(Duration::from_secs(1), \|\| {
2810	write_json_atomic(
2811	&store.state_path,
2812	&RuntimeStoreState {
2813	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
2814	next_seq: u64::MAX,
2815	},
2816	)
2817	})?;
2818	let state_before = std::fs::read(&store.state_path)?;
2819	let thread_id = "thr_sequence_exhausted";
2820	let error = store
2821	.append_event_transaction(
2822	thread_id.to_string(),
2823	None,
2824	None,
2825	"must.not.append".to_string(),
2826	json!({}),
2827	Duration::from_secs(1),
2828	)
2829	.expect_err("u64 sequence exhaustion must fail closed");
2830	assert!(
2831	format!("{error:#}").contains("Runtime event sequence exhausted"),
2832	"unexpected exhaustion error: {error:#}"
2833	);
2834	assert_eq!(std::fs::read(&store.state_path)?, state_before);
2835	assert!(!store.events_path(thread_id)?.exists());
2836
2837	std::fs::remove_dir_all(dir)?;
2838	Ok(())
2839	}
2840
2841	#[tokio::test]
2842	async fn store_open_truncates_only_torn_final_event_record_and_preserves_sequence_gap() {
2843	let dir = test_runtime_dir();
2844	let store = RuntimeThreadStore::open(dir.clone()).expect("open store");
2845	let first = store
2846	.append_event("thr_torn_tail", None, None, "first", json!({ "value": 1 }))
2847	.await
2848	.expect("append first event");
2849	let torn = store
2850	.append_event("thr_torn_tail", None, None, "torn", json!({ "value": 2 }))
2851	.await
2852	.expect("append event to tear");
2853	let path = store.events_path("thr_torn_tail").expect("event path");
2854	let original_len = std::fs::metadata(&path).expect("event metadata").len();
2855	assert!(original_len > 16);
2856	std::fs::OpenOptions::new()
2857	.write(true)
2858	.open(&path)
2859	.expect("open event log for simulated crash")
2860	.set_len(original_len - 16)
2861	.expect("tear final event record");
2862	drop(store);
2863
2864	let reopened = RuntimeThreadStore::open(dir.clone()).expect("repair torn event tail");
2865	let replay = reopened
2866	.events_since("thr_torn_tail", None)
2867	.expect("replay repaired event log");
2868	assert_eq!(replay.len(), 1);
2869	assert_eq!(replay[0].seq, first.seq);
2870
2871	let after_repair = reopened
2872	.append_event(
2873	"thr_torn_tail",
2874	None,
2875	None,
2876	"after_repair",
2877	json!({ "value": 3 }),
2878	)
2879	.await
2880	.expect("append after repair");
2881	assert_eq!(after_repair.seq, torn.seq.saturating_add(1));
2882	assert_eq!(
2883	reopened
2884	.events_since("thr_torn_tail", None)
2885	.expect("replay repaired and appended events")
2886	.iter()
2887	.map(\|event\| event.seq)
2888	.collect::<Vec<_>>(),
2889	vec![first.seq, after_repair.seq]
2890	);
2891
2892	let _ = std::fs::remove_dir_all(dir);
2893	}
2894
2895	#[tokio::test]
2896	async fn store_open_treats_valid_json_without_newline_as_uncommitted_append() {
2897	let dir = test_runtime_dir();
2898	let store = RuntimeThreadStore::open(dir.clone()).expect("open store");
2899	let committed = store
2900	.append_event(
2901	"thr_missing_commit_marker",
2902	None,
2903	None,
2904	"committed",
2905	json!({ "value": 1 }),
2906	)
2907	.await
2908	.expect("append committed event");
2909	let uncommitted = store
2910	.append_event(
2911	"thr_missing_commit_marker",
2912	None,
2913	None,
2914	"missing_marker",
2915	json!({ "value": 2 }),
2916	)
2917	.await
2918	.expect("append event whose commit marker will be removed");
2919	let path = store
2920	.events_path("thr_missing_commit_marker")
2921	.expect("event path");
2922	let encoded = std::fs::read(&path).expect("read event log");
2923	assert_eq!(encoded.last(), Some(&b'\n'));
2924	let without_marker = &encoded[..encoded.len() - 1];
2925	let final_json = without_marker
2926	.rsplit(\|byte\| *byte == b'\n')
2927	.next()
2928	.expect("final JSON record");
2929	serde_json::from_slice::<RuntimeEventRecord>(final_json)
2930	.expect("the unterminated tail must otherwise be valid JSON");
2931	std::fs::OpenOptions::new()
2932	.write(true)
2933	.open(&path)
2934	.expect("open event log for simulated crash")
2935	.set_len(u64::try_from(without_marker.len()).expect("event log length fits u64"))
2936	.expect("remove only the newline commit marker");
2937	drop(store);
2938
2939	let reopened = RuntimeThreadStore::open(dir.clone()).expect("repair uncommitted event tail");
2940	let replay = reopened
2941	.events_since("thr_missing_commit_marker", None)
2942	.expect("replay repaired event log");
2943	assert_eq!(replay.len(), 1);
2944	assert_eq!(replay[0].seq, committed.seq);
2945
2946	let after_repair = reopened
2947	.append_event(
2948	"thr_missing_commit_marker",
2949	None,
2950	None,
2951	"after_repair",
2952	json!({ "value": 3 }),
2953	)
2954	.await
2955	.expect("append after repair");
2956	assert_eq!(after_repair.seq, uncommitted.seq.saturating_add(1));
2957
2958	let _ = std::fs::remove_dir_all(dir);
2959	}
2960
2961	#[tokio::test]
2962	async fn store_open_does_not_discard_newline_terminated_malformed_event() {
2963	let dir = test_runtime_dir();
2964	let store = RuntimeThreadStore::open(dir.clone()).expect("open store");
2965	store
2966	.append_event("thr_bad_tail", None, None, "valid", json!({}))
2967	.await
2968	.expect("append valid event");
2969	let path = store.events_path("thr_bad_tail").expect("event path");
2970	let mut file = std::fs::OpenOptions::new()
2971	.append(true)
2972	.open(&path)
2973	.expect("open event log");
2974	std::io::Write::write_all(&mut file, b"{malformed-but-terminated}\n")
2975	.expect("append malformed event");
2976	std::io::Write::flush(&mut file).expect("flush malformed event");
2977	drop(file);
2978	drop(store);
2979
2980	let reopened = RuntimeThreadStore::open(dir.clone()).expect("open terminated event log");
2981	let error = reopened
2982	.events_since("thr_bad_tail", None)
2983	.expect_err("terminated malformed event must fail closed");
2984	assert!(
2985	format!("{error:#}").contains("Failed to parse event line"),
2986	"unexpected replay error: {error:#}"
2987	);
2988
2989	let _ = std::fs::remove_dir_all(dir);
2990	}
2991
2992	#[tokio::test]
2993	async fn event_replay_is_bounded_and_tail_cursor_skips_only_omitted_history() -> Result<()> {
2994	let manager = test_manager(test_runtime_dir())?;
2995	let thread_id = "thr_bounded_replay";
2996	let path = manager.store.events_path(thread_id)?;
2997	let mut encoded = Vec::new();
2998	for seq in 1_u64..=600 {
2999	let event = RuntimeEventRecord {
3000	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
3001	seq,
3002	timestamp: Utc::now(),
3003	thread_id: thread_id.to_string(),
3004	turn_id: None,
3005	item_id: None,
3006	event: "test.event".to_string(),
3007	payload: json!({ "seq": seq }),
3008	};
3009	serde_json::to_writer(&mut encoded, &event)?;
3010	encoded.push(b'\n');
3011	}
3012	std::fs::write(path, encoded)?;
3013
3014	let mut full = manager.replay_events(thread_id, None, None).await?;
3015	assert_eq!(full.base_seq, 0);
3016	let mut full_sequences = Vec::new();
3017	while let Some(batch) = full.batches.recv().await {
3018	let batch = batch.map_err(anyhow::Error::msg)?;
3019	assert!(
3020	batch.len() <= RUNTIME_EVENT_REPLAY_BATCH_SIZE,
3021	"replay batch exceeded its memory bound"
3022	);
3023	full_sequences.extend(batch.into_iter().map(\|event\| event.seq));
3024	}
3025	assert_eq!(full_sequences, (1_u64..=600).collect::<Vec<_>>());
3026
3027	let mut tail = manager.replay_events(thread_id, None, Some(10)).await?;
3028	assert_eq!(tail.base_seq, 590);
3029	let mut tail_sequences = Vec::new();
3030	while let Some(batch) = tail.batches.recv().await {
3031	tail_sequences.extend(
3032	batch
3033	.map_err(anyhow::Error::msg)?
3034	.into_iter()
3035	.map(\|event\| event.seq),
3036	);
3037	}
3038	assert_eq!(tail_sequences, (591_u64..=600).collect::<Vec<_>>());
3039
3040	let mut empty_tail = manager.replay_events(thread_id, None, Some(0)).await?;
3041	assert_eq!(empty_tail.base_seq, 600);
3042	assert!(empty_tail.batches.recv().await.is_none());
3043	assert!(
3044	manager
3045	.replay_events(
3046	thread_id,
3047	None,
3048	Some(MAX_RUNTIME_EVENT_REPLAY_TAIL.saturating_add(1)),
3049	)
3050	.await
3051	.is_err()
3052	);
3053	Ok(())
3054	}
3055
3056	#[tokio::test]
3057	async fn event_reader_ignores_an_unterminated_live_append_tail() -> Result<()> {
3058	let dir = test_runtime_dir();
3059	let store = RuntimeThreadStore::open(dir.clone())?;
3060	let committed = store
3061	.append_event("thr_live_tail", None, None, "committed", json!({}))
3062	.await?;
3063	let path = store.events_path("thr_live_tail")?;
3064	let mut file = std::fs::OpenOptions::new().append(true).open(path)?;
3065	std::io::Write::write_all(&mut file, br#"{"schema_version":2,"seq":999"#)?;
3066	std::io::Write::flush(&mut file)?;
3067
3068	let replay = store.events_since("thr_live_tail", None)?;
3069	assert_eq!(replay.len(), 1);
3070	assert_eq!(replay[0].seq, committed.seq);
3071	let _ = std::fs::remove_dir_all(dir);
3072	Ok(())
3073	}
3074
3075	#[cfg(unix)]
3076	#[test]
3077	fn store_open_rejects_symlinked_state_file() {
3078	let dir = test_runtime_dir();
3079	std::fs::create_dir_all(&dir).expect("mkdir runtime dir");
3080	let target = dir.join("outside-state.json");
3081	let link = dir.join("state.json");
3082	std::fs::write(
3083	&target,
3084	serde_json::to_string(&RuntimeStoreState::default()).unwrap(),
3085	)
3086	.expect("write target");
3087	std::os::unix::fs::symlink(&target, &link).expect("symlink state");
3088
3089	let err = RuntimeThreadStore::open(dir.clone()).expect_err("symlink state should fail");
3090	assert!(format!("{err:#}").contains("must not be a symlink"));
3091
3092	let _ = std::fs::remove_dir_all(dir);
3093	}
3094
3095	#[cfg(unix)]
3096	#[test]
3097	fn event_append_rollback_handle_rejects_a_swapped_symlink_target() -> Result<()> {
3098	let dir = test_runtime_dir();
3099	std::fs::create_dir_all(&dir)?;
3100	let outside = dir.join("outside.jsonl");
3101	let events_path = dir.join("events.jsonl");
3102	std::fs::write(&outside, b"must-remain-intact\n")?;
3103	std::os::unix::fs::symlink(&outside, &events_path)?;
3104
3105	let error = open_runtime_store_file(&events_path, "Runtime event rollback", \|options\| {
3106	options.write(true);
3107	})
3108	.expect_err("rollback followed a swapped symlink target");
3109	assert!(format!("{error:#}").contains("must not be a symlink"));
3110	assert_eq!(std::fs::read(&outside)?, b"must-remain-intact\n");
3111	std::fs::remove_dir_all(&dir)?;
3112	Ok(())
3113	}
3114
3115	#[test]
3116	fn store_open_rejects_root_traversal() {
3117	let dir = test_runtime_dir();
3118	let bad_root = dir.join("runtime").join("..").join("outside");
3119
3120	let err = RuntimeThreadStore::open(bad_root).expect_err("traversal root should fail");
3121	assert!(format!("{err:#}").contains("cannot contain '..'"));
3122
3123	let _ = std::fs::remove_dir_all(dir);
3124	}
3125
3126	#[cfg(unix)]
3127	#[test]
3128	fn store_open_rejects_symlinked_store_directory() {
3129	let dir = test_runtime_dir();
3130	std::fs::create_dir_all(&dir).expect("mkdir runtime dir");
3131	let outside = dir.join("outside-items");
3132	let link = dir.join("items");
3133	std::fs::create_dir_all(&outside).expect("mkdir outside");
3134	std::os::unix::fs::symlink(&outside, &link).expect("symlink items dir");
3135
3136	let err = RuntimeThreadStore::open(dir.clone()).expect_err("symlink items dir should fail");
3137	assert!(
3138	format!("{err:#}").contains("directory must not be a symlink"),
3139	"got: {err:#}"
3140	);
3141
3142	let _ = std::fs::remove_dir_all(dir);
3143	}
3144
3145	#[cfg(unix)]
3146	#[test]
3147	fn store_list_items_rejects_symlinked_item_file() {
3148	let dir = test_runtime_dir();
3149	let store = RuntimeThreadStore::open(dir.clone()).expect("open store");
3150	let item = sample_item("turn_link", "item_link", TurnItemLifecycleStatus::Completed);
3151	let target = dir.join("outside-item.json");
3152	let link = store.items_dir.join(format!("{}.json", item.id));
3153	std::fs::write(&target, serde_json::to_string(&item).unwrap()).expect("write target");
3154	std::os::unix::fs::symlink(&target, &link).expect("symlink item");
3155
3156	let err = store
3157	.list_items_for_turn(&item.turn_id)
3158	.expect_err("symlink item should fail");
3159	assert!(format!("{err:#}").contains("must not be a symlink"));
3160
3161	let _ = std::fs::remove_dir_all(dir);
3162	}
3163
3164	#[cfg(unix)]
3165	#[test]
3166	fn store_list_items_rejects_swapped_symlinked_store_directory() {
3167	let dir = test_runtime_dir();
3168	let store = RuntimeThreadStore::open(dir.clone()).expect("open store");
3169	let outside = dir.join("outside-items");
3170	std::fs::create_dir_all(&outside).expect("mkdir outside");
3171	std::fs::remove_dir_all(&store.items_dir).expect("remove items dir");
3172	std::os::unix::fs::symlink(&outside, &store.items_dir).expect("symlink items dir");
3173
3174	let err = store
3175	.list_items_for_turn("turn_link")
3176	.expect_err("swapped symlink items dir should fail");
3177	assert!(
3178	format!("{err:#}").contains("directory must not be a symlink"),
3179	"got: {err:#}"
3180	);
3181
3182	let _ = std::fs::remove_dir_all(dir);
3183	}
3184
3185	#[test]
3186	fn store_load_thread_defaults_missing_session_id() {
3187	let dir = test_runtime_dir();
3188	let store = RuntimeThreadStore::open(dir.clone()).expect("open store");
3189	let thread = sample_thread("thr_legacy_session");
3190	let path = store.threads_dir.join(format!("{}.json", thread.id));
3191	std::fs::create_dir_all(path.parent().unwrap()).expect("mkdirs");
3192	let mut payload = serde_json::to_value(&thread).expect("serialize thread");
3193	payload
3194	.as_object_mut()
3195	.expect("thread object")
3196	.remove("session_id");
3197	std::fs::write(
3198	&path,
3199	serde_json::to_string(&payload).expect("encode thread"),
3200	)
3201	.expect("write thread");
3202
3203	let loaded = store
3204	.load_thread(&thread.id)
3205	.expect("legacy thread should load");
3206	assert_eq!(loaded.session_id, None);
3207
3208	let _ = std::fs::remove_dir_all(dir);
3209	}
3210
3211	#[tokio::test]
3212	async fn seed_thread_keeps_tool_results_on_preceding_turn() -> Result<()> {
3213	let dir = test_runtime_dir();
3214	let manager = test_manager(dir.clone())?;
3215	let thread = sample_thread("thr_seed_blocks");
3216	manager.store.save_thread(&thread)?;
3217	let messages = vec![
3218	Message {
3219	role: "user".to_string(),
3220	content: vec![ContentBlock::Text {
3221	text: "check the files".to_string(),
3222	cache_control: None,
3223	}],
3224	},
3225	Message {
3226	role: "assistant".to_string(),
3227	content: vec![
3228	ContentBlock::Thinking {
3229	thinking: "need a tool".to_string(),
3230	signature: Some("sig-1".to_string()),
3231	},
3232	ContentBlock::ToolUse {
3233	id: "tool-1".to_string(),
3234	name: "shell".to_string(),
3235	input: json!({ "cmd": "one" }),
3236	caller: None,
3237	},
3238	ContentBlock::ToolUse {
3239	id: "tool-2".to_string(),
3240	name: "shell".to_string(),
3241	input: json!({ "cmd": "two" }),
3242	caller: None,
3243	},
3244	],
3245	},
3246	Message {
3247	role: "user".to_string(),
3248	content: vec![ContentBlock::ToolResult {
3249	tool_use_id: "tool-1".to_string(),
3250	content: "one".to_string(),
3251	is_error: None,
3252	content_blocks: Some(vec![json!({
3253	"type": "text",
3254	"text": "structured one"
3255	})]),
3256	}],
3257	},
3258	Message {
3259	role: "user".to_string(),
3260	content: vec![ContentBlock::ToolResult {
3261	tool_use_id: "tool-2".to_string(),
3262	content: "two".to_string(),
3263	is_error: Some(true),
3264	content_blocks: None,
3265	}],
3266	},
3267	Message {
3268	role: "assistant".to_string(),
3269	content: vec![ContentBlock::Text {
3270	text: "done".to_string(),
3271	cache_control: None,
3272	}],
3273	},
3274	];
3275
3276	manager
3277	.seed_thread_from_messages(&thread.id, &messages)
3278	.await?;
3279	let turns = manager.store.list_turns_for_thread(&thread.id)?;
3280	assert_eq!(turns.len(), 1);
3281
3282	let restored = manager.reconstruct_messages_from_turns(&turns)?;
3283	let roles = restored
3284	.iter()
3285	.map(\|message\| message.role.as_str())
3286	.collect::<Vec<_>>();
3287	assert_eq!(roles, vec!["user", "assistant", "user", "assistant"]);
3288	assert_eq!(restored[2].content.len(), 2);
3289
3290	match &restored[2].content[0] {
3291	ContentBlock::ToolResult {
3292	tool_use_id,
3293	content,
3294	is_error,
3295	content_blocks,
3296	} => {
3297	assert_eq!(tool_use_id, "tool-1");
3298	assert_eq!(content, "one");
3299	assert_eq!(*is_error, None);
3300	assert_eq!(
3301	content_blocks
3302	.as_ref()
3303	.and_then(\|blocks\| blocks[0].get("text")),
3304	Some(&json!("structured one"))
3305	);
3306	}
3307	other => panic!("expected first tool result, got {other:?}"),
3308	}
3309	match &restored[2].content[1] {
3310	ContentBlock::ToolResult {
3311	tool_use_id,
3312	content,
3313	is_error,
3314	content_blocks,
3315	} => {
3316	assert_eq!(tool_use_id, "tool-2");
3317	assert_eq!(content, "two");
3318	assert_eq!(*is_error, Some(true));
3319	assert!(content_blocks.is_none());
3320	}
3321	other => panic!("expected second tool result, got {other:?}"),
3322	}
3323
3324	let _ = std::fs::remove_dir_all(dir);
3325	Ok(())
3326	}
3327
3328	#[test]
3329	fn current_runtime_schema_version_is_two_on_v066() {
3330	// Locks the bump in (issue #124). Bump deliberately when persisted
3331	// shape changes.
3332	assert_eq!(CURRENT_RUNTIME_SCHEMA_VERSION, 2);
3333	}
3334
3335	#[test]
3336	fn store_rejects_path_like_record_ids() {
3337	let dir = test_runtime_dir();
3338	let store = RuntimeThreadStore::open(dir.clone()).expect("open store");
3339
3340	let err = store
3341	.load_thread("../outside")
3342	.expect_err("path traversal id should fail");
3343	assert!(
3344	format!("{err:#}").contains("unsupported characters"),
3345	"got: {err:#}"
3346	);
3347
3348	let mut thread = sample_thread("thr_bad/id");
3349	let err = store
3350	.save_thread(&thread)
3351	.expect_err("path separator id should fail");
3352	assert!(
3353	format!("{err:#}").contains("unsupported characters"),
3354	"got: {err:#}"
3355	);
3356
3357	thread.id = " thr_bad".to_string();
3358	let err = store
3359	.save_thread(&thread)
3360	.expect_err("whitespace id should fail");
3361	assert!(format!("{err:#}").contains("whitespace"), "got: {err:#}");
3362
3363	let _ = std::fs::remove_dir_all(dir);
3364	}
3365
3366	#[test]
3367	fn store_load_turn_rejects_newer_schema_version() {
3368	let dir = test_runtime_dir();
3369	let store = RuntimeThreadStore::open(dir.clone()).expect("open store");
3370
3371	let mut turn = sample_turn("thr_t", "trn_future", RuntimeTurnStatus::InProgress);
3372	turn.schema_version = CURRENT_RUNTIME_SCHEMA_VERSION + 1;
3373
3374	let path = store.turns_dir.join(format!("{}.json", turn.id));
3375	std::fs::create_dir_all(path.parent().unwrap()).expect("mkdirs");
3376	std::fs::write(&path, serde_json::to_string(&turn).expect("serialize turn"))
3377	.expect("write turn");
3378
3379	let err = store
3380	.load_turn(&turn.id)
3381	.expect_err("load_turn must reject newer schema");
3382	assert!(
3383	format!("{err:#}").contains("newer than supported"),
3384	"got: {err:#}"
3385	);
3386
3387	let _ = std::fs::remove_dir_all(dir);
3388	}
3389
3390	#[test]
3391	fn store_load_item_rejects_newer_schema_version() {
3392	let dir = test_runtime_dir();
3393	let store = RuntimeThreadStore::open(dir.clone()).expect("open store");
3394
3395	let mut item = sample_item("trn_t", "itm_future", TurnItemLifecycleStatus::InProgress);
3396	item.schema_version = CURRENT_RUNTIME_SCHEMA_VERSION + 1;
3397
3398	let path = store.items_dir.join(format!("{}.json", item.id));
3399	std::fs::create_dir_all(path.parent().unwrap()).expect("mkdirs");
3400	std::fs::write(&path, serde_json::to_string(&item).expect("serialize item"))
3401	.expect("write item");
3402
3403	let err = store
3404	.load_item(&item.id)
3405	.expect_err("load_item must reject newer schema");
3406	assert!(
3407	format!("{err:#}").contains("newer than supported"),
3408	"got: {err:#}"
3409	);
3410
3411	let _ = std::fs::remove_dir_all(dir);
3412	}
3413
3414	#[test]
3415	fn enforce_lru_capacity_does_not_loop_when_all_threads_are_active() {
3416	let mut active = ActiveThreads::default();
3417	let harness_a = mock_engine_handle();
3418	let harness_b = mock_engine_handle();
3419
3420	active.engines.insert(
3421	"thr_a".to_string(),
3422	ActiveThreadState {
3423	engine: harness_a.handle,
3424	active_turn: Some(ActiveTurnState {
3425	turn_id: "turn_a".to_string(),
3426	interrupt_requested: false,
3427	}),
3428	route_identity: crate::config::ProviderIdentity {
3429	provider: ApiProvider::Deepseek,
3430	key: "deepseek".to_string(),
3431	exact_id: Some("deepseek".to_string()),
3432	},
3433	route_model: DEFAULT_TEXT_MODEL.to_string(),
3434	client_preflight_required: false,
3435	},
3436	);
3437	active.engines.insert(
3438	"thr_b".to_string(),
3439	ActiveThreadState {
3440	engine: harness_b.handle,
3441	active_turn: Some(ActiveTurnState {
3442	turn_id: "turn_b".to_string(),
3443	interrupt_requested: false,
3444	}),
3445	route_identity: crate::config::ProviderIdentity {
3446	provider: ApiProvider::Deepseek,
3447	key: "deepseek".to_string(),
3448	exact_id: Some("deepseek".to_string()),
3449	},
3450	route_model: DEFAULT_TEXT_MODEL.to_string(),
3451	client_preflight_required: false,
3452	},
3453	);
3454	active.lru.push_back("thr_a".to_string());
3455	active.lru.push_back("thr_b".to_string());
3456
3457	let evicted = enforce_lru_capacity(&mut active, 2);
3458	assert!(evicted.is_empty(), "no idle threads should be evicted");
3459	assert_eq!(active.engines.len(), 2);
3460	assert_eq!(active.lru.len(), 2);
3461	}
3462
3463	#[test]
3464	fn approval_decision_keeps_trust_mode_out_of_tool_approval() {
3465	assert!(matches!(
3466	RuntimeThreadManager::approval_decision(false, false, false),
3467	RuntimeApprovalDecision::DenyTool
3468	));
3469	assert!(matches!(
3470	RuntimeThreadManager::approval_decision(false, true, false),
3471	RuntimeApprovalDecision::DenyTool
3472	));
3473	assert!(matches!(
3474	RuntimeThreadManager::approval_decision(true, false, false),
3475	RuntimeApprovalDecision::ApproveTool
3476	));
3477	assert!(matches!(
3478	RuntimeThreadManager::approval_decision(true, false, true),
3479	RuntimeApprovalDecision::DenyTool
3480	));
3481	assert!(matches!(
3482	RuntimeThreadManager::approval_decision(true, true, true),
3483	RuntimeApprovalDecision::RetryWithFullAccess
3484	));
3485	}
3486
3487	#[test]
3488	fn open_recovers_queued_and_in_progress_turns() -> Result<()> {
3489	let runtime_dir = test_runtime_dir();
3490	let store = RuntimeThreadStore::open(runtime_dir.clone())?;
3491	let thread = sample_thread("thr_recover");
3492	store.save_thread(&thread)?;
3493
3494	let mut queued_turn = sample_turn(&thread.id, "turn_queued", RuntimeTurnStatus::Queued);
3495	let mut in_progress_turn =
3496	sample_turn(&thread.id, "turn_running", RuntimeTurnStatus::InProgress);
3497	let completed_turn = sample_turn(&thread.id, "turn_done", RuntimeTurnStatus::Completed);
3498
3499	let queued_item = sample_item(
3500	&queued_turn.id,
3501	"item_queued",
3502	TurnItemLifecycleStatus::Queued,
3503	);
3504	let in_progress_item = sample_item(
3505	&in_progress_turn.id,
3506	"item_running",
3507	TurnItemLifecycleStatus::InProgress,
3508	);
3509	let completed_item = sample_item(
3510	&completed_turn.id,
3511	"item_done",
3512	TurnItemLifecycleStatus::Completed,
3513	);
3514
3515	queued_turn.item_ids = vec![queued_item.id.clone()];
3516	in_progress_turn.item_ids = vec![in_progress_item.id.clone()];
3517
3518	store.save_item(&queued_item)?;
3519	store.save_item(&in_progress_item)?;
3520	store.save_item(&completed_item)?;
3521	store.save_turn(&queued_turn)?;
3522	store.save_turn(&in_progress_turn)?;
3523	store.save_turn(&completed_turn)?;
3524
3525	let manager = test_manager(runtime_dir)?;
3526
3527	let queued_turn = manager.store.load_turn(&queued_turn.id)?;
3528	assert_eq!(queued_turn.status, RuntimeTurnStatus::Interrupted);
3529	assert_eq!(queued_turn.error.as_deref(), Some(RUNTIME_RESTART_REASON));
3530	assert!(queued_turn.ended_at.is_some());
3531	assert!(queued_turn.duration_ms.is_some());
3532
3533	let in_progress_turn = manager.store.load_turn(&in_progress_turn.id)?;
3534	assert_eq!(in_progress_turn.status, RuntimeTurnStatus::Interrupted);
3535	assert_eq!(
3536	in_progress_turn.error.as_deref(),
3537	Some(RUNTIME_RESTART_REASON)
3538	);
3539	assert!(in_progress_turn.ended_at.is_some());
3540	assert!(in_progress_turn.duration_ms.is_some());
3541
3542	let completed_turn = manager.store.load_turn(&completed_turn.id)?;
3543	assert_eq!(completed_turn.status, RuntimeTurnStatus::Completed);
3544	assert!(completed_turn.error.is_none());
3545
3546	let queued_item = manager.store.load_item("item_queued")?;
3547	assert_eq!(queued_item.status, TurnItemLifecycleStatus::Interrupted);
3548	assert!(queued_item.ended_at.is_some());
3549
3550	let in_progress_item = manager.store.load_item("item_running")?;
3551	assert_eq!(
3552	in_progress_item.status,
3553	TurnItemLifecycleStatus::Interrupted
3554	);
3555	assert!(in_progress_item.ended_at.is_some());
3556
3557	let completed_item = manager.store.load_item("item_done")?;
3558	assert_eq!(completed_item.status, TurnItemLifecycleStatus::Completed);
3559
3560	Ok(())
3561	}
3562
3563	#[tokio::test]
3564	async fn thread_lifecycle_persists_across_restart() -> Result<()> {
3565	let runtime_dir = test_runtime_dir();
3566	let manager = test_manager(runtime_dir.clone())?;
3567	let thread = manager
3568	.create_thread(CreateThreadRequest {
3569	model: None,
3570	workspace: None,
3571	mode: None,
3572	allow_shell: None,
3573	trust_mode: None,
3574	auto_approve: None,
3575	archived: false,
3576	system_prompt: None,
3577	task_id: None,
3578	..Default::default()
3579	})
3580	.await?;
3581
3582	let harness = install_mock_engine(&manager, &thread.id).await;
3583	let mut rx_op = harness.rx_op;
3584	let tx_event = harness.tx_event;
3585	tokio::spawn(async move {
3586	if matches!(rx_op.recv().await, Some(Op::SendMessage { .. })) {
3587	let _ = tx_event
3588	.send(EngineEvent::TurnStarted {
3589	turn_id: "engine_turn_1".to_string(),
3590	created_at: chrono::Utc::now(),
3591	route: None,
3592	})
3593	.await;
3594	let _ = tx_event
3595	.send(EngineEvent::MessageStarted { index: 0 })
3596	.await;
3597	let _ = tx_event
3598	.send(EngineEvent::MessageDelta {
3599	index: 0,
3600	content: "mock response".to_string(),
3601	})
3602	.await;
3603	let _ = tx_event
3604	.send(EngineEvent::MessageComplete { index: 0 })
3605	.await;
3606	let _ = tx_event
3607	.send(EngineEvent::TurnComplete {
3608	usage: Usage {
3609	input_tokens: 10,
3610	output_tokens: 12,
3611	..Usage::default()
3612	},
3613	status: TurnOutcomeStatus::Completed,
3614	error: None,
3615	tool_catalog: None,
3616	base_url: None,
3617	})
3618	.await;
3619	}
3620	});
3621
3622	let turn = manager
3623	.start_turn(
3624	&thread.id,
3625	StartTurnRequest {
3626	prompt: "first prompt".to_string(),
3627	input_summary: None,
3628	model: None,
3629	mode: None,
3630	allow_shell: None,
3631	trust_mode: None,
3632	auto_approve: None,
3633	..Default::default()
3634	},
3635	)
3636	.await?;
3637	let completed = wait_for_terminal_turn(&manager, &turn.id, Duration::from_secs(2)).await?;
3638	assert_eq!(completed.status, RuntimeTurnStatus::Completed);
3639
3640	drop(manager);
3641
3642	let reopened = test_manager(runtime_dir)?;
3643	let detail = reopened.get_thread_detail(&thread.id).await?;
3644	assert_eq!(detail.thread.id, thread.id);
3645	assert_eq!(detail.turns.len(), 1);
3646	assert!(detail.latest_seq >= 1);
3647	assert!(!detail.items.is_empty());
3648	let events = reopened.events_since(&thread.id, None)?;
3649	assert!(
3650	events.iter().any(\|ev\| ev.event == "turn.completed"),
3651	"expected turn.completed event after restart"
3652	);
3653	Ok(())
3654	}
3655
3656	#[tokio::test]
3657	async fn monitor_separates_lifecycle_start_from_billing_dispatch_and_child_usage() -> Result<()> {
3658	let manager = test_manager(test_runtime_dir())?;
3659	let thread = manager
3660	.create_thread(CreateThreadRequest::default())
3661	.await?;
3662	let mut harness = install_mock_engine(&manager, &thread.id).await;
3663	let turn = manager
3664	.start_turn(
3665	&thread.id,
3666	StartTurnRequest {
3667	prompt: "capture immutable route".to_string(),
3668	..StartTurnRequest::default()
3669	},
3670	)
3671	.await?;
3672	assert!(matches!(
3673	harness.rx_op.recv().await,
3674	Some(Op::SendMessage { .. })
3675	));
3676
3677	let started_at = Utc::now() - chrono::Duration::minutes(5);
3678	let dispatched_at = Utc::now() - chrono::Duration::seconds(2);
3679	let endpoint_fingerprint = "d".repeat(64);
3680	harness
3681	.tx_event
3682	.send(EngineEvent::TurnStarted {
3683	turn_id: "engine_route_receipt".to_string(),
3684	created_at: started_at,
3685	route: None,
3686	})
3687	.await?;
3688	harness
3689	.tx_event
3690	.send(EngineEvent::RouteDispatched {
3691	turn_id: "engine_route_receipt".to_string(),
3692	route: crate::core::events::TurnRoute {
3693	provider: ApiProvider::Stepfun,
3694	provider_identity: "stepfun-payg-primary".to_string(),
3695	model: "step-3.7-flash".to_string(),
3696	auto_model: false,
3697	receipt: None,
3698	billing: Some(crate::core::events::RouteBillingEnvelope {
3699	billing_surface: Some(crate::pricing::STEPFUN_PAYG_BILLING_SURFACE.to_string()),
3700	endpoint_fingerprint: Some(endpoint_fingerprint.clone()),
3701	billing_mode: crate::cost_status::RouteBillingMode::Metered,
3702	dispatched_at,
3703	}),
3704	base_url: ApiProvider::Stepfun.default_base_url().to_string(),
3705	billing_product: crate::route_billing::RouteProduct::Unproven,
3706	},
3707	})
3708	.await?;
3709	harness
3710	.tx_event
3711	.send(EngineEvent::SubAgentMailbox {
3712	turn_id: "engine_route_receipt".to_string(),
3713	seq: 77,
3714	message: crate::tools::subagent::MailboxMessage::TokenUsage {
3715	agent_id: "agent_child".to_string(),
3716	source_id: "response-child".to_string(),
3717	route: crate::cost_status::EffectiveRouteEnvelope {
3718	provider: ApiProvider::OpenaiCodex,
3719	provider_identity: "codex-child".to_string(),
3720	model: "gpt-5.5".to_string(),
3721	billing_surface: Some(
3722	crate::pricing::OAUTH_SUBSCRIPTION_BILLING_SURFACE.to_string(),
3723	),
3724	endpoint_fingerprint: None,
3725	billing_mode: crate::cost_status::RouteBillingMode::Subscription,
3726	dispatched_at,
3727	},
3728	usage: Usage {
3729	input_tokens: 3,
3730	output_tokens: 2,
3731	reasoning_tokens: Some(2),
3732	..Usage::default()
3733	},
3734	},
3735	})
3736	.await?;
3737	harness
3738	.tx_event
3739	.send(EngineEvent::TurnComplete {
3740	usage: Usage {
3741	input_tokens: 10,
3742	output_tokens: 4,
3743	reasoning_replay_tokens: Some(6),
3744	server_tool_use: Some(crate::models::ServerToolUsage {
3745	code_execution_requests: Some(2),
3746	tool_search_requests: Some(3),
3747	}),
3748	..Usage::default()
3749	},
3750	status: TurnOutcomeStatus::Completed,
3751	error: None,
3752	tool_catalog: None,
3753	// Completion-time endpoint evidence must be ignored.
3754	base_url: Some("https://api.stepfun.com/v1/coding".to_string()),
3755	})
3756	.await?;
3757
3758	let completed = wait_for_terminal_turn(&manager, &turn.id, Duration::from_secs(2)).await?;
3759	assert_eq!(completed.effective_provider.as_deref(), Some("stepfun"));
3760	assert_eq!(
3761	completed.effective_provider_id.as_deref(),
3762	Some("stepfun-payg-primary")
3763	);
3764	assert_eq!(
3765	completed.effective_billing_surface.as_deref(),
3766	Some(crate::pricing::STEPFUN_PAYG_BILLING_SURFACE)
3767	);
3768	assert_eq!(
3769	completed.effective_endpoint_fingerprint.as_deref(),
3770	Some(endpoint_fingerprint.as_str())
3771	);
3772	assert_eq!(
3773	completed.effective_billing_mode,
3774	Some(crate::cost_status::RouteBillingMode::Metered)
3775	);
3776	assert_eq!(completed.effective_dispatched_at, Some(dispatched_at));
3777	assert_eq!(completed.started_at, Some(started_at));
3778	assert_eq!(completed.routed_usage.len(), 1);
3779	assert_eq!(completed.routed_usage[0].usage.reasoning_tokens, Some(2));
3780	let persisted_usage = completed.usage.expect("parent usage");
3781	assert_eq!(persisted_usage.reasoning_replay_tokens, Some(6));
3782	assert_eq!(
3783	persisted_usage.server_tool_use,
3784	Some(crate::models::ServerToolUsage {
3785	code_execution_requests: Some(2),
3786	tool_search_requests: Some(3),
3787	})
3788	);
3789
3790	// A new engine mailbox starts at the same sequence number. Runtime
3791	// dedupe is scoped by the engine turn identity, so the second receipt is
3792	// persisted exactly once instead of colliding with the prior turn.
3793	let second = manager
3794	.start_turn(
3795	&thread.id,
3796	StartTurnRequest {
3797	prompt: "capture second mailbox".to_string(),
3798	..StartTurnRequest::default()
3799	},
3800	)
3801	.await?;
3802	assert!(matches!(
3803	harness.rx_op.recv().await,
3804	Some(Op::SendMessage { .. })
3805	));
3806	let second_engine_turn = "engine_route_receipt_second";
3807	harness
3808	.tx_event
3809	.send(EngineEvent::TurnStarted {
3810	turn_id: second_engine_turn.to_string(),
3811	created_at: Utc::now(),
3812	route: None,
3813	})
3814	.await?;
3815	harness
3816	.tx_event
3817	.send(EngineEvent::SubAgentMailbox {
3818	turn_id: second_engine_turn.to_string(),
3819	seq: 77,
3820	message: crate::tools::subagent::MailboxMessage::TokenUsage {
3821	agent_id: "agent-child-second".to_string(),
3822	source_id: "response-child-second".to_string(),
3823	route: crate::cost_status::EffectiveRouteEnvelope {
3824	provider: ApiProvider::OpenaiCodex,
3825	provider_identity: "codex-child".to_string(),
3826	model: "gpt-5.5".to_string(),
3827	billing_surface: Some(
3828	crate::pricing::OAUTH_SUBSCRIPTION_BILLING_SURFACE.to_string(),
3829	),
3830	endpoint_fingerprint: None,
3831	billing_mode: crate::cost_status::RouteBillingMode::Subscription,
3832	dispatched_at: Utc::now(),
3833	},
3834	usage: Usage {
3835	input_tokens: 5,
3836	output_tokens: 1,
3837	..Usage::default()
3838	},
3839	},
3840	})
3841	.await?;
3842	harness
3843	.tx_event
3844	.send(EngineEvent::TurnComplete {
3845	usage: Usage::default(),
3846	status: TurnOutcomeStatus::Completed,
3847	error: None,
3848	tool_catalog: None,
3849	base_url: None,
3850	})
3851	.await?;
3852	let second = wait_for_terminal_turn(&manager, &second.id, Duration::from_secs(2)).await?;
3853	assert_eq!(second.routed_usage.len(), 1);
3854	assert_eq!(second.routed_usage[0].usage.input_tokens, 5);
3855	Ok(())
3856	}
3857
3858	#[tokio::test]
3859	async fn completed_turn_without_engine_output_fails() -> Result<()> {
3860	let manager = test_manager(test_runtime_dir())?;
3861	let thread = manager
3862	.create_thread(CreateThreadRequest {
3863	model: None,
3864	workspace: None,
3865	mode: None,
3866	allow_shell: None,
3867	trust_mode: None,
3868	auto_approve: None,
3869	archived: false,
3870	system_prompt: None,
3871	task_id: None,
3872	..Default::default()
3873	})
3874	.await?;
3875
3876	let harness = install_mock_engine(&manager, &thread.id).await;
3877	let mut rx_op = harness.rx_op;
3878	let tx_event = harness.tx_event;
3879	tokio::spawn(async move {
3880	if matches!(rx_op.recv().await, Some(Op::SendMessage { .. })) {
3881	let _ = tx_event
3882	.send(EngineEvent::TurnStarted {
3883	turn_id: "engine_empty_turn".to_string(),
3884	created_at: chrono::Utc::now(),
3885	route: None,
3886	})
3887	.await;
3888	let _ = tx_event
3889	.send(EngineEvent::TurnComplete {
3890	usage: Usage {
3891	input_tokens: 10,
3892	output_tokens: 0,
3893	..Usage::default()
3894	},
3895	status: TurnOutcomeStatus::Completed,
3896	error: None,
3897	tool_catalog: None,
3898	base_url: None,
3899	})
3900	.await;
3901	}
3902	});
3903
3904	let turn = manager
3905	.start_turn(
3906	&thread.id,
3907	StartTurnRequest {
3908	prompt: "empty turn".to_string(),
3909	input_summary: None,
3910	model: None,
3911	mode: None,
3912	allow_shell: None,
3913	trust_mode: None,
3914	auto_approve: None,
3915	..Default::default()
3916	},
3917	)
3918	.await?;
3919
3920	let failed = wait_for_terminal_turn(&manager, &turn.id, Duration::from_secs(2)).await?;
3921	assert_eq!(failed.status, RuntimeTurnStatus::Failed);
3922	assert_eq!(failed.error.as_deref(), Some(EMPTY_TURN_REASON));
3923
3924	let events = manager.events_since(&thread.id, None)?;
3925	assert!(events.iter().any(\|ev\| {
3926	ev.event == "item.failed"
3927	&& ev
3928	.payload
3929	.get("item")
3930	.and_then(\|item\| item.get("kind"))
3931	.and_then(Value::as_str)
3932	== Some("error")
3933	}));
3934	assert!(events.iter().any(\|ev\| {
3935	ev.event == "turn.completed"
3936	&& ev
3937	.payload
3938	.get("turn")
3939	.and_then(\|turn\| turn.get("status"))
3940	.and_then(Value::as_str)
3941	== Some("failed")
3942	}));
3943	Ok(())
3944	}
3945
3946	#[tokio::test]
3947	async fn preturn_control_status_does_not_make_empty_turn_succeed() -> Result<()> {
3948	let manager = test_manager(test_runtime_dir())?;
3949	let thread = manager
3950	.create_thread(CreateThreadRequest::default())
3951	.await?;
3952	let harness = install_mock_engine(&manager, &thread.id).await;
3953	let mut rx_op = harness.rx_op;
3954	let tx_event = harness.tx_event;
3955	tokio::spawn(async move {
3956	if matches!(rx_op.recv().await, Some(Op::SendMessage { .. })) {
3957	let _ = tx_event
3958	.send(EngineEvent::AgentComplete {
3959	id: "stale_agent".to_string(),
3960	result: "stale completion".to_string(),
3961	})
3962	.await;
3963	let _ = tx_event
3964	.send(EngineEvent::status("Compaction settings updated"))
3965	.await;
3966	let _ = tx_event
3967	.send(EngineEvent::TurnStarted {
3968	turn_id: "engine_empty_after_control_status".to_string(),
3969	created_at: chrono::Utc::now(),
3970	route: None,
3971	})
3972	.await;
3973	let _ = tx_event
3974	.send(EngineEvent::TurnComplete {
3975	usage: Usage::default(),
3976	status: TurnOutcomeStatus::Completed,
3977	error: None,
3978	tool_catalog: None,
3979	base_url: None,
3980	})
3981	.await;
3982	}
3983	});
3984
3985	let turn = manager
3986	.start_turn(
3987	&thread.id,
3988	StartTurnRequest {
3989	prompt: "empty after setup".to_string(),
3990	..Default::default()
3991	},
3992	)
3993	.await?;
3994	let terminal = wait_for_terminal_turn(&manager, &turn.id, Duration::from_secs(2)).await?;
3995	assert_eq!(terminal.status, RuntimeTurnStatus::Failed);
3996	assert_eq!(terminal.error.as_deref(), Some(EMPTY_TURN_REASON));
3997	assert!(
3998	manager
3999	.store
4000	.list_items_for_turn(&turn.id)?
4001	.iter()
4002	.all(\|item\| {
4003	item.summary != "Compaction settings updated"
4004	&& !item.summary.contains("stale_agent")
4005	})
4006	);
4007	Ok(())
4008	}
4009
4010	#[tokio::test]
4011	async fn engine_error_remains_failed_after_nominal_turn_complete() -> Result<()> {
4012	let manager = test_manager(test_runtime_dir())?;
4013	let thread = manager
4014	.create_thread(CreateThreadRequest::default())
4015	.await?;
4016	let harness = install_mock_engine(&manager, &thread.id).await;
4017	let mut rx_op = harness.rx_op;
4018	let tx_event = harness.tx_event;
4019	tokio::spawn(async move {
4020	if matches!(rx_op.recv().await, Some(Op::SendMessage { .. })) {
4021	let _ = tx_event
4022	.send(EngineEvent::TurnStarted {
4023	turn_id: "engine_error_then_complete".to_string(),
4024	created_at: chrono::Utc::now(),
4025	route: None,
4026	})
4027	.await;
4028	let _ = tx_event
4029	.send(EngineEvent::error(
4030	crate::error_taxonomy::ErrorEnvelope::fatal("provider exploded"),
4031	))
4032	.await;
4033	let _ = tx_event
4034	.send(EngineEvent::TurnComplete {
4035	usage: Usage::default(),
4036	status: TurnOutcomeStatus::Completed,
4037	error: None,
4038	tool_catalog: None,
4039	base_url: None,
4040	})
4041	.await;
4042	}
4043	});
4044
4045	let turn = manager
4046	.start_turn(
4047	&thread.id,
4048	StartTurnRequest {
4049	prompt: "surface the failure".to_string(),
4050	..Default::default()
4051	},
4052	)
4053	.await?;
4054	let terminal = wait_for_terminal_turn(&manager, &turn.id, Duration::from_secs(2)).await?;
4055	assert_eq!(terminal.status, RuntimeTurnStatus::Failed);
4056	assert_eq!(terminal.error.as_deref(), Some("provider exploded"));
4057	Ok(())
4058	}
4059
4060	#[tokio::test]
4061	async fn create_thread_defaults_auto_approve_to_false() -> Result<()> {
4062	let manager = test_manager(test_runtime_dir())?;
4063	let thread = manager
4064	.create_thread(CreateThreadRequest {
4065	model: None,
4066	workspace: None,
4067	mode: None,
4068	allow_shell: None,
4069	trust_mode: None,
4070	auto_approve: None,
4071	archived: false,
4072	system_prompt: None,
4073	task_id: None,
4074	..Default::default()
4075	})
4076	.await?;
4077
4078	assert!(!thread.auto_approve);
4079	Ok(())
4080	}
4081
4082	#[tokio::test]
4083	async fn update_thread_workspace_persists_event_and_evicts_idle_engine() -> Result<()> {
4084	let manager = test_manager(test_runtime_dir())?;
4085	let old_workspace = std::env::temp_dir().join("codewhale-runtime-old-workspace");
4086	let new_workspace = std::env::temp_dir().join("codewhale-runtime-new-workspace");
4087	let thread = manager
4088	.create_thread(CreateThreadRequest {
4089	model: None,
4090	workspace: Some(old_workspace.clone()),
4091	mode: None,
4092	allow_shell: None,
4093	trust_mode: None,
4094	auto_approve: None,
4095	archived: false,
4096	system_prompt: None,
4097	task_id: None,
4098	..Default::default()
4099	})
4100	.await?;
4101
4102	let harness = install_mock_engine(&manager, &thread.id).await;
4103	let mut rx_op = harness.rx_op;
4104
4105	let updated = manager
4106	.update_thread(
4107	&thread.id,
4108	UpdateThreadRequest {
4109	workspace: Some(new_workspace.clone()),
4110	..UpdateThreadRequest::default()
4111	},
4112	)
4113	.await?;
4114
4115	assert_eq!(updated.workspace, new_workspace);
4116	assert_eq!(
4117	manager.store.load_thread(&thread.id)?.workspace,
4118	new_workspace
4119	);
4120	{
4121	let active = manager.active.lock().await;
4122	assert!(
4123	!active.engines.contains_key(&thread.id),
4124	"workspace changes must evict the stale cached engine"
4125	);
4126	assert!(!active.lru.iter().any(\|id\| id == &thread.id));
4127	}
4128
4129	match tokio::time::timeout(Duration::from_secs(1), rx_op.recv()).await {
4130	Ok(Some(Op::Shutdown)) => {}
4131	other => panic!("expected cached engine shutdown, got {other:?}"),
4132	}
4133
4134	let events = manager.events_since(&thread.id, None)?;
4135	let event = events
4136	.iter()
4137	.rev()
4138	.find(\|event\| event.event == "thread.updated")
4139	.expect("thread.updated event");
4140	let workspace_value = serde_json::to_value(&updated.workspace)?;
4141	assert_eq!(
4142	event
4143	.payload
4144	.get("changes")
4145	.and_then(\|changes\| changes.get("workspace")),
4146	Some(&workspace_value)
4147	);
4148	Ok(())
4149	}
4150
4151	#[tokio::test]
4152	async fn update_thread_workspace_rejects_empty_path() -> Result<()> {
4153	let manager = test_manager(test_runtime_dir())?;
4154	let thread = manager
4155	.create_thread(CreateThreadRequest {
4156	model: None,
4157	workspace: None,
4158	mode: None,
4159	allow_shell: None,
4160	trust_mode: None,
4161	auto_approve: None,
4162	archived: false,
4163	system_prompt: None,
4164	task_id: None,
4165	..Default::default()
4166	})
4167	.await?;
4168
4169	let err = manager
4170	.update_thread(
4171	&thread.id,
4172	UpdateThreadRequest {
4173	workspace: Some(PathBuf::new()),
4174	..UpdateThreadRequest::default()
4175	},
4176	)
4177	.await
4178	.expect_err("empty workspace must be rejected");
4179	assert!(format!("{err:#}").contains("workspace must not be empty"));
4180	Ok(())
4181	}
4182
4183	#[tokio::test]
4184	async fn update_thread_workspace_rejects_active_turn() -> Result<()> {
4185	let manager = test_manager(test_runtime_dir())?;
4186	let old_workspace = std::env::temp_dir().join("codewhale-runtime-active-old");
4187	let new_workspace = std::env::temp_dir().join("codewhale-runtime-active-new");
4188	let thread = manager
4189	.create_thread(CreateThreadRequest {
4190	model: None,
4191	workspace: Some(old_workspace.clone()),
4192	mode: None,
4193	allow_shell: None,
4194	trust_mode: None,
4195	auto_approve: None,
4196	archived: false,
4197	system_prompt: None,
4198	task_id: None,
4199	..Default::default()
4200	})
4201	.await?;
4202
4203	let harness = install_mock_engine(&manager, &thread.id).await;
4204	let mut rx_op = harness.rx_op;
4205	{
4206	let mut active = manager.active.lock().await;
4207	let state = active.engines.get_mut(&thread.id).expect("mock engine");
4208	state.active_turn = Some(ActiveTurnState {
4209	turn_id: "turn_live".to_string(),
4210	interrupt_requested: false,
4211	});
4212	}
4213
4214	let err = manager
4215	.update_thread(
4216	&thread.id,
4217	UpdateThreadRequest {
4218	workspace: Some(new_workspace),
4219	..UpdateThreadRequest::default()
4220	},
4221	)
4222	.await
4223	.expect_err("workspace update during active turn must fail");
4224
4225	assert!(format!("{err:#}").contains("active turn"));
4226	assert_eq!(
4227	manager.store.load_thread(&thread.id)?.workspace,
4228	old_workspace
4229	);
4230	{
4231	let active = manager.active.lock().await;
4232	assert!(
4233	active.engines.contains_key(&thread.id),
4234	"active engine should stay cached after rejected update"
4235	);
4236	}
4237	assert!(
4238	tokio::time::timeout(Duration::from_millis(100), rx_op.recv())
4239	.await
4240	.is_err(),
4241	"rejected workspace update must not shut down the active engine"
4242	);
4243	Ok(())
4244	}
4245
4246	#[tokio::test]
4247	async fn start_turn_passes_effective_auto_approve_to_engine() -> Result<()> {
4248	let manager = test_manager(test_runtime_dir())?;
4249	let thread = manager
4250	.create_thread(CreateThreadRequest {
4251	model: None,
4252	workspace: None,
4253	mode: None,
4254	allow_shell: None,
4255	trust_mode: None,
4256	auto_approve: Some(false),
4257	archived: false,
4258	system_prompt: None,
4259	task_id: None,
4260	..Default::default()
4261	})
4262	.await?;
4263
4264	let harness = install_mock_engine(&manager, &thread.id).await;
4265	let mut rx_op = harness.rx_op;
4266
4267	let turn = manager
4268	.start_turn(
4269	&thread.id,
4270	StartTurnRequest {
4271	prompt: "override approval".to_string(),
4272	input_summary: None,
4273	model: None,
4274	mode: None,
4275	allow_shell: None,
4276	trust_mode: None,
4277	auto_approve: Some(true),
4278	..Default::default()
4279	},
4280	)
4281	.await?;
4282	assert_eq!(turn.permission_posture.as_deref(), Some("full_access"));
4283
4284	match rx_op.recv().await {
4285	Some(Op::SendMessage {
4286	auto_approve,
4287	approval_mode,
4288	..
4289	}) => {
4290	assert!(auto_approve);
4291	assert_eq!(approval_mode, crate::tui::approval::ApprovalMode::Bypass);
4292	}
4293	other => panic!("expected SendMessage op, got {other:?}"),
4294	}
4295
4296	Ok(())
4297	}
4298
4299	#[tokio::test]
4300	async fn start_turn_can_override_thread_auto_approve_to_false() -> Result<()> {
4301	let manager = test_manager(test_runtime_dir())?;
4302	let thread = manager
4303	.create_thread(CreateThreadRequest {
4304	model: None,
4305	workspace: None,
4306	mode: None,
4307	allow_shell: None,
4308	trust_mode: None,
4309	auto_approve: Some(true),
4310	archived: false,
4311	system_prompt: None,
4312	task_id: None,
4313	..Default::default()
4314	})
4315	.await?;
4316
4317	let harness = install_mock_engine(&manager, &thread.id).await;
4318	let mut rx_op = harness.rx_op;
4319
4320	let turn = manager
4321	.start_turn(
4322	&thread.id,
4323	StartTurnRequest {
4324	prompt: "disable approval".to_string(),
4325	input_summary: None,
4326	model: None,
4327	mode: None,
4328	allow_shell: None,
4329	trust_mode: None,
4330	auto_approve: Some(false),
4331	..Default::default()
4332	},
4333	)
4334	.await?;
4335	assert_eq!(turn.permission_posture.as_deref(), Some("ask"));
4336
4337	match rx_op.recv().await {
4338	Some(Op::SendMessage {
4339	auto_approve,
4340	approval_mode,
4341	..
4342	}) => {
4343	assert!(!auto_approve);
4344	assert_eq!(approval_mode, crate::tui::approval::ApprovalMode::Suggest);
4345	}
4346	other => panic!("expected SendMessage op, got {other:?}"),
4347	}
4348
4349	Ok(())
4350	}
4351
4352	#[tokio::test]
4353	async fn start_turn_enforces_and_records_auto_review_without_legacy_bypass() -> Result<()> {
4354	let manager = test_manager(test_runtime_dir())?;
4355	let thread = manager
4356	.create_thread(CreateThreadRequest {
4357	permission_posture: Some("ask".to_string()),
4358	..Default::default()
4359	})
4360	.await?;
4361	let harness = install_mock_engine(&manager, &thread.id).await;
4362	let mut rx_op = harness.rx_op;
4363
4364	let turn = manager
4365	.start_turn(
4366	&thread.id,
4367	StartTurnRequest {
4368	prompt: "review autonomously".to_string(),
4369	permission_posture: Some("auto-review".to_string()),
4370	..Default::default()
4371	},
4372	)
4373	.await?;
4374
4375	assert_eq!(turn.permission_posture.as_deref(), Some("auto_review"));
4376	assert_eq!(
4377	manager
4378	.store
4379	.load_turn(&turn.id)?
4380	.permission_posture
4381	.as_deref(),
4382	Some("auto_review")
4383	);
4384	match rx_op.recv().await {
4385	Some(Op::SendMessage {
4386	auto_approve,
4387	approval_mode,
4388	..
4389	}) => {
4390	assert!(!auto_approve);
4391	assert_eq!(approval_mode, crate::tui::approval::ApprovalMode::Auto);
4392	}
4393	other => panic!("expected SendMessage op, got {other:?}"),
4394	}
4395	Ok(())
4396	}
4397
4398	#[tokio::test]
4399	async fn active_turn_permission_posture_switches_use_the_engine_live_authority() -> Result<()> {
4400	let manager = test_manager(test_runtime_dir())?;
4401	let thread = manager
4402	.create_thread(CreateThreadRequest {
4403	permission_posture: Some("ask".to_string()),
4404	..Default::default()
4405	})
4406	.await?;
4407	let mut harness = install_mock_engine(&manager, &thread.id).await;
4408	let turn = manager
4409	.start_turn(
4410	&thread.id,
4411	StartTurnRequest {
4412	prompt: "keep working while permissions change".to_string(),
4413	..Default::default()
4414	},
4415	)
4416	.await?;
4417	assert!(matches!(
4418	harness.rx_op.recv().await,
4419	Some(Op::SendMessage {
4420	approval_mode: crate::tui::approval::ApprovalMode::Suggest,
4421	..
4422	})
4423	));
4424
4425	for (requested, canonical, expected_auto, expected_approval) in [
4426	(
4427	"auto-review",
4428	"auto_review",
4429	false,
4430	crate::tui::approval::ApprovalMode::Auto,
4431	),
4432	(
4433	"full-access",
4434	"full_access",
4435	true,
4436	crate::tui::approval::ApprovalMode::Bypass,
4437	),
4438	(
4439	"ask",
4440	"ask",
4441	false,
4442	crate::tui::approval::ApprovalMode::Suggest,
4443	),
4444	] {
4445	let updated = manager
4446	.update_thread(
4447	&thread.id,
4448	UpdateThreadRequest {
4449	permission_posture: Some(requested.to_string()),
4450	..Default::default()
4451	},
4452	)
4453	.await?;
4454	assert_eq!(updated.permission_posture.as_deref(), Some(canonical));
4455	assert_eq!(updated.auto_approve, expected_auto);
4456
4457	match harness.rx_op.recv().await {
4458	Some(Op::ChangeMode {
4459	auto_approve,
4460	approval_mode,
4461	..
4462	}) => {
4463	assert_eq!(auto_approve, expected_auto, "{requested}");
4464	assert_eq!(approval_mode, expected_approval, "{requested}");
4465	}
4466	other => panic!("expected ChangeMode for {requested}, got {other:?}"),
4467	}
4468
4469	let authority = {
4470	let active = manager.active.lock().await;
4471	active
4472	.engines
4473	.get(&thread.id)
4474	.expect("active engine")
4475	.engine
4476	.runtime_permission_authority()
4477	};
4478	assert_eq!(authority.auto_approve, expected_auto, "{requested}");
4479	assert_eq!(authority.approval_mode, expected_approval, "{requested}");
4480	assert_eq!(
4481	manager.active_turn_flags(&thread.id, &turn.id).await,
4482	Some((expected_auto, false)),
4483	"{requested} must replace the running turn's authority immediately"
4484	);
4485	}
4486
4487	Ok(())
4488	}
4489
4490	#[tokio::test]
4491	async fn compact_thread_preserves_thread_auto_approve_policy() -> Result<()> {
4492	let manager = test_manager(test_runtime_dir())?;
4493	let thread = manager
4494	.create_thread(CreateThreadRequest {
4495	model: None,
4496	workspace: None,
4497	mode: None,
4498	allow_shell: None,
4499	trust_mode: None,
4500	auto_approve: Some(false),
4501	archived: false,
4502	system_prompt: None,
4503	task_id: None,
4504	..Default::default()
4505	})
4506	.await?;
4507
4508	let harness = install_mock_engine(&manager, &thread.id).await;
4509	let mut rx_op = harness.rx_op;
4510
4511	let turn = manager
4512	.compact_thread(&thread.id, CompactThreadRequest::default())
4513	.await?;
4514
4515	match rx_op.recv().await {
4516	Some(Op::CompactContext { compaction, .. }) => {
4517	assert_eq!(
4518	compaction.runtime_cost_owner.as_deref(),
4519	Some(turn.id.as_str())
4520	);
4521	}
4522	other => panic!("expected CompactContext op, got {other:?}"),
4523	}
4524	assert_eq!(
4525	manager.active_turn_flags(&thread.id, &turn.id).await,
4526	Some((false, false))
4527	);
4528
4529	Ok(())
4530	}
4531
4532	#[tokio::test]
4533	async fn closed_compaction_mailbox_rolls_back_durable_records_and_active_claim() -> Result<()> {
4534	let manager = test_manager(test_runtime_dir())?;
4535	let thread = manager
4536	.create_thread(CreateThreadRequest::default())
4537	.await?;
4538	let harness = install_mock_engine(&manager, &thread.id).await;
4539	let before_active = {
4540	let active = manager.active.lock().await;
4541	let state = active.engines.get(&thread.id).expect("installed engine");
4542	(
4543	state.active_turn.as_ref().map(\|turn\| turn.turn_id.clone()),
4544	state.route_identity.clone(),
4545	state.route_model.clone(),
4546	active.lru.clone(),
4547	)
4548	};
4549	let before_thread = serde_json::to_value(manager.get_thread(&thread.id).await?)?;
4550	let before_events = serde_json::to_value(manager.events_since(&thread.id, None)?)?;
4551	drop(harness.rx_op);
4552
4553	let error = manager
4554	.compact_thread(&thread.id, CompactThreadRequest::default())
4555	.await
4556	.expect_err("closed mailbox must reject compaction")
4557	.to_string();
4558	assert!(error.contains("Failed to trigger compaction"), "{error}");
4559
4560	assert!(manager.store.list_turns_for_thread(&thread.id)?.is_empty());
4561	assert_eq!(
4562	serde_json::to_value(manager.get_thread(&thread.id).await?)?,
4563	before_thread
4564	);
4565	assert_eq!(
4566	serde_json::to_value(manager.events_since(&thread.id, None)?)?,
4567	before_events
4568	);
4569	let after_active = {
4570	let active = manager.active.lock().await;
4571	let state = active.engines.get(&thread.id).expect("installed engine");
4572	(
4573	state.active_turn.as_ref().map(\|turn\| turn.turn_id.clone()),
4574	state.route_identity.clone(),
4575	state.route_model.clone(),
4576	active.lru.clone(),
4577	)
4578	};
4579	assert_eq!(after_active, before_active);
4580	Ok(())
4581	}
4582
4583	#[tokio::test]
4584	async fn compact_thread_receipt_keeps_exact_named_custom_identity() -> Result<()> {
4585	let mut custom = std::collections::HashMap::new();
4586	custom.insert(
4587	"lm-studio".to_string(),
4588	crate::config::ProviderConfig {
4589	kind: Some("openai-compatible".to_string()),
4590	base_url: Some("http://127.0.0.1:1234/v1".to_string()),
4591	model: Some("local-code-model".to_string()),
4592	..Default::default()
4593	},
4594	);
4595	let manager = RuntimeThreadManager::open(
4596	Config {
4597	provider: Some("lm-studio".to_string()),
4598	providers: Some(crate::config::ProvidersConfig {
4599	custom,
4600	..Default::default()
4601	}),
4602	..Default::default()
4603	},
4604	PathBuf::from("."),
4605	test_manager_config(test_runtime_dir()),
4606	)?;
4607	let thread = manager
4608	.create_thread(CreateThreadRequest::default())
4609	.await?;
4610	let harness = install_mock_engine(&manager, &thread.id).await;
4611	let mut rx_op = harness.rx_op;
4612
4613	let turn = manager
4614	.compact_thread(&thread.id, CompactThreadRequest::default())
4615	.await?;
4616
4617	assert!(matches!(
4618	rx_op.recv().await,
4619	Some(Op::CompactContext { .. })
4620	));
4621	assert_eq!(turn.effective_provider.as_deref(), Some("custom"));
4622	assert_eq!(turn.effective_provider_id.as_deref(), Some("lm-studio"));
4623	Ok(())
4624	}
4625
4626	#[tokio::test]
4627	async fn compact_thread_with_real_engine_reaches_terminal_status() -> Result<()> {
4628	let manager = RuntimeThreadManager::open(
4629	Config {
4630	// This test intentionally crosses the real-engine boundary. Give
4631	// client preflight a hermetic credential and closed-loopback URL;
4632	// the assertion permits the resulting terminal failure.
4633	api_key: Some("runtime-thread-test-key".to_string()),
4634	base_url: Some("http://127.0.0.1:1/v1".to_string()),
4635	..Config::default()
4636	},
4637	PathBuf::from("."),
4638	test_manager_config(test_runtime_dir()),
4639	)?;
4640	let thread = manager
4641	.create_thread(CreateThreadRequest {
4642	model: None,
4643	workspace: None,
4644	mode: None,
4645	allow_shell: None,
4646	trust_mode: None,
4647	auto_approve: None,
4648	archived: false,
4649	system_prompt: None,
4650	task_id: None,
4651	..Default::default()
4652	})
4653	.await?;
4654
4655	let turn = manager
4656	.compact_thread(&thread.id, CompactThreadRequest::default())
4657	.await?;
4658	let terminal = wait_for_terminal_turn(&manager, &turn.id, Duration::from_secs(2)).await?;
4659
4660	assert!(matches!(
4661	terminal.status,
4662	RuntimeTurnStatus::Completed \| RuntimeTurnStatus::Failed
4663	));
4664	assert!(
4665	terminal.ended_at.is_some(),
4666	"manual compaction should reach a terminal turn state"
4667	);
4668	assert_eq!(manager.active_turn_flags(&thread.id, &turn.id).await, None);
4669
4670	let expected_status = match terminal.status {
4671	RuntimeTurnStatus::Completed => "completed",
4672	RuntimeTurnStatus::Failed => "failed",
4673	other => panic!("unexpected non-terminal compaction status: {other:?}"),
4674	};
4675	let events = manager.events_since(&thread.id, None)?;
4676	assert!(events.iter().any(\|ev\| {
4677	ev.event == "turn.completed"
4678	&& ev
4679	.payload
4680	.get("turn")
4681	.and_then(\|turn\| turn.get("status"))
4682	.and_then(Value::as_str)
4683	== Some(expected_status)
4684	}));
4685	Ok(())
4686	}
4687
4688	#[tokio::test]
4689	async fn multi_turn_continuity_same_thread() -> Result<()> {
4690	let manager = test_manager(test_runtime_dir())?;
4691	let thread = manager
4692	.create_thread(CreateThreadRequest {
4693	model: None,
4694	workspace: None,
4695	mode: None,
4696	allow_shell: None,
4697	trust_mode: None,
4698	auto_approve: None,
4699	archived: false,
4700	system_prompt: None,
4701	task_id: None,
4702	..Default::default()
4703	})
4704	.await?;
4705
4706	let harness = install_mock_engine(&manager, &thread.id).await;
4707	let mut rx_op = harness.rx_op;
4708	let tx_event = harness.tx_event;
4709	tokio::spawn(async move {
4710	let mut turn_index = 0u8;
4711	while let Some(op) = rx_op.recv().await {
4712	if !matches!(op, Op::SendMessage { .. }) {
4713	continue;
4714	}
4715	turn_index = turn_index.saturating_add(1);
4716	let _ = tx_event
4717	.send(EngineEvent::TurnStarted {
4718	turn_id: format!("engine_turn_{turn_index}"),
4719	created_at: chrono::Utc::now(),
4720	route: None,
4721	})
4722	.await;
4723	let _ = tx_event
4724	.send(EngineEvent::MessageStarted { index: 0 })
4725	.await;
4726	let _ = tx_event
4727	.send(EngineEvent::MessageDelta {
4728	index: 0,
4729	content: format!("reply {turn_index}"),
4730	})
4731	.await;
4732	let _ = tx_event
4733	.send(EngineEvent::MessageComplete { index: 0 })
4734	.await;
4735	let _ = tx_event
4736	.send(EngineEvent::TurnComplete {
4737	usage: Usage {
4738	input_tokens: 5,
4739	output_tokens: 5,
4740	..Usage::default()
4741	},
4742	status: TurnOutcomeStatus::Completed,
4743	error: None,
4744	tool_catalog: None,
4745	base_url: None,
4746	})
4747	.await;
4748	if turn_index >= 2 {
4749	break;
4750	}
4751	}
4752	});
4753
4754	let turn_1 = manager
4755	.start_turn(
4756	&thread.id,
4757	StartTurnRequest {
4758	prompt: "first".to_string(),
4759	input_summary: None,
4760	model: None,
4761	mode: None,
4762	allow_shell: None,
4763	trust_mode: None,
4764	auto_approve: None,
4765	..Default::default()
4766	},
4767	)
4768	.await?;
4769	let turn_1 = wait_for_terminal_turn(&manager, &turn_1.id, Duration::from_secs(2)).await?;
4770	assert_eq!(turn_1.status, RuntimeTurnStatus::Completed);
4771
4772	let turn_2 = manager
4773	.start_turn(
4774	&thread.id,
4775	StartTurnRequest {
4776	prompt: "second".to_string(),
4777	input_summary: None,
4778	model: None,
4779	mode: None,
4780	allow_shell: None,
4781	trust_mode: None,
4782	auto_approve: None,
4783	..Default::default()
4784	},
4785	)
4786	.await?;
4787	let turn_2 = wait_for_terminal_turn(&manager, &turn_2.id, Duration::from_secs(2)).await?;
4788	assert_eq!(turn_2.status, RuntimeTurnStatus::Completed);
4789
4790	let detail = manager.get_thread_detail(&thread.id).await?;
4791	assert_eq!(
4792	detail.thread.latest_turn_id.as_deref(),
4793	Some(turn_2.id.as_str())
4794	);
4795	assert_eq!(detail.turns.len(), 2);
4796	assert!(detail.items.iter().any(\|item\| {
4797	item.kind == TurnItemKind::UserMessage && item.detail.as_deref() == Some("first")
4798	}));
4799	assert!(detail.items.iter().any(\|item\| {
4800	item.kind == TurnItemKind::UserMessage && item.detail.as_deref() == Some("second")
4801	}));
4802
4803	let events = manager.events_since(&thread.id, None)?;
4804	let started = events
4805	.iter()
4806	.filter(\|ev\| ev.event == "turn.started")
4807	.count();
4808	let completed = events
4809	.iter()
4810	.filter(\|ev\| ev.event == "turn.completed")
4811	.count();
4812	assert_eq!(started, 2);
4813	assert_eq!(completed, 2);
4814	Ok(())
4815	}
4816
4817	#[tokio::test]
4818	async fn get_thread_detail_batches_items_by_turn_without_losing_order() -> Result<()> {
4819	let manager = test_manager(test_runtime_dir())?;
4820	let thread = manager
4821	.create_thread(CreateThreadRequest {
4822	model: None,
4823	workspace: None,
4824	mode: None,
4825	allow_shell: None,
4826	trust_mode: None,
4827	auto_approve: None,
4828	archived: false,
4829	system_prompt: None,
4830	task_id: None,
4831	..Default::default()
4832	})
4833	.await?;
4834
4835	let base = Utc::now();
4836	let mut first_turn = sample_turn(
4837	&thread.id,
4838	"turn_detail_batch_first",
4839	RuntimeTurnStatus::Completed,
4840	);
4841	first_turn.created_at = base;
4842	let mut second_turn = sample_turn(
4843	&thread.id,
4844	"turn_detail_batch_second",
4845	RuntimeTurnStatus::Completed,
4846	);
4847	second_turn.created_at = base + chrono::Duration::seconds(1);
4848	manager.store.save_turn(&first_turn)?;
4849	manager.store.save_turn(&second_turn)?;
4850
4851	let mut first_late = sample_item(
4852	&first_turn.id,
4853	"item_detail_first_late",
4854	TurnItemLifecycleStatus::Completed,
4855	);
4856	first_late.started_at = Some(base + chrono::Duration::seconds(5));
4857	let mut first_early = sample_item(
4858	&first_turn.id,
4859	"item_detail_first_early",
4860	TurnItemLifecycleStatus::Completed,
4861	);
4862	first_early.started_at = Some(base + chrono::Duration::seconds(1));
4863	let mut second_item = sample_item(
4864	&second_turn.id,
4865	"item_detail_second",
4866	TurnItemLifecycleStatus::Completed,
4867	);
4868	second_item.started_at = Some(base + chrono::Duration::seconds(2));
4869	let unrelated = sample_item(
4870	"turn_detail_batch_unrelated",
4871	"item_detail_unrelated",
4872	TurnItemLifecycleStatus::Completed,
4873	);
4874
4875	manager.store.save_item(&first_late)?;
4876	manager.store.save_item(&second_item)?;
4877	manager.store.save_item(&unrelated)?;
4878	manager.store.save_item(&first_early)?;
4879
4880	let detail = manager.get_thread_detail(&thread.id).await?;
4881	let item_ids: Vec<&str> = detail.items.iter().map(\|item\| item.id.as_str()).collect();
4882	assert_eq!(
4883	item_ids,
4884	vec![
4885	"item_detail_first_early",
4886	"item_detail_first_late",
4887	"item_detail_second"
4888	]
4889	);
4890	Ok(())
4891	}
4892
4893	#[tokio::test]
4894	async fn interrupt_turn_marks_interrupted_after_cleanup() -> Result<()> {
4895	let manager = test_manager(test_runtime_dir())?;
4896	let thread = manager
4897	.create_thread(CreateThreadRequest {
4898	model: None,
4899	workspace: None,
4900	mode: None,
4901	allow_shell: None,
4902	trust_mode: None,
4903	auto_approve: None,
4904	archived: false,
4905	system_prompt: None,
4906	task_id: None,
4907	..Default::default()
4908	})
4909	.await?;
4910
4911	let harness = install_mock_engine(&manager, &thread.id).await;
4912	let mut rx_op = harness.rx_op;
4913	let tx_event = harness.tx_event;
4914	let cancel_token = harness.cancel_token;
4915	let cleanup_delay = Duration::from_millis(140);
4916	tokio::spawn(async move {
4917	if matches!(rx_op.recv().await, Some(Op::SendMessage { .. })) {
4918	let _ = tx_event
4919	.send(EngineEvent::TurnStarted {
4920	turn_id: "engine_turn_interrupt".to_string(),
4921	created_at: chrono::Utc::now(),
4922	route: None,
4923	})
4924	.await;
4925	let _ = tx_event
4926	.send(EngineEvent::MessageStarted { index: 0 })
4927	.await;
4928	let _ = tx_event
4929	.send(EngineEvent::MessageDelta {
4930	index: 0,
4931	content: "partial".to_string(),
4932	})
4933	.await;
4934	cancel_token.cancelled().await;
4935	sleep(cleanup_delay).await;
4936	}
4937	});
4938
4939	let turn = manager
4940	.start_turn(
4941	&thread.id,
4942	StartTurnRequest {
4943	prompt: "interrupt me".to_string(),
4944	input_summary: None,
4945	model: None,
4946	mode: None,
4947	allow_shell: None,
4948	trust_mode: None,
4949	auto_approve: None,
4950	..Default::default()
4951	},
4952	)
4953	.await?;
4954
4955	sleep(Duration::from_millis(20)).await;
4956	let interrupted_at = Instant::now();
4957	let interrupt_result = manager.interrupt_turn(&thread.id, &turn.id).await?;
4958	assert_eq!(interrupt_result.status, RuntimeTurnStatus::InProgress);
4959
4960	let final_turn = wait_for_terminal_turn(&manager, &turn.id, Duration::from_secs(3)).await?;
4961	assert_eq!(final_turn.status, RuntimeTurnStatus::Interrupted);
4962	assert!(
4963	interrupted_at.elapsed() >= cleanup_delay,
4964	"turn transitioned before cleanup finished"
4965	);
4966
4967	let events = manager.events_since(&thread.id, None)?;
4968	let interrupt_seq = events
4969	.iter()
4970	.find(\|ev\| ev.event == "turn.interrupt_requested")
4971	.map(\|ev\| ev.seq)
4972	.context("missing turn.interrupt_requested event")?;
4973	let completed = events
4974	.iter()
4975	.find(\|ev\| ev.event == "turn.completed")
4976	.context("missing turn.completed event")?;
4977	assert!(completed.seq > interrupt_seq);
4978	assert_eq!(
4979	completed
4980	.payload
4981	.get("turn")
4982	.and_then(\|turn\| turn.get("status"))
4983	.and_then(Value::as_str),
4984	Some("interrupted")
4985	);
4986	Ok(())
4987	}
4988
4989	#[tokio::test]
4990	async fn approval_required_with_stale_active_turn_is_denied() -> Result<()> {
4991	let manager = test_manager(test_runtime_dir())?;
4992	let thread = manager
4993	.create_thread(CreateThreadRequest {
4994	model: None,
4995	workspace: None,
4996	mode: None,
4997	allow_shell: None,
4998	trust_mode: None,
4999	auto_approve: Some(true),
5000	archived: false,
5001	system_prompt: None,
5002	task_id: None,
5003	..Default::default()
5004	})
5005	.await?;
5006
5007	let mut harness = install_mock_engine(&manager, &thread.id).await;
5008	let turn = manager
5009	.start_turn(
5010	&thread.id,
5011	StartTurnRequest {
5012	prompt: "needs approval".to_string(),
5013	input_summary: None,
5014	model: None,
5015	mode: None,
5016	allow_shell: None,
5017	trust_mode: None,
5018	auto_approve: Some(true),
5019	..Default::default()
5020	},
5021	)
5022	.await?;
5023
5024	assert!(matches!(
5025	harness.rx_op.recv().await,
5026	Some(Op::SendMessage { .. })
5027	));
5028	{
5029	let mut active = manager.active.lock().await;
5030	let state = active
5031	.engines
5032	.get_mut(&thread.id)
5033	.context("missing active thread state")?;
5034	state.active_turn = None;
5035	}
5036
5037	harness
5038	.tx_event
5039	.send(EngineEvent::ApprovalRequired {
5040	approval_key: "test_key".to_string(),
5041	approval_grouping_key: "test_key".to_string(),
5042	id: "tool_stale".to_string(),
5043	tool_name: "exec_command".to_string(),
5044	description: "stale approval".to_string(),
5045	input: serde_json::json!({}),
5046	intent_summary: None,
5047	approval_force_prompt: false,
5048	})
5049	.await?;
5050
5051	assert_eq!(
5052	harness.recv_approval_event().await,
5053	Some(MockApprovalEvent::Denied {
5054	id: "tool_stale".to_string(),
5055	})
5056	);
5057
5058	harness
5059	.tx_event
5060	.send(EngineEvent::TurnComplete {
5061	usage: Usage {
5062	input_tokens: 0,
5063	output_tokens: 0,
5064	..Usage::default()
5065	},
5066	status: TurnOutcomeStatus::Completed,
5067	error: None,
5068	tool_catalog: None,
5069	base_url: None,
5070	})
5071	.await?;
5072
5073	let terminal = wait_for_terminal_turn(&manager, &turn.id, Duration::from_secs(2)).await?;
5074	assert_eq!(terminal.status, RuntimeTurnStatus::Completed);
5075	Ok(())
5076	}
5077
5078	#[tokio::test]
5079	async fn approval_required_awaits_external_decision_allow() -> Result<()> {
5080	let manager = test_manager(test_runtime_dir())?;
5081	let thread = manager
5082	.create_thread(CreateThreadRequest {
5083	model: None,
5084	workspace: None,
5085	mode: None,
5086	allow_shell: None,
5087	trust_mode: None,
5088	auto_approve: None,
5089	archived: false,
5090	system_prompt: None,
5091	task_id: None,
5092	..Default::default()
5093	})
5094	.await?;
5095
5096	let mut harness = install_mock_engine(&manager, &thread.id).await;
5097	let _turn = manager
5098	.start_turn(
5099	&thread.id,
5100	StartTurnRequest {
5101	prompt: "needs approval".to_string(),
5102	input_summary: None,
5103	model: None,
5104	mode: None,
5105	allow_shell: None,
5106	trust_mode: None,
5107	auto_approve: None,
5108	..Default::default()
5109	},
5110	)
5111	.await?;
5112	assert!(matches!(
5113	harness.rx_op.recv().await,
5114	Some(Op::SendMessage { .. })
5115	));
5116
5117	harness
5118	.tx_event
5119	.send(EngineEvent::ApprovalRequired {
5120	approval_key: "key1".to_string(),
5121	approval_grouping_key: "key1".to_string(),
5122	id: "tool_external_allow".to_string(),
5123	tool_name: "exec_command".to_string(),
5124	description: "external allow".to_string(),
5125	input: serde_json::json!({}),
5126	intent_summary: Some("I will update the config file.".to_string()),
5127	approval_force_prompt: false,
5128	})
5129	.await?;
5130
5131	let deadline = Instant::now() + Duration::from_secs(2);
5132	while Instant::now() < deadline && manager.pending_approvals_count() == 0 {
5133	sleep(Duration::from_millis(20)).await;
5134	}
5135	assert_eq!(manager.pending_approvals_count(), 1);
5136
5137	let detail = manager.get_thread_detail(&thread.id).await?;
5138	assert_eq!(detail.pending_approvals.len(), 1);
5139	assert_eq!(detail.pending_approvals[0].id, "tool_external_allow");
5140	assert_eq!(detail.pending_approvals[0].turn_id, _turn.id);
5141	assert_eq!(detail.pending_approvals[0].tool_name, "exec_command");
5142	assert_eq!(detail.pending_user_inputs.len(), 0);
5143
5144	let events = manager.events_since(&thread.id, None)?;
5145	let approval_event = events
5146	.iter()
5147	.rev()
5148	.find(\|event\| event.event == "approval.required")
5149	.context("missing approval.required event")?;
5150	assert_eq!(
5151	approval_event
5152	.payload
5153	.get("intent_summary")
5154	.and_then(Value::as_str),
5155	Some("I will update the config file.")
5156	);
5157
5158	assert!(manager.deliver_external_approval(
5159	"tool_external_allow",
5160	ExternalApprovalDecision::Allow { remember: false },
5161	));
5162	assert_eq!(
5163	harness.recv_approval_event().await,
5164	Some(MockApprovalEvent::Approved {
5165	id: "tool_external_allow".to_string(),
5166	})
5167	);
5168	assert_eq!(manager.pending_approvals_count(), 0);
5169	assert!(
5170	manager
5171	.get_thread_detail(&thread.id)
5172	.await?
5173	.pending_approvals
5174	.is_empty()
5175	);
5176
5177	harness
5178	.tx_event
5179	.send(EngineEvent::TurnComplete {
5180	usage: Usage::default(),
5181	status: TurnOutcomeStatus::Completed,
5182	error: None,
5183	tool_catalog: None,
5184	base_url: None,
5185	})
5186	.await?;
5187	Ok(())
5188	}
5189
5190	#[tokio::test]
5191	async fn user_input_snapshot_survives_reload_and_clears_after_submission() -> Result<()> {
5192	let manager = test_manager(test_runtime_dir())?;
5193	let thread = manager
5194	.create_thread(CreateThreadRequest::default())
5195	.await?;
5196	let mut harness = install_mock_engine(&manager, &thread.id).await;
5197	let turn = manager
5198	.start_turn(
5199	&thread.id,
5200	StartTurnRequest {
5201	prompt: "needs a choice".to_string(),
5202	..StartTurnRequest::default()
5203	},
5204	)
5205	.await?;
5206	assert!(matches!(
5207	harness.rx_op.recv().await,
5208	Some(Op::SendMessage { .. })
5209	));
5210
5211	harness
5212	.tx_event
5213	.send(EngineEvent::UserInputRequired {
5214	id: "input_reload".to_string(),
5215	request: crate::tools::user_input::UserInputRequest {
5216	questions: vec![crate::tools::user_input::UserInputQuestion {
5217	header: "Continue".to_string(),
5218	id: "continue".to_string(),
5219	question: "Continue with the check?".to_string(),
5220	options: vec![
5221	crate::tools::user_input::UserInputOption {
5222	label: "Yes".to_string(),
5223	description: "Continue now".to_string(),
5224	},
5225	crate::tools::user_input::UserInputOption {
5226	label: "No".to_string(),
5227	description: "Stop here".to_string(),
5228	},
5229	],
5230	allow_free_text: false,
5231	multi_select: false,
5232	}],
5233	},
5234	})
5235	.await?;
5236
5237	let deadline = Instant::now() + Duration::from_secs(2);
5238	let detail = loop {
5239	let detail = manager.get_thread_detail(&thread.id).await?;
5240	if !detail.pending_user_inputs.is_empty() {
5241	break detail;
5242	}
5243	if Instant::now() >= deadline {
5244	bail!("pending user input did not reach the canonical snapshot");
5245	}
5246	sleep(Duration::from_millis(20)).await;
5247	};
5248	assert_eq!(detail.pending_approvals.len(), 0);
5249	assert_eq!(detail.pending_user_inputs.len(), 1);
5250	assert_eq!(detail.pending_user_inputs[0].id, "input_reload");
5251	assert_eq!(detail.pending_user_inputs[0].turn_id, turn.id);
5252	assert_eq!(
5253	detail.pending_user_inputs[0].request.questions[0].question,
5254	"Continue with the check?"
5255	);
5256
5257	manager
5258	.submit_user_input(
5259	&thread.id,
5260	"input_reload",
5261	crate::tools::user_input::UserInputResponse {
5262	answers: vec![crate::tools::user_input::UserInputAnswer {
5263	id: "continue".to_string(),
5264	label: "Yes".to_string(),
5265	value: "Yes".to_string(),
5266	}],
5267	},
5268	)
5269	.await?;
5270	match harness.recv_user_input_submission().await {
5271	Some((id, response)) => {
5272	assert_eq!(id, "input_reload");
5273	assert_eq!(response.answers[0].id, "continue");
5274	}
5275	other => panic!("expected submitted user input, got {other:?}"),
5276	}
5277	assert!(
5278	manager
5279	.get_thread_detail(&thread.id)
5280	.await?
5281	.pending_user_inputs
5282	.is_empty()
5283	);
5284	assert!(manager.events_since(&thread.id, None)?.iter().any(\|event\| {
5285	event.event == "user_input.answered"
5286	&& event.payload.get("input_id").and_then(Value::as_str) == Some("input_reload")
5287	}));
5288
5289	harness
5290	.tx_event
5291	.send(EngineEvent::TurnComplete {
5292	usage: Usage::default(),
5293	status: TurnOutcomeStatus::Completed,
5294	error: None,
5295	tool_catalog: None,
5296	base_url: None,
5297	})
5298	.await?;
5299	Ok(())
5300	}
5301
5302	#[tokio::test]
5303	async fn unknown_user_input_id_is_not_delivered_to_engine() -> Result<()> {
5304	let manager = test_manager(test_runtime_dir())?;
5305	let thread = manager
5306	.create_thread(CreateThreadRequest::default())
5307	.await?;
5308	let mut harness = install_mock_engine(&manager, &thread.id).await;
5309	let delivered = manager
5310	.submit_user_input(
5311	&thread.id,
5312	"input_missing",
5313	crate::tools::user_input::UserInputResponse {
5314	answers: vec![crate::tools::user_input::UserInputAnswer {
5315	id: "choice".to_string(),
5316	label: "Missing".to_string(),
5317	value: "must-not-enter-engine-mailbox".to_string(),
5318	}],
5319	},
5320	)
5321	.await?;
5322	assert!(!delivered);
5323	assert!(
5324	tokio::time::timeout(
5325	Duration::from_millis(25),
5326	harness.recv_user_input_submission()
5327	)
5328	.await
5329	.is_err(),
5330	"unknown request entered the engine mailbox"
5331	);
5332	assert!(manager.events_since(&thread.id, None)?.iter().all(\|event\| {
5333	!matches!(
5334	event.event.as_str(),
5335	"user_input.answered" \| "user_input.canceled"
5336	)
5337	}));
5338	Ok(())
5339	}
5340
5341	#[tokio::test]
5342	async fn user_input_receipt_append_failure_restores_request_without_delivery() -> Result<()> {
5343	const SECRET: &str = "answer-only-for-engine-after-retry";
5344	let manager = test_manager(test_runtime_dir())?;
5345	let thread = manager
5346	.create_thread(CreateThreadRequest::default())
5347	.await?;
5348	let mut harness = install_mock_engine(&manager, &thread.id).await;
5349	manager.register_pending_user_input(
5350	&thread.id,
5351	PendingUserInputRequest {
5352	id: "input_retry".to_string(),
5353	turn_id: "turn_retry".to_string(),
5354	request: crate::tools::user_input::UserInputRequest {
5355	questions: Vec::new(),
5356	},
5357	},
5358	);
5359	let response = \|\| crate::tools::user_input::UserInputResponse {
5360	answers: vec![crate::tools::user_input::UserInputAnswer {
5361	id: "choice".to_string(),
5362	label: "Retry".to_string(),
5363	value: SECRET.to_string(),
5364	}],
5365	};
5366
5367	let fault_guard = EventAppendFaultGuard::arm(&thread.id, EventAppendTestFault::AfterSync);
5368	let error = manager
5369	.submit_user_input(&thread.id, "input_retry", response())
5370	.await
5371	.expect_err("injected receipt append unexpectedly succeeded");
5372	drop(fault_guard);
5373	assert!(format!("{error:#}").contains("rolled back"));
5374	assert_eq!(
5375	manager
5376	.get_thread_detail(&thread.id)
5377	.await?
5378	.pending_user_inputs
5379	.len(),
5380	1,
5381	"retry-safe append failure removed the authoritative prompt"
5382	);
5383	assert!(
5384	tokio::time::timeout(
5385	Duration::from_millis(25),
5386	harness.recv_user_input_submission()
5387	)
5388	.await
5389	.is_err(),
5390	"answer reached the engine before its receipt was durable"
5391	);
5392
5393	assert!(
5394	manager
5395	.submit_user_input(&thread.id, "input_retry", response())
5396	.await?,
5397	"restored request was not retryable"
5398	);
5399	let (_, delivered) =
5400	tokio::time::timeout(Duration::from_secs(2), harness.recv_user_input_submission())
5401	.await
5402	.context("retried answer did not reach the engine")?
5403	.context("retried answer was canceled")?;
5404	assert_eq!(delivered.answers[0].value, SECRET);
5405	let events = manager.events_since(&thread.id, None)?;
5406	assert_eq!(
5407	events
5408	.iter()
5409	.filter(\|event\| event.event == "user_input.answered")
5410	.count(),
5411	1
5412	);
5413	assert!(!serde_json::to_string(&events)?.contains(SECRET));
5414	Ok(())
5415	}
5416
5417	#[tokio::test]
5418	async fn user_input_settlement_outlives_canceled_api_future() -> Result<()> {
5419	const SECRET: &str = "answer-survives-request-disconnect";
5420	let manager = test_manager(test_runtime_dir())?;
5421	let thread = manager
5422	.create_thread(CreateThreadRequest::default())
5423	.await?;
5424	let mut harness = install_mock_engine(&manager, &thread.id).await;
5425	manager.register_pending_user_input(
5426	&thread.id,
5427	PendingUserInputRequest {
5428	id: "input_detached".to_string(),
5429	turn_id: "turn_detached".to_string(),
5430	request: crate::tools::user_input::UserInputRequest {
5431	questions: Vec::new(),
5432	},
5433	},
5434	);
5435
5436	let emit_guard = manager.event_emit.lock().await;
5437	let submit_manager = manager.clone();
5438	let thread_id = thread.id.clone();
5439	let submission = tokio::spawn(async move {
5440	submit_manager
5441	.submit_user_input(
5442	&thread_id,
5443	"input_detached",
5444	crate::tools::user_input::UserInputResponse {
5445	answers: vec![crate::tools::user_input::UserInputAnswer {
5446	id: "choice".to_string(),
5447	label: "Continue".to_string(),
5448	value: SECRET.to_string(),
5449	}],
5450	},
5451	)
5452	.await
5453	});
5454	tokio::time::timeout(Duration::from_secs(2), async {
5455	loop {
5456	if manager
5457	.pending_user_inputs
5458	.lock()
5459	.get(&(thread.id.clone(), "input_detached".to_string()))
5460	.is_some_and(\|entry\| entry.settling)
5461	{
5462	break;
5463	}
5464	tokio::task::yield_now().await;
5465	}
5466	})
5467	.await
5468	.context("submission did not claim the pending request")?;
5469	submission.abort();
5470	assert!(
5471	tokio::time::timeout(
5472	Duration::from_millis(25),
5473	harness.recv_user_input_submission()
5474	)
5475	.await
5476	.is_err(),
5477	"answer reached the engine before its receipt append was released"
5478	);
5479	drop(emit_guard);
5480
5481	let (_, delivered) =
5482	tokio::time::timeout(Duration::from_secs(2), harness.recv_user_input_submission())
5483	.await
5484	.context("detached settlement did not reach the engine")?
5485	.context("detached settlement was canceled")?;
5486	assert_eq!(delivered.answers[0].value, SECRET);
5487	let detail = manager.get_thread_detail(&thread.id).await?;
5488	assert!(detail.pending_user_inputs.is_empty());
5489	let events = manager.events_since(&thread.id, None)?;
5490	assert_eq!(
5491	events
5492	.iter()
5493	.filter(\|event\| event.event == "user_input.answered")
5494	.count(),
5495	1
5496	);
5497	assert!(!serde_json::to_string(&events)?.contains(SECRET));
5498	Ok(())
5499	}
5500
5501	#[tokio::test]
5502	async fn terminal_user_input_cancellation_is_durable_before_engine_delivery() -> Result<()> {
5503	let manager = test_manager(test_runtime_dir())?;
5504	let thread = manager
5505	.create_thread(CreateThreadRequest::default())
5506	.await?;
5507	let mut harness = install_mock_engine(&manager, &thread.id).await;
5508	manager.register_pending_user_input(
5509	&thread.id,
5510	PendingUserInputRequest {
5511	id: "input_terminal_order".to_string(),
5512	turn_id: "turn_terminal_order".to_string(),
5513	request: crate::tools::user_input::UserInputRequest {
5514	questions: Vec::new(),
5515	},
5516	},
5517	);
5518
5519	let emit_guard = manager.event_emit.lock().await;
5520	let engine = harness.handle.clone();
5521	let settle_manager = manager.clone();
5522	let thread_id = thread.id.clone();
5523	let settlement = tokio::spawn(async move {
5524	settle_manager
5525	.settle_user_inputs_for_terminal_turn(&thread_id, "turn_terminal_order", Some(engine))
5526	.await
5527	});
5528	tokio::time::timeout(Duration::from_secs(2), async {
5529	loop {
5530	if manager
5531	.pending_user_inputs
5532	.lock()
5533	.get(&(thread.id.clone(), "input_terminal_order".to_string()))
5534	.is_some_and(\|entry\| entry.settling)
5535	{
5536	break;
5537	}
5538	tokio::task::yield_now().await;
5539	}
5540	})
5541	.await
5542	.context("terminal cancellation did not claim the request")?;
5543	assert!(
5544	tokio::time::timeout(
5545	Duration::from_millis(25),
5546	harness.recv_user_input_submission()
5547	)
5548	.await
5549	.is_err(),
5550	"terminal cancellation reached the engine before durable append"
5551	);
5552	drop(emit_guard);
5553	settlement
5554	.await
5555	.context("terminal settlement task panicked")??;
5556	assert!(
5557	tokio::time::timeout(Duration::from_secs(2), harness.recv_user_input_submission())
5558	.await
5559	.context("engine did not receive terminal cancellation")?
5560	.is_none(),
5561	"terminal cancellation delivered a submitted response"
5562	);
5563	let events = manager.events_since(&thread.id, None)?;
5564	let canceled = events
5565	.iter()
5566	.find(\|event\| {
5567	event.event == "user_input.canceled"
5568	&& event.payload.get("input_id").and_then(Value::as_str)
5569	== Some("input_terminal_order")
5570	})
5571	.context("missing terminal cancellation receipt")?;
5572	assert_eq!(canceled.payload["terminal"], true);
5573	assert!(
5574	manager
5575	.get_thread_detail(&thread.id)
5576	.await?
5577	.pending_user_inputs
5578	.is_empty()
5579	);
5580	Ok(())
5581	}
5582
5583	#[tokio::test]
5584	async fn thread_detail_cursor_precedes_projection_reads_at_terminal_boundary() -> Result<()> {
5585	let manager = test_manager(test_runtime_dir())?;
5586	let thread = manager
5587	.create_thread(CreateThreadRequest::default())
5588	.await?;
5589	let mut harness = install_mock_engine(&manager, &thread.id).await;
5590	let turn = manager
5591	.start_turn(
5592	&thread.id,
5593	StartTurnRequest {
5594	prompt: "complete while the snapshot is paused".to_string(),
5595	..StartTurnRequest::default()
5596	},
5597	)
5598	.await?;
5599	assert!(matches!(
5600	harness.rx_op.recv().await,
5601	Some(Op::SendMessage { .. })
5602	));
5603
5604	let (hook_tx, mut hook_rx) = mpsc::unbounded_channel();
5605	manager.set_snapshot_test_hook(hook_tx);
5606	let snapshot_manager = manager.clone();
5607	let snapshot_thread_id = thread.id.clone();
5608	let snapshot_task = tokio::spawn(async move {
5609	snapshot_manager
5610	.get_thread_detail(&snapshot_thread_id)
5611	.await
5612	});
5613
5614	let point = tokio::time::timeout(Duration::from_secs(2), hook_rx.recv())
5615	.await
5616	.context("snapshot did not capture its replay cursor")?
5617	.context("snapshot test hook closed")?;
5618	assert_eq!(point.thread_id, thread.id);
5619
5620	harness
5621	.tx_event
5622	.send(EngineEvent::TurnStarted {
5623	turn_id: "snapshot_terminal".to_string(),
5624	created_at: Utc::now(),
5625	route: None,
5626	})
5627	.await?;
5628	harness
5629	.tx_event
5630	.send(EngineEvent::MessageStarted { index: 0 })
5631	.await?;
5632	harness
5633	.tx_event
5634	.send(EngineEvent::MessageComplete { index: 0 })
5635	.await?;
5636	harness
5637	.tx_event
5638	.send(EngineEvent::TurnComplete {
5639	usage: Usage::default(),
5640	status: TurnOutcomeStatus::Completed,
5641	error: None,
5642	tool_catalog: None,
5643	base_url: None,
5644	})
5645	.await?;
5646
5647	assert!(
5648	tokio::time::timeout(Duration::from_millis(100), async {
5649	loop {
5650	if manager
5651	.events_since(&thread.id, Some(point.latest_seq))?
5652	.iter()
5653	.any(\|event\| {
5654	event.turn_id.as_deref() == Some(&turn.id)
5655	&& event.event == "turn.completed"
5656	})
5657	{
5658	break Ok::<_, anyhow::Error>(());
5659	}
5660	sleep(Duration::from_millis(10)).await;
5661	}
5662	})
5663	.await
5664	.is_err(),
5665	"terminal publication crossed a snapshot projection boundary"
5666	);
5667	point
5668	.resume
5669	.send(())
5670	.map_err(\|_\| anyhow!("snapshot dropped its resume barrier"))?;
5671
5672	let detail = snapshot_task.await.context("snapshot task panicked")??;
5673	assert_eq!(detail.latest_seq, point.latest_seq);
5674	assert_eq!(
5675	detail
5676	.turns
5677	.iter()
5678	.find(\|record\| record.id == turn.id)
5679	.map(\|record\| record.status),
5680	Some(RuntimeTurnStatus::InProgress),
5681	"the paused snapshot must retain the pre-terminal projection"
5682	);
5683
5684	let completed_event = tokio::time::timeout(Duration::from_secs(2), async {
5685	loop {
5686	if let Some(event) = manager
5687	.events_since(&thread.id, Some(detail.latest_seq))?
5688	.into_iter()
5689	.find(\|event\| {
5690	event.turn_id.as_deref() == Some(&turn.id) && event.event == "turn.completed"
5691	})
5692	{
5693	break Ok::<_, anyhow::Error>(event);
5694	}
5695	sleep(Duration::from_millis(10)).await;
5696	}
5697	})
5698	.await
5699	.context("terminal event did not follow the released snapshot boundary")??;
5700	assert!(completed_event.seq > detail.latest_seq);
5701	assert!(
5702	manager
5703	.events_since(&thread.id, Some(detail.latest_seq))?
5704	.iter()
5705	.any(\|event\| event.seq == completed_event.seq),
5706	"the same terminal transition must remain replayable from the snapshot cursor"
5707	);
5708	Ok(())
5709	}
5710
5711	#[tokio::test]
5712	async fn thread_detail_does_not_reenter_recovery_while_projection_is_locked() -> Result<()> {
5713	let manager = test_manager(test_runtime_dir())?;
5714	let thread = manager
5715	.create_thread(CreateThreadRequest::default())
5716	.await?;
5717	let turn_id = "turn_recovery_queued_during_snapshot";
5718	let mut turn = sample_turn(&thread.id, turn_id, RuntimeTurnStatus::Completed);
5719	turn.ended_at = Some(Utc::now());
5720	turn.duration_ms = Some(1);
5721	manager.store.save_turn(&turn)?;
5722	{
5723	let _thread_mutation = manager.store.thread_mutation.lock();
5724	let mut persisted_thread = manager.store.load_thread(&thread.id)?;
5725	persisted_thread.latest_turn_id = Some(turn_id.to_string());
5726	manager.store.save_thread(&persisted_thread)?;
5727	}
5728
5729	let (hook_tx, mut hook_rx) = mpsc::unbounded_channel();
5730	manager.set_snapshot_test_hook(hook_tx);
5731	let snapshot_manager = manager.clone();
5732	let snapshot_thread_id = thread.id.clone();
5733	let snapshot_task = tokio::spawn(async move {
5734	snapshot_manager
5735	.get_thread_detail(&snapshot_thread_id)
5736	.await
5737	});
5738	let point = tokio::time::timeout(Duration::from_secs(2), hook_rx.recv())
5739	.await
5740	.context("snapshot did not acquire its projection boundary")?
5741	.context("snapshot test hook closed")?;
5742
5743	// Queue after get_thread_detail's initial recovery flush, while its
5744	// projection lock is held. A nested get_thread call would see this receipt
5745	// and deadlock trying to reacquire that same lock.
5746	manager.queue_recovery_receipt(RecoveredTurnReceipt {
5747	turn: turn.clone(),
5748	unresolved_dynamic_tools: Vec::new(),
5749	});
5750	point
5751	.resume
5752	.send(())
5753	.map_err(\|_\| anyhow!("snapshot dropped its resume barrier"))?;
5754	let detail = tokio::time::timeout(Duration::from_secs(2), snapshot_task)
5755	.await
5756	.context("snapshot re-entered recovery while holding its projection lock")?
5757	.context("snapshot task panicked")??;
5758	assert_eq!(detail.thread.id, thread.id);
5759	assert_eq!(detail.latest_seq, point.latest_seq);
5760	assert!(manager.recovery_receipts.lock().contains_key(&thread.id));
5761
5762	// The next top-level observation flushes the receipt after the prior
5763	// snapshot has released its projection boundary.
5764	manager.get_thread(&thread.id).await?;
5765	let completed = manager
5766	.events_since(&thread.id, None)?
5767	.into_iter()
5768	.filter(\|event\| {
5769	event.event == "turn.completed" && event.turn_id.as_deref() == Some(turn_id)
5770	})
5771	.collect::<Vec<_>>();
5772	assert_eq!(completed.len(), 1);
5773	assert_eq!(completed[0].payload["recovered"], true);
5774	assert!(!manager.recovery_receipts.lock().contains_key(&thread.id));
5775	Ok(())
5776	}
5777
5778	#[tokio::test]
5779	async fn thread_detail_materializes_stream_prefixes_before_their_delta_cursor() -> Result<()> {
5780	let manager = test_manager(test_runtime_dir())?;
5781	let thread = manager
5782	.create_thread(CreateThreadRequest::default())
5783	.await?;
5784	let mut harness = install_mock_engine(&manager, &thread.id).await;
5785	let turn = manager
5786	.start_turn(
5787	&thread.id,
5788	StartTurnRequest {
5789	prompt: "snapshot both streamed prefixes".to_string(),
5790	..StartTurnRequest::default()
5791	},
5792	)
5793	.await?;
5794	assert!(matches!(
5795	harness.rx_op.recv().await,
5796	Some(Op::SendMessage { .. })
5797	));
5798
5799	harness
5800	.tx_event
5801	.send(EngineEvent::TurnStarted {
5802	turn_id: "delta_snapshot".to_string(),
5803	created_at: Utc::now(),
5804	route: None,
5805	})
5806	.await?;
5807	harness
5808	.tx_event
5809	.send(EngineEvent::MessageStarted { index: 0 })
5810	.await?;
5811	harness
5812	.tx_event
5813	.send(EngineEvent::MessageDelta {
5814	index: 0,
5815	content: "durable message prefix".to_string(),
5816	})
5817	.await?;
5818	harness
5819	.tx_event
5820	.send(EngineEvent::ThinkingStarted { index: 1 })
5821	.await?;
5822	harness
5823	.tx_event
5824	.send(EngineEvent::ThinkingDelta {
5825	index: 1,
5826	content: "durable reasoning prefix".to_string(),
5827	})
5828	.await?;
5829
5830	let deltas = tokio::time::timeout(Duration::from_secs(2), async {
5831	loop {
5832	let deltas = manager
5833	.events_since(&thread.id, None)?
5834	.into_iter()
5835	.filter(\|event\| {
5836	event.turn_id.as_deref() == Some(&turn.id) && event.event == "item.delta"
5837	})
5838	.collect::<Vec<_>>();
5839	if deltas.len() == 2 {
5840	break Ok::<_, anyhow::Error>(deltas);
5841	}
5842	sleep(Duration::from_millis(10)).await;
5843	}
5844	})
5845	.await
5846	.context("stream deltas were not durably sequenced")??;
5847
5848	let detail = manager.get_thread_detail(&thread.id).await?;
5849	let latest_delta_seq = deltas.iter().map(\|event\| event.seq).max().unwrap_or(0);
5850	assert!(detail.latest_seq >= latest_delta_seq);
5851	let message = detail
5852	.items
5853	.iter()
5854	.find(\|item\| item.kind == TurnItemKind::AgentMessage)
5855	.context("snapshot omitted the streaming message item")?;
5856	let reasoning = detail
5857	.items
5858	.iter()
5859	.find(\|item\| item.kind == TurnItemKind::AgentReasoning)
5860	.context("snapshot omitted the streaming reasoning item")?;
5861	assert_eq!(message.status, TurnItemLifecycleStatus::InProgress);
5862	assert_eq!(message.detail.as_deref(), Some("durable message prefix"));
5863	assert_eq!(reasoning.status, TurnItemLifecycleStatus::InProgress);
5864	assert_eq!(
5865	reasoning.detail.as_deref(),
5866	Some("durable reasoning prefix")
5867	);
5868	assert_eq!(
5869	manager.store.load_item(&message.id)?.detail,
5870	message.detail,
5871	"message prefix must already be on disk before its delta cursor"
5872	);
5873	assert_eq!(
5874	manager.store.load_item(&reasoning.id)?.detail,
5875	reasoning.detail,
5876	"reasoning prefix must already be on disk before its delta cursor"
5877	);
5878	assert!(
5879	manager
5880	.events_since(&thread.id, Some(detail.latest_seq))?
5881	.iter()
5882	.all(\|event\| event.event != "item.delta"),
5883	"the snapshot itself must carry every delta at or before latest_seq"
5884	);
5885
5886	harness
5887	.tx_event
5888	.send(EngineEvent::TurnComplete {
5889	usage: Usage::default(),
5890	status: TurnOutcomeStatus::Interrupted,
5891	error: None,
5892	tool_catalog: None,
5893	base_url: None,
5894	})
5895	.await?;
5896	Ok(())
5897	}
5898
5899	#[tokio::test]
5900	async fn thread_detail_delta_boundary_is_replay_idempotent() -> Result<()> {
5901	let manager = test_manager(test_runtime_dir())?;
5902	let thread = manager
5903	.create_thread(CreateThreadRequest::default())
5904	.await?;
5905	let mut harness = install_mock_engine(&manager, &thread.id).await;
5906	let turn = manager
5907	.start_turn(
5908	&thread.id,
5909	StartTurnRequest {
5910	prompt: "pause a snapshot across streamed deltas".to_string(),
5911	..StartTurnRequest::default()
5912	},
5913	)
5914	.await?;
5915	assert!(matches!(
5916	harness.rx_op.recv().await,
5917	Some(Op::SendMessage { .. })
5918	));
5919
5920	harness
5921	.tx_event
5922	.send(EngineEvent::TurnStarted {
5923	turn_id: "delta_boundary".to_string(),
5924	created_at: Utc::now(),
5925	route: None,
5926	})
5927	.await?;
5928	harness
5929	.tx_event
5930	.send(EngineEvent::MessageStarted { index: 0 })
5931	.await?;
5932
5933	let started = tokio::time::timeout(Duration::from_secs(2), async {
5934	loop {
5935	if let Some(event) = manager
5936	.events_since(&thread.id, None)?
5937	.into_iter()
5938	.find(\|event\| {
5939	event.turn_id.as_deref() == Some(&turn.id)
5940	&& event.event == "item.started"
5941	&& event.payload.pointer("/item/kind").and_then(Value::as_str)
5942	== Some("agent_message")
5943	})
5944	{
5945	break Ok::<_, anyhow::Error>(event);
5946	}
5947	sleep(Duration::from_millis(10)).await;
5948	}
5949	})
5950	.await
5951	.context("message item did not start")??;
5952	let item_id = started.item_id.context("started event omitted item id")?;
5953
5954	let (hook_tx, mut hook_rx) = mpsc::unbounded_channel();
5955	manager.set_snapshot_test_hook(hook_tx);
5956	let snapshot_manager = manager.clone();
5957	let snapshot_thread_id = thread.id.clone();
5958	let snapshot_task = tokio::spawn(async move {
5959	snapshot_manager
5960	.get_thread_detail(&snapshot_thread_id)
5961	.await
5962	});
5963	let point = tokio::time::timeout(Duration::from_secs(2), hook_rx.recv())
5964	.await
5965	.context("snapshot did not capture its replay cursor")?
5966	.context("snapshot test hook closed")?;
5967	assert!(point.latest_seq >= started.seq);
5968
5969	for content in ["A", "B"] {
5970	harness
5971	.tx_event
5972	.send(EngineEvent::MessageDelta {
5973	index: 0,
5974	content: content.to_string(),
5975	})
5976	.await?;
5977	}
5978	sleep(STREAM_DELTA_BATCH_MAX_LATENCY + Duration::from_millis(50)).await;
5979	assert!(
5980	manager
5981	.events_since(&thread.id, Some(point.latest_seq))?
5982	.iter()
5983	.all(\|event\| event.event != "item.delta"),
5984	"a delta must not publish while the snapshot holds its projection boundary"
5985	);
5986
5987	point
5988	.resume
5989	.send(())
5990	.map_err(\|_\| anyhow!("snapshot dropped its resume barrier"))?;
5991	let detail = snapshot_task.await.context("snapshot task panicked")??;
5992	let snapshotted = detail
5993	.items
5994	.iter()
5995	.find(\|item\| item.id == item_id)
5996	.context("snapshot omitted the streaming item")?;
5997	assert_eq!(snapshotted.detail.as_deref(), Some(""));
5998
5999	let delta = tokio::time::timeout(Duration::from_secs(2), async {
6000	loop {
6001	if let Some(event) = manager
6002	.events_since(&thread.id, Some(detail.latest_seq))?
6003	.into_iter()
6004	.find(\|event\| {
6005	event.item_id.as_deref() == Some(&item_id) && event.event == "item.delta"
6006	})
6007	{
6008	break Ok::<_, anyhow::Error>(event);
6009	}
6010	sleep(Duration::from_millis(10)).await;
6011	}
6012	})
6013	.await
6014	.context("batched delta did not publish after snapshot release")??;
6015	assert_eq!(
6016	delta.payload.get("delta").and_then(Value::as_str),
6017	Some("AB")
6018	);
6019	assert_eq!(
6020	manager.store.load_item(&item_id)?.detail.as_deref(),
6021	Some("AB")
6022	);
6023
6024	harness
6025	.tx_event
6026	.send(EngineEvent::TurnComplete {
6027	usage: Usage::default(),
6028	status: TurnOutcomeStatus::Interrupted,
6029	error: None,
6030	tool_catalog: None,
6031	base_url: None,
6032	})
6033	.await?;
6034	Ok(())
6035	}
6036
6037	#[tokio::test]
6038	async fn terminal_turn_cancels_pending_user_input_and_clears_snapshot() -> Result<()> {
6039	let manager = test_manager(test_runtime_dir())?;
6040	let thread = manager
6041	.create_thread(CreateThreadRequest::default())
6042	.await?;
6043	let mut harness = install_mock_engine(&manager, &thread.id).await;
6044	let turn = manager
6045	.start_turn(
6046	&thread.id,
6047	StartTurnRequest {
6048	prompt: "needs input before completion".to_string(),
6049	..StartTurnRequest::default()
6050	},
6051	)
6052	.await?;
6053	assert!(matches!(
6054	harness.rx_op.recv().await,
6055	Some(Op::SendMessage { .. })
6056	));
6057	harness
6058	.tx_event
6059	.send(EngineEvent::UserInputRequired {
6060	id: "input_terminal".to_string(),
6061	request: crate::tools::user_input::UserInputRequest {
6062	questions: vec![crate::tools::user_input::UserInputQuestion {
6063	header: "Continue".to_string(),
6064	id: "continue".to_string(),
6065	question: "Continue?".to_string(),
6066	options: vec![crate::tools::user_input::UserInputOption {
6067	label: "Yes".to_string(),
6068	description: "Continue now".to_string(),
6069	}],
6070	allow_free_text: false,
6071	multi_select: false,
6072	}],
6073	},
6074	})
6075	.await?;
6076
6077	let deadline = Instant::now() + Duration::from_secs(2);
6078	loop {
6079	if !manager
6080	.get_thread_detail(&thread.id)
6081	.await?
6082	.pending_user_inputs
6083	.is_empty()
6084	{
6085	break;
6086	}
6087	if Instant::now() >= deadline {
6088	bail!("pending user input did not reach the canonical snapshot");
6089	}
6090	sleep(Duration::from_millis(20)).await;
6091	}
6092
6093	harness
6094	.tx_event
6095	.send(EngineEvent::TurnComplete {
6096	usage: Usage::default(),
6097	status: TurnOutcomeStatus::Completed,
6098	error: None,
6099	tool_catalog: None,
6100	base_url: None,
6101	})
6102	.await?;
6103	let canceled = tokio::time::timeout(
6104	Duration::from_secs(2),
6105	harness.recv_user_input_cancellation(),
6106	)
6107	.await
6108	.expect("terminal user-input cancellation timed out");
6109	assert_eq!(canceled.as_deref(), Some("input_terminal"));
6110
6111	let deadline = Instant::now() + Duration::from_secs(2);
6112	loop {
6113	let detail = manager.get_thread_detail(&thread.id).await?;
6114	if detail.pending_user_inputs.is_empty()
6115	&& manager.events_since(&thread.id, None)?.iter().any(\|event\| {
6116	event.event == "user_input.canceled"
6117	&& event.turn_id.as_deref() == Some(turn.id.as_str())
6118	&& event.payload.get("input_id").and_then(Value::as_str)
6119	== Some("input_terminal")
6120	&& event.payload.get("terminal").and_then(Value::as_bool) == Some(true)
6121	})
6122	{
6123	break;
6124	}
6125	if Instant::now() >= deadline {
6126	bail!(
6127	"terminal user input was not cleared from the snapshot with a cancellation event"
6128	);
6129	}
6130	sleep(Duration::from_millis(20)).await;
6131	}
6132	Ok(())
6133	}
6134
6135	#[tokio::test]
6136	async fn dynamic_tool_result_settles_snapshot_and_emits_one_safe_resolution() -> Result<()> {
6137	use crate::tools::spec::DynamicToolExecutor;
6138
6139	let manager = test_manager(test_runtime_dir())?;
6140	let thread = manager
6141	.create_thread(CreateThreadRequest::default())
6142	.await?;
6143	let mut harness = install_mock_engine(&manager, &thread.id).await;
6144	let turn = manager
6145	.start_turn(
6146	&thread.id,
6147	StartTurnRequest {
6148	prompt: "run an external lookup".to_string(),
6149	..StartTurnRequest::default()
6150	},
6151	)
6152	.await?;
6153	assert!(matches!(
6154	harness.rx_op.recv().await,
6155	Some(Op::SendMessage { .. })
6156	));
6157	harness
6158	.tx_event
6159	.send(EngineEvent::TurnStarted {
6160	turn_id: "dynamic_result".to_string(),
6161	created_at: Utc::now(),
6162	route: None,
6163	})
6164	.await?;
6165
6166	const RESULT_SECRET: &str = "dynamic-result-secret";
6167	let executor = manager.clone();
6168	let executor_thread_id = thread.id.clone();
6169	let execution = tokio::spawn(async move {
6170	DynamicToolExecutor::execute_dynamic_tool(
6171	&executor,
6172	Some(executor_thread_id),
6173	Some("bench".to_string()),
6174	"lookup".to_string(),
6175	json!({ "record_id": "record-7" }),
6176	)
6177	.await
6178	});
6179
6180	let pending = tokio::time::timeout(Duration::from_secs(2), async {
6181	loop {
6182	let detail = manager.get_thread_detail(&thread.id).await?;
6183	if let Some(call) = detail.pending_dynamic_tool_calls.first() {
6184	break Ok::<_, anyhow::Error>((detail.latest_seq, call.clone()));
6185	}
6186	sleep(Duration::from_millis(10)).await;
6187	}
6188	})
6189	.await
6190	.context("dynamic tool call did not reach the canonical snapshot")??;
6191	let (snapshot_seq, call) = pending;
6192	assert_eq!(call.thread_id, thread.id);
6193	assert_eq!(call.turn_id, turn.id);
6194	assert_eq!(call.namespace.as_deref(), Some("bench"));
6195	assert_eq!(call.tool, "lookup");
6196	assert_eq!(call.arguments["record_id"], "record-7");
6197	let requested = manager
6198	.events_since(&thread.id, None)?
6199	.into_iter()
6200	.find(\|event\| {
6201	event.event == "tool_call.requested"
6202	&& event.payload.get("call_id").and_then(Value::as_str)
6203	== Some(call.call_id.as_str())
6204	})
6205	.context("dynamic tool request was not durable")?;
6206	assert!(requested.seq <= snapshot_seq);
6207	assert!(
6208	manager
6209	.events_since(&thread.id, Some(snapshot_seq))?
6210	.iter()
6211	.all(\|event\| event.event != "tool_call.requested"),
6212	"the pending call must be recoverable from the snapshot once replay starts at latest_seq"
6213	);
6214
6215	assert!(
6216	manager
6217	.deliver_dynamic_tool_result(
6218	&thread.id,
6219	&turn.id,
6220	&call.call_id,
6221	DynamicToolCallResult {
6222	success: true,
6223	content: vec![DynamicToolCallContent::InputText {
6224	text: RESULT_SECRET.to_string(),
6225	}],
6226	},
6227	)
6228	.await?
6229	);
6230	let result = execution.await.context("dynamic tool task panicked")??;
6231	assert_eq!(
6232	result.content, RESULT_SECRET,
6233	"the model-facing result changed"
6234	);
6235	assert!(
6236	manager
6237	.get_thread_detail(&thread.id)
6238	.await?
6239	.pending_dynamic_tool_calls
6240	.is_empty()
6241	);
6242	let resolved = manager
6243	.events_since(&thread.id, None)?
6244	.into_iter()
6245	.filter(\|event\| {
6246	event.event == "tool_call.resolved"
6247	&& event.payload.get("call_id").and_then(Value::as_str)
6248	== Some(call.call_id.as_str())
6249	})
6250	.collect::<Vec<_>>();
6251	assert_eq!(resolved.len(), 1);
6252	assert_eq!(resolved[0].payload["status"], "resolved");
6253	assert_eq!(resolved[0].payload["success"], true);
6254	assert!(
6255	!serde_json::to_string(&resolved)?.contains(RESULT_SECRET),
6256	"terminal dynamic-tool lifecycle must not echo result content"
6257	);
6258	assert!(
6259	!manager
6260	.deliver_dynamic_tool_result(
6261	&thread.id,
6262	&turn.id,
6263	&call.call_id,
6264	DynamicToolCallResult {
6265	success: true,
6266	content: Vec::new(),
6267	},
6268	)
6269	.await?,
6270	"a duplicate result must not settle or emit twice"
6271	);
6272	assert_eq!(
6273	manager
6274	.events_since(&thread.id, None)?
6275	.iter()
6276	.filter(\|event\| event.event == "tool_call.resolved")
6277	.count(),
6278	1
6279	);
6280
6281	harness
6282	.tx_event
6283	.send(EngineEvent::TurnComplete {
6284	usage: Usage::default(),
6285	status: TurnOutcomeStatus::Completed,
6286	error: None,
6287	tool_catalog: None,
6288	base_url: None,
6289	})
6290	.await?;
6291	Ok(())
6292	}
6293
6294	#[tokio::test]
6295	async fn dynamic_tool_result_receipt_outlives_canceled_delivery_future() -> Result<()> {
6296	use crate::tools::spec::DynamicToolExecutor;
6297
6298	let manager = test_manager(test_runtime_dir())?;
6299	let thread = manager
6300	.create_thread(CreateThreadRequest::default())
6301	.await?;
6302	let mut harness = install_mock_engine(&manager, &thread.id).await;
6303	let turn = manager
6304	.start_turn(
6305	&thread.id,
6306	StartTurnRequest {
6307	prompt: "settle a result after its HTTP future disappears".to_string(),
6308	..StartTurnRequest::default()
6309	},
6310	)
6311	.await?;
6312	assert!(matches!(
6313	harness.rx_op.recv().await,
6314	Some(Op::SendMessage { .. })
6315	));
6316	harness
6317	.tx_event
6318	.send(EngineEvent::TurnStarted {
6319	turn_id: "dynamic_detached_settlement".to_string(),
6320	created_at: Utc::now(),
6321	route: None,
6322	})
6323	.await?;
6324
6325	const RESULT_SECRET: &str = "detached-dynamic-result-secret";
6326	let executor = manager.clone();
6327	let executor_thread_id = thread.id.clone();
6328	let execution = tokio::spawn(async move {
6329	DynamicToolExecutor::execute_dynamic_tool(
6330	&executor,
6331	Some(executor_thread_id),
6332	Some("bench".to_string()),
6333	"detached_lookup".to_string(),
6334	json!({ "record_id": "record-detached" }),
6335	)
6336	.await
6337	});
6338
6339	let call = tokio::time::timeout(Duration::from_secs(2), async {
6340	loop {
6341	if let Some(call) = manager
6342	.get_thread_detail(&thread.id)
6343	.await?
6344	.pending_dynamic_tool_calls
6345	.first()
6346	.cloned()
6347	{
6348	break Ok::<_, anyhow::Error>(call);
6349	}
6350	tokio::task::yield_now().await;
6351	}
6352	})
6353	.await
6354	.context("dynamic tool call did not become pending")??;
6355
6356	// Stall terminal publication, submit the result, and wait until that path
6357	// owns the call. Canceling the API future from this point must not cancel
6358	// its detached receipt task or wake the model before the receipt is durable.
6359	let emit_guard = manager.event_emit.lock().await;
6360	let delivery_manager = manager.clone();
6361	let delivery_thread_id = thread.id.clone();
6362	let delivery_turn_id = turn.id.clone();
6363	let delivery_call_id = call.call_id.clone();
6364	let delivery = tokio::spawn(async move {
6365	delivery_manager
6366	.deliver_dynamic_tool_result(
6367	&delivery_thread_id,
6368	&delivery_turn_id,
6369	&delivery_call_id,
6370	DynamicToolCallResult {
6371	success: true,
6372	content: vec![DynamicToolCallContent::InputText {
6373	text: RESULT_SECRET.to_string(),
6374	}],
6375	},
6376	)
6377	.await
6378	});
6379	tokio::time::timeout(Duration::from_secs(2), async {
6380	loop {
6381	let settling = manager
6382	.pending_dynamic_tools
6383	.lock()
6384	.get(&call.call_id)
6385	.is_some_and(\|entry\| entry.sender.is_none());
6386	if settling {
6387	break;
6388	}
6389	tokio::task::yield_now().await;
6390	}
6391	})
6392	.await
6393	.context("result delivery never claimed the pending call")?;
6394	assert!(
6395	!execution.is_finished(),
6396	"the model consumed the result before its terminal receipt could commit"
6397	);
6398	assert!(
6399	!manager
6400	.deliver_dynamic_tool_result(
6401	&thread.id,
6402	&turn.id,
6403	&call.call_id,
6404	DynamicToolCallResult {
6405	success: false,
6406	content: Vec::new(),
6407	},
6408	)
6409	.await?,
6410	"a duplicate result stole a call whose terminal receipt was settling"
6411	);
6412	delivery.abort();
6413	assert!(
6414	delivery
6415	.await
6416	.expect_err("delivery API future must be canceled")
6417	.is_cancelled()
6418	);
6419	assert!(
6420	!execution.is_finished(),
6421	"canceling the delivery future woke the model without a receipt"
6422	);
6423
6424	drop(emit_guard);
6425	let model_result = tokio::time::timeout(Duration::from_secs(2), execution)
6426	.await
6427	.context("model did not receive the result after terminal publication")?
6428	.context("dynamic tool task panicked")??;
6429	assert_eq!(model_result.content, RESULT_SECRET);
6430
6431	let detail = manager.get_thread_detail(&thread.id).await?;
6432	assert!(detail.pending_dynamic_tool_calls.is_empty());
6433	let terminal = manager
6434	.events_since(&thread.id, None)?
6435	.into_iter()
6436	.filter(\|event\| {
6437	matches!(
6438	event.event.as_str(),
6439	"tool_call.resolved" \| "tool_call.timeout" \| "tool_call.canceled"
6440	) && event.payload.get("call_id").and_then(Value::as_str) == Some(call.call_id.as_str())
6441	})
6442	.collect::<Vec<_>>();
6443	assert_eq!(terminal.len(), 1);
6444	assert_eq!(terminal[0].event, "tool_call.resolved");
6445	assert_eq!(terminal[0].payload["success"], true);
6446	assert!(
6447	!serde_json::to_string(&terminal)?.contains(RESULT_SECRET),
6448	"the terminal receipt exposed result content"
6449	);
6450	assert!(
6451	!manager
6452	.deliver_dynamic_tool_result(
6453	&thread.id,
6454	&turn.id,
6455	&call.call_id,
6456	DynamicToolCallResult {
6457	success: true,
6458	content: Vec::new(),
6459	},
6460	)
6461	.await?,
6462	"a duplicate retry settled an already terminal call"
6463	);
6464
6465	harness
6466	.tx_event
6467	.send(EngineEvent::TurnComplete {
6468	usage: Usage::default(),
6469	status: TurnOutcomeStatus::Completed,
6470	error: None,
6471	tool_catalog: None,
6472	base_url: None,
6473	})
6474	.await?;
6475	Ok(())
6476	}
6477
6478	#[tokio::test]
6479	async fn dynamic_tool_result_acceptance_survives_receiver_close_before_append() -> Result<()> {
6480	let manager = test_manager(test_runtime_dir())?;
6481	let thread = manager
6482	.create_thread(CreateThreadRequest::default())
6483	.await?;
6484	let receiver = manager.register_pending_dynamic_tool_for_test(
6485	&thread.id,
6486	"turn_closed_receiver",
6487	"call_closed_receiver",
6488	)?;
6489	let emit_guard = manager.event_emit.lock().await;
6490	let delivery_manager = manager.clone();
6491	let delivery_thread_id = thread.id.clone();
6492	let delivery = tokio::spawn(async move {
6493	delivery_manager
6494	.deliver_dynamic_tool_result(
6495	&delivery_thread_id,
6496	"turn_closed_receiver",
6497	"call_closed_receiver",
6498	DynamicToolCallResult {
6499	success: true,
6500	content: vec![DynamicToolCallContent::InputText {
6501	text: "closed-receiver-secret".to_string(),
6502	}],
6503	},
6504	)
6505	.await
6506	});
6507	tokio::time::timeout(Duration::from_secs(2), async {
6508	loop {
6509	if manager
6510	.pending_dynamic_tools
6511	.lock()
6512	.get("call_closed_receiver")
6513	.is_some_and(\|entry\| entry.sender.is_none())
6514	{
6515	break;
6516	}
6517	tokio::task::yield_now().await;
6518	}
6519	})
6520	.await
6521	.context("result did not claim the call before append")?;
6522	drop(receiver);
6523	drop(emit_guard);
6524
6525	assert!(
6526	delivery.await.context("delivery task panicked")??,
6527	"durably accepted result was reported as missing after receiver close"
6528	);
6529	assert!(
6530	manager
6531	.get_thread_detail(&thread.id)
6532	.await?
6533	.pending_dynamic_tool_calls
6534	.is_empty()
6535	);
6536	let terminal = manager
6537	.events_since(&thread.id, None)?
6538	.into_iter()
6539	.filter(\|event\| {
6540	event.payload.get("call_id").and_then(Value::as_str) == Some("call_closed_receiver")
6541	&& matches!(
6542	event.event.as_str(),
6543	"tool_call.resolved" \| "tool_call.timeout" \| "tool_call.canceled"
6544	)
6545	})
6546	.collect::<Vec<_>>();
6547	assert_eq!(terminal.len(), 1);
6548	assert_eq!(terminal[0].event, "tool_call.resolved");
6549	assert_eq!(terminal[0].payload["result_accepted"], true);
6550	assert!(
6551	!serde_json::to_string(&terminal)?.contains("closed-receiver-secret"),
6552	"closed-receiver acceptance exposed result content"
6553	);
6554	Ok(())
6555	}
6556
6557	#[cfg(unix)]
6558	#[tokio::test]
6559	async fn dynamic_tool_receipt_append_failure_rolls_back_for_retry() -> Result<()> {
6560	let manager = test_manager(test_runtime_dir())?;
6561	let thread = manager
6562	.create_thread(CreateThreadRequest::default())
6563	.await?;
6564	let mut receiver = manager.register_pending_dynamic_tool_for_test(
6565	&thread.id,
6566	"turn_retry_after_append_failure",
6567	"call_retry_after_append_failure",
6568	)?;
6569	let claim = match manager.claim_pending_dynamic_tool(
6570	&thread.id,
6571	"turn_retry_after_append_failure",
6572	"call_retry_after_append_failure",
6573	) {
6574	PendingDynamicToolClaim::Claimed(claim) => claim,
6575	PendingDynamicToolClaim::Settling(_)
6576	\| PendingDynamicToolClaim::Indeterminate
6577	\| PendingDynamicToolClaim::Missing => {
6578	bail!("failed to claim the append-failure fixture")
6579	}
6580	};
6581
6582	// Replace this throwaway thread's event log with a symlink. Runtime store
6583	// hardening rejects the append deterministically, exercising settlement
6584	// rollback without relying on platform permission behavior.
6585	let events_path = manager.store.events_path(&thread.id)?;
6586	let backup_path = events_path.with_extension("jsonl.append-failure-backup");
6587	std::fs::rename(&events_path, &backup_path)?;
6588	std::os::unix::fs::symlink(&backup_path, &events_path)?;
6589	let ack = manager.spawn_dynamic_tool_settlement(
6590	claim,
6591	DynamicToolTerminalOutcome::Resolved(DynamicToolCallResult {
6592	success: true,
6593	content: vec![DynamicToolCallContent::InputText {
6594	text: "discarded-before-retry".to_string(),
6595	}],
6596	}),
6597	);
6598	let failed = RuntimeThreadManager::await_dynamic_tool_settlement(ack).await;
6599	std::fs::remove_file(&events_path)?;
6600	std::fs::rename(&backup_path, &events_path)?;
6601	assert!(
6602	failed.is_err(),
6603	"symlinked event append unexpectedly succeeded"
6604	);
6605
6606	{
6607	let pending = manager.pending_dynamic_tools.lock();
6608	let entry = pending
6609	.get("call_retry_after_append_failure")
6610	.context("failed receipt append stranded or removed the pending call")?;
6611	assert!(
6612	entry
6613	.sender
6614	.as_ref()
6615	.is_some_and(\|sender\| !sender.is_closed()),
6616	"failed receipt append left a Settling entry without its sender"
6617	);
6618	}
6619	assert_eq!(
6620	manager
6621	.get_thread_detail(&thread.id)
6622	.await?
6623	.pending_dynamic_tool_calls
6624	.len(),
6625	1,
6626	"rollback removed the snapshot-authoritative pending request"
6627	);
6628
6629	assert!(
6630	manager
6631	.deliver_dynamic_tool_result(
6632	&thread.id,
6633	"turn_retry_after_append_failure",
6634	"call_retry_after_append_failure",
6635	DynamicToolCallResult {
6636	success: true,
6637	content: vec![DynamicToolCallContent::InputText {
6638	text: "retry-result".to_string(),
6639	}],
6640	},
6641	)
6642	.await?,
6643	"retry did not settle the restored call"
6644	);
6645	let delivered = tokio::time::timeout(Duration::from_secs(2), &mut receiver)
6646	.await
6647	.context("restored receiver was not woken by retry")??;
6648	assert_eq!(
6649	delivered.content,
6650	vec![DynamicToolCallContent::InputText {
6651	text: "retry-result".to_string(),
6652	}]
6653	);
6654	let resolved = manager
6655	.events_since(&thread.id, None)?
6656	.into_iter()
6657	.filter(\|event\| {
6658	event.event == "tool_call.resolved"
6659	&& event.payload.get("call_id").and_then(Value::as_str)
6660	== Some("call_retry_after_append_failure")
6661	})
6662	.collect::<Vec<_>>();
6663	assert_eq!(resolved.len(), 1);
6664	assert!(
6665	!serde_json::to_string(&resolved)?.contains("retry-result"),
6666	"retried terminal receipt exposed result content"
6667	);
6668	Ok(())
6669	}
6670
6671	#[tokio::test]
6672	async fn dynamic_tool_post_write_failures_rollback_without_duplicate_receipts() -> Result<()> {
6673	let manager = test_manager(test_runtime_dir())?;
6674	let thread = manager
6675	.create_thread(CreateThreadRequest::default())
6676	.await?;
6677
6678	for (index, fault) in [
6679	EventAppendTestFault::AfterFlush,
6680	EventAppendTestFault::AfterSync,
6681	]
6682	.into_iter()
6683	.enumerate()
6684	{
6685	let turn_id = format!("turn_post_write_{index}");
6686	let call_id = format!("call_post_write_{index}");
6687	let result_text = format!("post-write-result-{index}");
6688	let mut receiver =
6689	manager.register_pending_dynamic_tool_for_test(&thread.id, &turn_id, &call_id)?;
6690	let fault_guard = EventAppendFaultGuard::arm(&thread.id, fault);
6691	let error = manager
6692	.deliver_dynamic_tool_result(
6693	&thread.id,
6694	&turn_id,
6695	&call_id,
6696	DynamicToolCallResult {
6697	success: true,
6698	content: vec![DynamicToolCallContent::InputText {
6699	text: result_text.clone(),
6700	}],
6701	},
6702	)
6703	.await
6704	.expect_err("injected post-write failure unexpectedly settled");
6705	drop(fault_guard);
6706	assert!(
6707	error.to_string().contains("rolled back"),
6708	"post-write failure was not classified retry-safe: {error}"
6709	);
6710
6711	let failed_snapshot = manager.get_thread_detail(&thread.id).await?;
6712	assert!(
6713	failed_snapshot
6714	.pending_dynamic_tool_calls
6715	.iter()
6716	.any(\|call\| call.call_id == call_id),
6717	"rolled-back call disappeared from the canonical snapshot"
6718	);
6719	assert!(
6720	manager
6721	.events_since(&thread.id, Some(failed_snapshot.latest_seq))?
6722	.is_empty(),
6723	"failed append left a replay-visible terminal suffix"
6724	);
6725	assert!(
6726	manager.events_since(&thread.id, None)?.iter().all(\|event\| {
6727	event.payload.get("call_id").and_then(Value::as_str) != Some(call_id.as_str())
6728	}),
6729	"failed append left a visible terminal record before retry"
6730	);
6731
6732	assert!(
6733	manager
6734	.deliver_dynamic_tool_result(
6735	&thread.id,
6736	&turn_id,
6737	&call_id,
6738	DynamicToolCallResult {
6739	success: true,
6740	content: vec![DynamicToolCallContent::InputText {
6741	text: result_text.clone(),
6742	}],
6743	},
6744	)
6745	.await?,
6746	"retry did not durably accept the rolled-back result"
6747	);
6748	let delivered = tokio::time::timeout(Duration::from_secs(2), &mut receiver)
6749	.await
6750	.context("retried result did not reach its model receiver")??;
6751	assert_eq!(
6752	delivered.content,
6753	vec![DynamicToolCallContent::InputText {
6754	text: result_text.clone(),
6755	}]
6756	);
6757
6758	let replay = manager.events_since(&thread.id, Some(failed_snapshot.latest_seq))?;
6759	let terminal = replay
6760	.iter()
6761	.filter(\|event\| {
6762	event.event == "tool_call.resolved"
6763	&& event.payload.get("call_id").and_then(Value::as_str)
6764	== Some(call_id.as_str())
6765	})
6766	.collect::<Vec<_>>();
6767	assert_eq!(terminal.len(), 1);
6768	assert!(terminal[0].seq > failed_snapshot.latest_seq);
6769	assert_eq!(terminal[0].payload["result_accepted"], true);
6770	assert!(
6771	!serde_json::to_string(&terminal)?.contains(&result_text),
6772	"retried terminal receipt exposed result content"
6773	);
6774	let settled_snapshot = manager.get_thread_detail(&thread.id).await?;
6775	assert!(
6776	settled_snapshot
6777	.pending_dynamic_tool_calls
6778	.iter()
6779	.all(\|call\| call.call_id != call_id)
6780	);
6781	assert!(settled_snapshot.latest_seq >= terminal[0].seq);
6782	}
6783	Ok(())
6784	}
6785
6786	#[tokio::test]
6787	async fn restart_recovers_terminal_turn_after_dynamic_receipt_append_failure() -> Result<()> {
6788	let data_dir = test_runtime_dir();
6789	let manager = test_manager(data_dir.clone())?;
6790	let thread = manager
6791	.create_thread(CreateThreadRequest::default())
6792	.await?;
6793	let turn_id = "turn_terminal_recovery";
6794	let call_id = "call_terminal_recovery";
6795	let mut turn = sample_turn(&thread.id, turn_id, RuntimeTurnStatus::Completed);
6796	turn.ended_at = Some(Utc::now());
6797	turn.duration_ms = Some(1);
6798	manager.store.save_turn(&turn)?;
6799	{
6800	let _thread_mutation = manager.store.thread_mutation.lock();
6801	let mut persisted_thread = manager.store.load_thread(&thread.id)?;
6802	persisted_thread.latest_turn_id = Some(turn_id.to_string());
6803	manager.store.save_thread(&persisted_thread)?;
6804	}
6805	let params = DynamicToolCallParams {
6806	thread_id: thread.id.clone(),
6807	turn_id: turn_id.to_string(),
6808	call_id: call_id.to_string(),
6809	namespace: Some("recovery".to_string()),
6810	tool: "recover_lookup".to_string(),
6811	arguments: json!({ "record": "recovery-only" }),
6812	};
6813	manager
6814	.emit_event_for_test(
6815	&thread.id,
6816	Some(turn_id),
6817	"tool_call.requested",
6818	json!(&params),
6819	)
6820	.await?;
6821	let receiver = manager.register_pending_dynamic_tool(params)?;
6822
6823	let fault_guard = EventAppendFaultGuard::arm(&thread.id, EventAppendTestFault::AfterSync);
6824	let error = manager
6825	.deliver_dynamic_tool_result(
6826	&thread.id,
6827	turn_id,
6828	call_id,
6829	DynamicToolCallResult {
6830	success: true,
6831	content: vec![DynamicToolCallContent::InputText {
6832	text: "never-committed-result".to_string(),
6833	}],
6834	},
6835	)
6836	.await
6837	.expect_err("injected terminal receipt append unexpectedly succeeded");
6838	drop(fault_guard);
6839	assert!(error.to_string().contains("rolled back"));
6840	assert!(manager.events_since(&thread.id, None)?.iter().all(\|event\| {
6841	event.event != "turn.completed"
6842	&& !matches!(
6843	event.event.as_str(),
6844	"tool_call.resolved" \| "tool_call.canceled" \| "tool_call.timeout"
6845	)
6846	}));
6847	drop(receiver);
6848	drop(manager);
6849
6850	let recovered = test_manager(data_dir.clone())?;
6851	// Opening is synchronous; the first async observation flushes queued
6852	// recovery receipts in terminal-call-before-turn order.
6853	let recovered_turn = recovered.get_thread(&thread.id).await?;
6854	assert_eq!(recovered_turn.latest_turn_id.as_deref(), Some(turn_id));
6855	let events = recovered.events_since(&thread.id, None)?;
6856	let canceled = events
6857	.iter()
6858	.filter(\|event\| {
6859	event.event == "tool_call.canceled"
6860	&& event.payload.get("call_id").and_then(Value::as_str) == Some(call_id)
6861	})
6862	.collect::<Vec<_>>();
6863	let completed = events
6864	.iter()
6865	.filter(\|event\| {
6866	event.event == "turn.completed" && event.turn_id.as_deref() == Some(turn_id)
6867	})
6868	.collect::<Vec<_>>();
6869	assert_eq!(canceled.len(), 1);
6870	assert_eq!(canceled[0].payload["reason"], "process_restart");
6871	assert_eq!(canceled[0].payload["recovered"], true);
6872	assert_eq!(completed.len(), 1);
6873	assert_eq!(completed[0].payload["recovered"], true);
6874	assert!(canceled[0].seq < completed[0].seq);
6875	assert_eq!(
6876	completed[0]
6877	.payload
6878	.pointer("/turn/status")
6879	.and_then(Value::as_str),
6880	Some("completed")
6881	);
6882
6883	// Re-observation and a second manager restart both remain idempotent.
6884	recovered.get_thread(&thread.id).await?;
6885	assert_eq!(
6886	recovered
6887	.events_since(&thread.id, None)?
6888	.iter()
6889	.filter(\|event\| {
6890	event.event == "turn.completed" && event.turn_id.as_deref() == Some(turn_id)
6891	})
6892	.count(),
6893	1
6894	);
6895	drop(recovered);
6896	let reopened = test_manager(data_dir)?;
6897	reopened.get_thread(&thread.id).await?;
6898	let reopened_events = reopened.events_since(&thread.id, None)?;
6899	assert_eq!(
6900	reopened_events
6901	.iter()
6902	.filter(\|event\| {
6903	event.event == "tool_call.canceled"
6904	&& event.payload.get("call_id").and_then(Value::as_str) == Some(call_id)
6905	})
6906	.count(),
6907	1
6908	);
6909	assert_eq!(
6910	reopened_events
6911	.iter()
6912	.filter(\|event\| {
6913	event.event == "turn.completed" && event.turn_id.as_deref() == Some(turn_id)
6914	})
6915	.count(),
6916	1
6917	);
6918	Ok(())
6919	}
6920
6921	#[tokio::test]
6922	async fn restart_reconciles_unresolved_dynamic_call_after_existing_turn_completion() -> Result<()> {
6923	let data_dir = test_runtime_dir();
6924	let manager = test_manager(data_dir.clone())?;
6925	let thread = manager
6926	.create_thread(CreateThreadRequest::default())
6927	.await?;
6928	let turn_id = "turn_legacy_completed_request";
6929	let call_id = "call_legacy_completed_request";
6930	let mut turn = sample_turn(&thread.id, turn_id, RuntimeTurnStatus::Completed);
6931	turn.ended_at = Some(Utc::now());
6932	turn.duration_ms = Some(1);
6933	manager.store.save_turn(&turn)?;
6934	let params = DynamicToolCallParams {
6935	thread_id: thread.id.clone(),
6936	turn_id: turn_id.to_string(),
6937	call_id: call_id.to_string(),
6938	namespace: Some("legacy".to_string()),
6939	tool: "legacy_lookup".to_string(),
6940	arguments: json!({ "record": "persisted-before-terminal-receipts" }),
6941	};
6942	manager
6943	.emit_event_for_test(
6944	&thread.id,
6945	Some(turn_id),
6946	"tool_call.requested",
6947	json!(&params),
6948	)
6949	.await?;
6950	manager
6951	.emit_event_for_test(
6952	&thread.id,
6953	Some(turn_id),
6954	"turn.completed",
6955	json!({ "turn": &turn }),
6956	)
6957	.await?;
6958	drop(manager);
6959
6960	let recovered = test_manager(data_dir)?;
6961	recovered.get_thread(&thread.id).await?;
6962	let events = recovered.events_since(&thread.id, None)?;
6963	let terminal_calls = events
6964	.iter()
6965	.filter(\|event\| {
6966	event.turn_id.as_deref() == Some(turn_id)
6967	&& event.payload.get("call_id").and_then(Value::as_str) == Some(call_id)
6968	&& matches!(
6969	event.event.as_str(),
6970	"tool_call.resolved" \| "tool_call.canceled" \| "tool_call.timeout"
6971	)
6972	})
6973	.collect::<Vec<_>>();
6974	assert_eq!(terminal_calls.len(), 1);
6975	assert_eq!(terminal_calls[0].event, "tool_call.canceled");
6976	assert_eq!(terminal_calls[0].payload["reason"], "process_restart");
6977	assert_eq!(terminal_calls[0].payload["recovered"], true);
6978	assert_eq!(
6979	events
6980	.iter()
6981	.filter(\|event\| {
6982	event.event == "turn.completed" && event.turn_id.as_deref() == Some(turn_id)
6983	})
6984	.count(),
6985	1,
6986	"recovery duplicated an already durable turn completion"
6987	);
6988	Ok(())
6989	}
6990
6991	#[tokio::test]
6992	async fn concurrent_recovery_dedupe_scans_emit_each_terminal_receipt_once() -> Result<()> {
6993	let manager = test_manager(test_runtime_dir())?;
6994	let thread = manager
6995	.create_thread(CreateThreadRequest::default())
6996	.await?;
6997	let turn_id = "turn_concurrent_recovery_dedupe";
6998	let call_id = "call_concurrent_recovery_dedupe";
6999	let mut turn = sample_turn(&thread.id, turn_id, RuntimeTurnStatus::Completed);
7000	turn.ended_at = Some(Utc::now());
7001	let params = DynamicToolCallParams {
7002	thread_id: thread.id.clone(),
7003	turn_id: turn_id.to_string(),
7004	call_id: call_id.to_string(),
7005	namespace: Some("recovery".to_string()),
7006	tool: "dedupe_lookup".to_string(),
7007	arguments: json!({ "record": "same-terminal-receipt" }),
7008	};
7009
7010	let (first_call, second_call) = tokio::join!(
7011	manager.emit_recovered_dynamic_cancellation_if_missing(&params),
7012	manager.emit_recovered_dynamic_cancellation_if_missing(&params),
7013	);
7014	assert_eq!(
7015	usize::from(first_call?) + usize::from(second_call?),
7016	1,
7017	"the event_emit boundary must linearize dynamic-terminal dedupe"
7018	);
7019
7020	let (first_turn, second_turn) = tokio::join!(
7021	manager.emit_turn_completed_if_missing(&turn, true),
7022	manager.emit_turn_completed_if_missing(&turn, true),
7023	);
7024	assert_eq!(
7025	usize::from(first_turn?) + usize::from(second_turn?),
7026	1,
7027	"the event_emit boundary must linearize turn-completion dedupe"
7028	);
7029
7030	let events = manager.events_since(&thread.id, None)?;
7031	assert_eq!(
7032	events
7033	.iter()
7034	.filter(\|event\| {
7035	event.event == "tool_call.canceled"
7036	&& event.turn_id.as_deref() == Some(turn_id)
7037	&& event.payload.get("call_id").and_then(Value::as_str) == Some(call_id)
7038	})
7039	.count(),
7040	1
7041	);
7042	assert_eq!(
7043	events
7044	.iter()
7045	.filter(\|event\| {
7046	event.event == "turn.completed" && event.turn_id.as_deref() == Some(turn_id)
7047	})
7048	.count(),
7049	1
7050	);
7051	Ok(())
7052	}
7053
7054	#[tokio::test]
7055	async fn consecutive_dynamic_receipt_failures_queue_in_process_recovery() -> Result<()> {
7056	let manager = test_manager(test_runtime_dir())?;
7057	let thread = manager
7058	.create_thread(CreateThreadRequest::default())
7059	.await?;
7060	let turn_id = "turn_consecutive_receipt_failures";
7061	let call_id = "call_consecutive_receipt_failures";
7062	let turn = sample_turn(&thread.id, turn_id, RuntimeTurnStatus::InProgress);
7063	manager.store.save_turn(&turn)?;
7064	{
7065	let _thread_mutation = manager.store.thread_mutation.lock();
7066	let mut persisted_thread = manager.store.load_thread(&thread.id)?;
7067	persisted_thread.latest_turn_id = Some(turn_id.to_string());
7068	manager.store.save_thread(&persisted_thread)?;
7069	}
7070	let params = DynamicToolCallParams {
7071	thread_id: thread.id.clone(),
7072	turn_id: turn_id.to_string(),
7073	call_id: call_id.to_string(),
7074	namespace: Some("recovery".to_string()),
7075	tool: "retry_lookup".to_string(),
7076	arguments: json!({ "record": "in-process" }),
7077	};
7078	let requested = manager
7079	.emit_event_for_test(
7080	&thread.id,
7081	Some(turn_id),
7082	"tool_call.requested",
7083	json!(&params),
7084	)
7085	.await?;
7086	let mut receiver = manager.register_pending_dynamic_tool(params)?;
7087
7088	// The submitted result and the monitor's terminal cancellation both fail
7089	// after fsync, with each JSONL line transactionally removed. The monitor
7090	// must retain an in-process recovery path instead of evicting the engine
7091	// with an Awaiting call and no future owner.
7092	let fault_guard =
7093	EventAppendFaultGuard::arm_repeated(&thread.id, EventAppendTestFault::AfterSync, 2);
7094	let result_error = manager
7095	.deliver_dynamic_tool_result(
7096	&thread.id,
7097	turn_id,
7098	call_id,
7099	DynamicToolCallResult {
7100	success: true,
7101	content: vec![DynamicToolCallContent::InputText {
7102	text: "rolled-back-result".to_string(),
7103	}],
7104	},
7105	)
7106	.await
7107	.expect_err("first injected receipt failure unexpectedly succeeded");
7108	assert!(result_error.to_string().contains("rolled back"));
7109	manager
7110	.settle_claimed_turn_failure(&thread.id, turn_id, "forced monitor failure")
7111	.await;
7112	drop(fault_guard);
7113
7114	assert!(
7115	manager
7116	.recovery_receipts
7117	.lock()
7118	.get(&thread.id)
7119	.is_some_and(\|receipts\| receipts.iter().any(\|receipt\| receipt.turn.id == turn_id)),
7120	"second retry-safe failure did not queue in-process recovery"
7121	);
7122	assert_eq!(manager.pending_dynamic_tools_count(), 1);
7123	assert!(
7124	tokio::time::timeout(Duration::from_millis(25), &mut receiver)
7125	.await
7126	.is_err(),
7127	"failed cancellation unexpectedly closed the model receiver"
7128	);
7129	assert!(manager.events_since(&thread.id, None)?.iter().all(\|event\| {
7130	event.event != "turn.completed"
7131	&& !matches!(
7132	event.event.as_str(),
7133	"tool_call.resolved" \| "tool_call.canceled" \| "tool_call.timeout"
7134	)
7135	}));
7136
7137	// The next async observation owns the queued retry. It durably cancels
7138	// the call before publishing exactly one recovered turn completion.
7139	manager.get_thread(&thread.id).await?;
7140	let closed = tokio::time::timeout(Duration::from_secs(2), &mut receiver)
7141	.await
7142	.context("recovery did not wake the model receiver")?;
7143	assert!(closed.is_err(), "terminal recovery delivered a tool result");
7144	let events = manager.events_since(&thread.id, None)?;
7145	let canceled = events
7146	.iter()
7147	.filter(\|event\| {
7148	event.event == "tool_call.canceled"
7149	&& event.payload.get("call_id").and_then(Value::as_str) == Some(call_id)
7150	})
7151	.collect::<Vec<_>>();
7152	let completed = events
7153	.iter()
7154	.filter(\|event\| {
7155	event.event == "turn.completed" && event.turn_id.as_deref() == Some(turn_id)
7156	})
7157	.collect::<Vec<_>>();
7158	assert_eq!(canceled.len(), 1);
7159	assert_eq!(canceled[0].payload["reason"], "turn_terminal");
7160	assert_eq!(canceled[0].payload["terminal"], true);
7161	assert_eq!(completed.len(), 1);
7162	assert_eq!(completed[0].payload["recovered"], true);
7163	assert!(canceled[0].seq < completed[0].seq);
7164	assert!(
7165	canceled[0].seq > requested.seq.saturating_add(1),
7166	"rolled-back append sequence values were unexpectedly reused"
7167	);
7168	assert_eq!(manager.pending_dynamic_tools_count(), 0);
7169	assert!(!manager.recovery_receipts.lock().contains_key(&thread.id));
7170
7171	manager.get_thread(&thread.id).await?;
7172	let replay = manager.events_since(&thread.id, None)?;
7173	assert_eq!(
7174	replay
7175	.iter()
7176	.filter(\|event\| {
7177	event.event == "tool_call.canceled"
7178	&& event.payload.get("call_id").and_then(Value::as_str) == Some(call_id)
7179	})
7180	.count(),
7181	1
7182	);
7183	assert_eq!(
7184	replay
7185	.iter()
7186	.filter(\|event\| {
7187	event.event == "turn.completed" && event.turn_id.as_deref() == Some(turn_id)
7188	})
7189	.count(),
7190	1
7191	);
7192	Ok(())
7193	}
7194
7195	#[test]
7196	fn pending_dynamic_tool_registry_rejects_duplicates_and_is_bounded() -> Result<()> {
7197	let manager = test_manager(test_runtime_dir())?;
7198	let mut receivers = Vec::with_capacity(MAX_PENDING_DYNAMIC_TOOL_CALLS);
7199	receivers.push(manager.register_pending_dynamic_tool_for_test(
7200	"thread-bound",
7201	"turn-bound",
7202	"call-0",
7203	)?);
7204	assert!(
7205	manager
7206	.register_pending_dynamic_tool_for_test("thread-bound", "turn-bound", "call-0",)
7207	.is_err(),
7208	"duplicate call IDs must not replace an existing result channel"
7209	);
7210
7211	for index in 1..MAX_PENDING_DYNAMIC_TOOL_CALLS {
7212	receivers.push(manager.register_pending_dynamic_tool_for_test(
7213	"thread-bound",
7214	"turn-bound",
7215	&format!("call-{index}"),
7216	)?);
7217	}
7218	assert_eq!(
7219	manager.pending_dynamic_tools_count(),
7220	MAX_PENDING_DYNAMIC_TOOL_CALLS
7221	);
7222	let error = manager
7223	.register_pending_dynamic_tool_for_test("thread-bound", "turn-bound", "call-over-limit")
7224	.expect_err("pending dynamic tool registry exceeded its hard limit");
7225	assert!(
7226	error
7227	.to_string()
7228	.contains("pending dynamic tool call limit")
7229	);
7230	Ok(())
7231	}
7232
7233	#[tokio::test]
7234	async fn dynamic_tool_timeout_clears_snapshot_and_emits_once() -> Result<()> {
7235	use crate::tools::spec::{DynamicToolExecutor, ToolError};
7236
7237	let _timeout_guard = test_dynamic_tool_timeout_ms(25);
7238	let manager = test_manager(test_runtime_dir())?;
7239	let thread = manager
7240	.create_thread(CreateThreadRequest::default())
7241	.await?;
7242	let mut harness = install_mock_engine(&manager, &thread.id).await;
7243	let turn = manager
7244	.start_turn(
7245	&thread.id,
7246	StartTurnRequest {
7247	prompt: "let an external lookup time out".to_string(),
7248	..StartTurnRequest::default()
7249	},
7250	)
7251	.await?;
7252	assert!(matches!(
7253	harness.rx_op.recv().await,
7254	Some(Op::SendMessage { .. })
7255	));
7256	harness
7257	.tx_event
7258	.send(EngineEvent::TurnStarted {
7259	turn_id: "dynamic_timeout".to_string(),
7260	created_at: Utc::now(),
7261	route: None,
7262	})
7263	.await?;
7264
7265	let error = DynamicToolExecutor::execute_dynamic_tool(
7266	&manager,
7267	Some(thread.id.clone()),
7268	None,
7269	"slow_lookup".to_string(),
7270	json!({ "marker": "request-only" }),
7271	)
7272	.await
7273	.expect_err("dynamic tool unexpectedly resolved");
7274	assert!(matches!(error, ToolError::Timeout { .. }));
7275	assert!(
7276	manager
7277	.get_thread_detail(&thread.id)
7278	.await?
7279	.pending_dynamic_tool_calls
7280	.is_empty()
7281	);
7282	let timeout_events = manager
7283	.events_since(&thread.id, None)?
7284	.into_iter()
7285	.filter(\|event\| event.event == "tool_call.timeout")
7286	.collect::<Vec<_>>();
7287	assert_eq!(timeout_events.len(), 1);
7288	assert_eq!(timeout_events[0].turn_id.as_deref(), Some(turn.id.as_str()));
7289	assert_eq!(timeout_events[0].payload["status"], "timeout");
7290	assert_eq!(timeout_events[0].payload["timeout_secs"], 0);
7291	assert!(timeout_events[0].payload.get("arguments").is_none());
7292
7293	harness
7294	.tx_event
7295	.send(EngineEvent::TurnComplete {
7296	usage: Usage::default(),
7297	status: TurnOutcomeStatus::Completed,
7298	error: None,
7299	tool_catalog: None,
7300	base_url: None,
7301	})
7302	.await?;
7303	Ok(())
7304	}
7305
7306	#[tokio::test]
7307	async fn terminal_turn_cancels_pending_dynamic_tool_exactly_once() -> Result<()> {
7308	use crate::tools::spec::DynamicToolExecutor;
7309
7310	let manager = test_manager(test_runtime_dir())?;
7311	let thread = manager
7312	.create_thread(CreateThreadRequest::default())
7313	.await?;
7314	let mut harness = install_mock_engine(&manager, &thread.id).await;
7315	let turn = manager
7316	.start_turn(
7317	&thread.id,
7318	StartTurnRequest {
7319	prompt: "cancel an external lookup with the turn".to_string(),
7320	..StartTurnRequest::default()
7321	},
7322	)
7323	.await?;
7324	assert!(matches!(
7325	harness.rx_op.recv().await,
7326	Some(Op::SendMessage { .. })
7327	));
7328	harness
7329	.tx_event
7330	.send(EngineEvent::TurnStarted {
7331	turn_id: "dynamic_cancel".to_string(),
7332	created_at: Utc::now(),
7333	route: None,
7334	})
7335	.await?;
7336
7337	let executor = manager.clone();
7338	let executor_thread_id = thread.id.clone();
7339	let execution = tokio::spawn(async move {
7340	DynamicToolExecutor::execute_dynamic_tool(
7341	&executor,
7342	Some(executor_thread_id),
7343	None,
7344	"cancel_lookup".to_string(),
7345	json!({ "id": "pending" }),
7346	)
7347	.await
7348	});
7349	tokio::time::timeout(Duration::from_secs(2), async {
7350	loop {
7351	if !manager
7352	.get_thread_detail(&thread.id)
7353	.await?
7354	.pending_dynamic_tool_calls
7355	.is_empty()
7356	{
7357	break Ok::<_, anyhow::Error>(());
7358	}
7359	sleep(Duration::from_millis(10)).await;
7360	}
7361	})
7362	.await
7363	.context("dynamic call did not become pending")??;
7364
7365	harness
7366	.tx_event
7367	.send(EngineEvent::TurnComplete {
7368	usage: Usage::default(),
7369	status: TurnOutcomeStatus::Interrupted,
7370	error: None,
7371	tool_catalog: None,
7372	base_url: None,
7373	})
7374	.await?;
7375	execution
7376	.await
7377	.context("dynamic tool task panicked")?
7378	.expect_err("terminal turn unexpectedly resolved the dynamic tool");
7379
7380	tokio::time::timeout(Duration::from_secs(2), async {
7381	loop {
7382	let detail = manager.get_thread_detail(&thread.id).await?;
7383	let canceled = manager
7384	.events_since(&thread.id, None)?
7385	.into_iter()
7386	.filter(\|event\| event.event == "tool_call.canceled")
7387	.collect::<Vec<_>>();
7388	if detail.pending_dynamic_tool_calls.is_empty() && canceled.len() == 1 {
7389	assert_eq!(canceled[0].payload["status"], "canceled");
7390	assert_eq!(canceled[0].payload["terminal"], true);
7391	break Ok::<_, anyhow::Error>(());
7392	}
7393	sleep(Duration::from_millis(10)).await;
7394	}
7395	})
7396	.await
7397	.context("terminal dynamic call did not disappear exactly once")??;
7398	assert_eq!(
7399	turn.id,
7400	manager
7401	.get_thread(&thread.id)
7402	.await?
7403	.latest_turn_id
7404	.unwrap()
7405	);
7406	Ok(())
7407	}
7408
7409	#[tokio::test]
7410	async fn approval_required_external_deny_is_denied() -> Result<()> {
7411	let manager = test_manager(test_runtime_dir())?;
7412	let thread = manager
7413	.create_thread(CreateThreadRequest {
7414	model: None,
7415	workspace: None,
7416	mode: None,
7417	allow_shell: None,
7418	trust_mode: None,
7419	auto_approve: None,
7420	archived: false,
7421	system_prompt: None,
7422	task_id: None,
7423	..Default::default()
7424	})
7425	.await?;
7426
7427	let mut harness = install_mock_engine(&manager, &thread.id).await;
7428	let _turn = manager
7429	.start_turn(
7430	&thread.id,
7431	StartTurnRequest {
7432	prompt: "needs approval".to_string(),
7433	input_summary: None,
7434	model: None,
7435	mode: None,
7436	allow_shell: None,
7437	trust_mode: None,
7438	auto_approve: None,
7439	..Default::default()
7440	},
7441	)
7442	.await?;
7443	assert!(matches!(
7444	harness.rx_op.recv().await,
7445	Some(Op::SendMessage { .. })
7446	));
7447
7448	harness
7449	.tx_event
7450	.send(EngineEvent::ApprovalRequired {
7451	approval_key: "key2".to_string(),
7452	approval_grouping_key: "key2".to_string(),
7453	id: "tool_external_deny".to_string(),
7454	tool_name: "exec_command".to_string(),
7455	description: "external deny".to_string(),
7456	input: serde_json::json!({}),
7457	intent_summary: None,
7458	approval_force_prompt: false,
7459	})
7460	.await?;
7461
7462	let deadline = Instant::now() + Duration::from_secs(2);
7463	while Instant::now() < deadline && manager.pending_approvals_count() == 0 {
7464	sleep(Duration::from_millis(20)).await;
7465	}
7466	assert_eq!(manager.pending_approvals_count(), 1);
7467
7468	assert!(manager.deliver_external_approval(
7469	"tool_external_deny",
7470	ExternalApprovalDecision::Deny { remember: false },
7471	));
7472	assert_eq!(
7473	harness.recv_approval_event().await,
7474	Some(MockApprovalEvent::Denied {
7475	id: "tool_external_deny".to_string(),
7476	})
7477	);
7478
7479	harness
7480	.tx_event
7481	.send(EngineEvent::TurnComplete {
7482	usage: Usage::default(),
7483	status: TurnOutcomeStatus::Completed,
7484	error: None,
7485	tool_catalog: None,
7486	base_url: None,
7487	})
7488	.await?;
7489	Ok(())
7490	}
7491
7492	#[tokio::test]
7493	async fn auto_review_force_prompt_is_denied_without_opening_a_modal() -> Result<()> {
7494	let manager = test_manager(test_runtime_dir())?;
7495	let thread = manager
7496	.create_thread(CreateThreadRequest {
7497	permission_posture: Some("auto-review".to_string()),
7498	..Default::default()
7499	})
7500	.await?;
7501	let mut harness = install_mock_engine(&manager, &thread.id).await;
7502	let turn = manager
7503	.start_turn(
7504	&thread.id,
7505	StartTurnRequest {
7506	prompt: "review the gated action".to_string(),
7507	..Default::default()
7508	},
7509	)
7510	.await?;
7511	assert!(matches!(
7512	harness.rx_op.recv().await,
7513	Some(Op::SendMessage {
7514	approval_mode: crate::tui::approval::ApprovalMode::Auto,
7515	..
7516	})
7517	));
7518
7519	harness
7520	.tx_event
7521	.send(EngineEvent::ApprovalRequired {
7522	approval_key: "auto_hold".to_string(),
7523	approval_grouping_key: "auto_hold".to_string(),
7524	id: "tool_auto_hold".to_string(),
7525	tool_name: "exec_command".to_string(),
7526	description: "policy hold under auto review".to_string(),
7527	input: serde_json::json!({}),
7528	intent_summary: None,
7529	approval_force_prompt: true,
7530	})
7531	.await?;
7532
7533	let decision = tokio::time::timeout(Duration::from_secs(2), harness.recv_approval_event())
7534	.await
7535	.context("Auto-Review hold should resolve without a modal")?;
7536	assert_eq!(
7537	decision,
7538	Some(MockApprovalEvent::Denied {
7539	id: "tool_auto_hold".to_string(),
7540	})
7541	);
7542	assert_eq!(manager.pending_approvals_count(), 0);
7543	assert!(manager.events_since(&thread.id, None)?.iter().any(\|event\| {
7544	event.event == "approval.decided"
7545	&& event.payload.get("approval_id").and_then(Value::as_str) == Some("tool_auto_hold")
7546	&& event.payload.get("posture").and_then(Value::as_str) == Some("auto_review")
7547	}));
7548
7549	harness
7550	.tx_event
7551	.send(EngineEvent::TurnComplete {
7552	usage: Usage::default(),
7553	status: TurnOutcomeStatus::Completed,
7554	error: None,
7555	tool_catalog: None,
7556	base_url: None,
7557	})
7558	.await?;
7559	let terminal = wait_for_terminal_turn(&manager, &turn.id, Duration::from_secs(2)).await?;
7560	assert_eq!(terminal.status, RuntimeTurnStatus::Completed);
7561	Ok(())
7562	}
7563
7564	#[tokio::test]
7565	async fn approval_timeout_denies_clears_ui_and_next_turn_can_start() -> Result<()> {
7566	let _timeout_guard = test_approval_timeout_ms(25);
7567	let manager = test_manager(test_runtime_dir())?;
7568	let thread = manager
7569	.create_thread(CreateThreadRequest {
7570	model: None,
7571	workspace: None,
7572	mode: None,
7573	allow_shell: None,
7574	trust_mode: None,
7575	auto_approve: None,
7576	archived: false,
7577	system_prompt: None,
7578	task_id: None,
7579	..Default::default()
7580	})
7581	.await?;
7582
7583	let mut harness = install_mock_engine(&manager, &thread.id).await;
7584	let turn = manager
7585	.start_turn(
7586	&thread.id,
7587	StartTurnRequest {
7588	prompt: "needs approval".to_string(),
7589	input_summary: None,
7590	model: None,
7591	mode: None,
7592	allow_shell: None,
7593	trust_mode: None,
7594	auto_approve: None,
7595	..Default::default()
7596	},
7597	)
7598	.await?;
7599	assert!(matches!(
7600	harness.rx_op.recv().await,
7601	Some(Op::SendMessage { .. })
7602	));
7603
7604	harness
7605	.tx_event
7606	.send(EngineEvent::ApprovalRequired {
7607	approval_key: "timeout_key".to_string(),
7608	approval_grouping_key: "timeout_key".to_string(),
7609	id: "tool_timeout".to_string(),
7610	tool_name: "exec_command".to_string(),
7611	description: "external timeout".to_string(),
7612	input: serde_json::json!({}),
7613	intent_summary: None,
7614	approval_force_prompt: false,
7615	})
7616	.await?;
7617
7618	let decision = tokio::time::timeout(Duration::from_secs(2), harness.recv_approval_event())
7619	.await
7620	.context("approval timeout should deny the engine")?;
7621	assert_eq!(
7622	decision,
7623	Some(MockApprovalEvent::Denied {
7624	id: "tool_timeout".to_string(),
7625	})
7626	);
7627	assert_eq!(manager.pending_approvals_count(), 0);
7628
7629	let events = manager.events_since(&thread.id, None)?;
7630	assert!(
7631	events.iter().any(\|event\| {
7632	event.event == "approval.timeout"
7633	&& event.payload.get("approval_id").and_then(Value::as_str) == Some("tool_timeout")
7634	}),
7635	"timeout event should be persisted"
7636	);
7637	assert!(
7638	events.iter().any(\|event\| {
7639	event.event == "approval.decided"
7640	&& event.payload.get("approval_id").and_then(Value::as_str) == Some("tool_timeout")
7641	&& event.payload.get("decision").and_then(Value::as_str) == Some("deny")
7642	&& event.payload.get("timeout").and_then(Value::as_bool) == Some(true)
7643	}),
7644	"timeout should also emit approval.decided so clients can clear pending UI"
7645	);
7646
7647	harness
7648	.tx_event
7649	.send(EngineEvent::TurnComplete {
7650	usage: Usage::default(),
7651	status: TurnOutcomeStatus::Completed,
7652	error: None,
7653	tool_catalog: None,
7654	base_url: None,
7655	})
7656	.await?;
7657	let terminal = wait_for_terminal_turn(&manager, &turn.id, Duration::from_secs(2)).await?;
7658	assert_eq!(terminal.status, RuntimeTurnStatus::Completed);
7659
7660	let _next = manager
7661	.start_turn(
7662	&thread.id,
7663	StartTurnRequest {
7664	prompt: "after timeout".to_string(),
7665	input_summary: None,
7666	model: None,
7667	mode: None,
7668	allow_shell: None,
7669	trust_mode: None,
7670	auto_approve: None,
7671	..Default::default()
7672	},
7673	)
7674	.await?;
7675	assert!(
7676	matches!(harness.rx_op.recv().await, Some(Op::SendMessage { .. })),
7677	"thread should accept a fresh turn after approval timeout cleanup"
7678	);
7679
7680	Ok(())
7681	}
7682
7683	#[tokio::test]
7684	async fn thinking_delta_emits_agent_reasoning_item() -> Result<()> {
7685	let manager = test_manager(test_runtime_dir())?;
7686	let thread = manager
7687	.create_thread(CreateThreadRequest {
7688	model: None,
7689	workspace: None,
7690	mode: None,
7691	allow_shell: None,
7692	trust_mode: None,
7693	auto_approve: Some(true),
7694	archived: false,
7695	system_prompt: None,
7696	task_id: None,
7697	..Default::default()
7698	})
7699	.await?;
7700	let mut harness = install_mock_engine(&manager, &thread.id).await;
7701	let mut event_rx = manager.subscribe_events();
7702	let _turn = manager
7703	.start_turn(
7704	&thread.id,
7705	StartTurnRequest {
7706	prompt: "show your thinking".to_string(),
7707	input_summary: None,
7708	model: None,
7709	mode: None,
7710	allow_shell: None,
7711	trust_mode: None,
7712	auto_approve: Some(true),
7713	..Default::default()
7714	},
7715	)
7716	.await?;
7717	assert!(matches!(
7718	harness.rx_op.recv().await,
7719	Some(Op::SendMessage { .. })
7720	));
7721
7722	harness
7723	.tx_event
7724	.send(EngineEvent::ThinkingStarted { index: 0 })
7725	.await?;
7726	harness
7727	.tx_event
7728	.send(EngineEvent::ThinkingDelta {
7729	index: 0,
7730	content: "Let me reason about this.".to_string(),
7731	})
7732	.await?;
7733	harness
7734	.tx_event
7735	.send(EngineEvent::ThinkingComplete { index: 0 })
7736	.await?;
7737	harness
7738	.tx_event
7739	.send(EngineEvent::TurnComplete {
7740	usage: Usage::default(),
7741	status: TurnOutcomeStatus::Completed,
7742	error: None,
7743	tool_catalog: None,
7744	base_url: None,
7745	})
7746	.await?;
7747
7748	// A busy or constrained runner can be quiet for more than one 200 ms poll
7749	// even though the engine is still making progress. Keep polling until the
7750	// actual deadline instead of treating the first quiet interval as failure.
7751	let deadline = Instant::now() + Duration::from_secs(5);
7752	let mut delta_seen = false;
7753	let mut completed_seen = false;
7754	while Instant::now() < deadline && (!delta_seen \|\| !completed_seen) {
7755	match tokio::time::timeout(Duration::from_millis(200), event_rx.recv()).await {
7756	Ok(Ok(record)) => {
7757	if record.event == "item.delta"
7758	&& record.payload.get("kind").and_then(\|v\| v.as_str())
7759	== Some("agent_reasoning")
7760	{
7761	delta_seen = true;
7762	assert_eq!(
7763	record.payload.get("delta").and_then(\|v\| v.as_str()),
7764	Some("Let me reason about this.")
7765	);
7766	}
7767	if record.event == "item.completed"
7768	&& record
7769	.payload
7770	.get("item")
7771	.and_then(\|v\| v.get("kind"))
7772	.and_then(\|v\| v.as_str())
7773	== Some("agent_reasoning")
7774	{
7775	completed_seen = true;
7776	}
7777	}
7778	Ok(Err(_)) => break,
7779	Err(_) => continue,
7780	}
7781	}
7782	assert!(delta_seen, "expected item.delta with kind=agent_reasoning");
7783	assert!(
7784	completed_seen,
7785	"expected item.completed for the reasoning item"
7786	);
7787	Ok(())
7788	}
7789
7790	#[tokio::test]
7791	async fn deliver_external_approval_for_unknown_id_returns_false() {
7792	let manager = test_manager(test_runtime_dir()).expect("manager");
7793	assert!(!manager.deliver_external_approval(
7794	"no_such_approval",
7795	ExternalApprovalDecision::Allow { remember: false },
7796	));
7797	assert_eq!(manager.pending_approvals_count(), 0);
7798	}
7799
7800	#[tokio::test]
7801	async fn approval_required_remember_flips_thread_auto_approve() -> Result<()> {
7802	let manager = test_manager(test_runtime_dir())?;
7803	let thread = manager
7804	.create_thread(CreateThreadRequest {
7805	model: None,
7806	workspace: None,
7807	mode: None,
7808	allow_shell: None,
7809	trust_mode: None,
7810	auto_approve: None,
7811	archived: false,
7812	system_prompt: None,
7813	task_id: None,
7814	..Default::default()
7815	})
7816	.await?;
7817	assert!(!manager.store.load_thread(&thread.id)?.auto_approve);
7818
7819	let mut harness = install_mock_engine(&manager, &thread.id).await;
7820	let turn = manager
7821	.start_turn(
7822	&thread.id,
7823	StartTurnRequest {
7824	prompt: "needs approval".to_string(),
7825	input_summary: None,
7826	model: None,
7827	mode: None,
7828	allow_shell: None,
7829	trust_mode: None,
7830	auto_approve: None,
7831	..Default::default()
7832	},
7833	)
7834	.await?;
7835	assert!(matches!(
7836	harness.rx_op.recv().await,
7837	Some(Op::SendMessage { .. })
7838	));
7839
7840	harness
7841	.tx_event
7842	.send(EngineEvent::ApprovalRequired {
7843	approval_key: "key3".to_string(),
7844	approval_grouping_key: "key3".to_string(),
7845	id: "tool_remember".to_string(),
7846	tool_name: "exec_command".to_string(),
7847	description: "remember=true".to_string(),
7848	input: serde_json::json!({}),
7849	intent_summary: None,
7850	approval_force_prompt: false,
7851	})
7852	.await?;
7853
7854	let deadline = Instant::now() + Duration::from_secs(2);
7855	while Instant::now() < deadline && manager.pending_approvals_count() == 0 {
7856	sleep(Duration::from_millis(20)).await;
7857	}
7858	assert!(manager.deliver_external_approval(
7859	"tool_remember",
7860	ExternalApprovalDecision::Allow { remember: true },
7861	));
7862	let _ = harness.recv_approval_event().await;
7863
7864	assert!(
7865	manager.store.load_thread(&thread.id)?.auto_approve,
7866	"remember=true should flip thread auto_approve"
7867	);
7868	assert_eq!(
7869	manager.active_turn_flags(&thread.id, &turn.id).await,
7870	Some((true, false)),
7871	"remember=true should update the active turn used by subsequent approvals"
7872	);
7873
7874	harness
7875	.tx_event
7876	.send(EngineEvent::TurnComplete {
7877	usage: Usage::default(),
7878	status: TurnOutcomeStatus::Completed,
7879	error: None,
7880	tool_catalog: None,
7881	base_url: None,
7882	})
7883	.await?;
7884	Ok(())
7885	}
7886
7887	#[tokio::test]
7888	async fn elevation_required_with_stale_active_turn_is_denied() -> Result<()> {
7889	let manager = test_manager(test_runtime_dir())?;
7890	let thread = manager
7891	.create_thread(CreateThreadRequest {
7892	model: None,
7893	workspace: None,
7894	mode: None,
7895	allow_shell: None,
7896	trust_mode: Some(true),
7897	auto_approve: Some(true),
7898	archived: false,
7899	system_prompt: None,
7900	task_id: None,
7901	..Default::default()
7902	})
7903	.await?;
7904
7905	let mut harness = install_mock_engine(&manager, &thread.id).await;
7906	let turn = manager
7907	.start_turn(
7908	&thread.id,
7909	StartTurnRequest {
7910	prompt: "needs elevation".to_string(),
7911	input_summary: None,
7912	model: None,
7913	mode: None,
7914	allow_shell: None,
7915	trust_mode: Some(true),
7916	auto_approve: Some(true),
7917	..Default::default()
7918	},
7919	)
7920	.await?;
7921
7922	assert!(matches!(
7923	harness.rx_op.recv().await,
7924	Some(Op::SendMessage { .. })
7925	));
7926	{
7927	let mut active = manager.active.lock().await;
7928	let state = active
7929	.engines
7930	.get_mut(&thread.id)
7931	.context("missing active thread state")?;
7932	state.active_turn = None;
7933	}
7934
7935	harness
7936	.tx_event
7937	.send(EngineEvent::ElevationRequired {
7938	tool_id: "tool_stale_elevated".to_string(),
7939	tool_name: "exec_command".to_string(),
7940	command: None,
7941	denial_reason: "sandbox denied".to_string(),
7942	blocked_network: false,
7943	blocked_write: false,
7944	})
7945	.await?;
7946
7947	assert_eq!(
7948	harness.recv_approval_event().await,
7949	Some(MockApprovalEvent::Denied {
7950	id: "tool_stale_elevated".to_string(),
7951	})
7952	);
7953
7954	harness
7955	.tx_event
7956	.send(EngineEvent::TurnComplete {
7957	usage: Usage {
7958	input_tokens: 0,
7959	output_tokens: 0,
7960	..Usage::default()
7961	},
7962	status: TurnOutcomeStatus::Completed,
7963	error: None,
7964	tool_catalog: None,
7965	base_url: None,
7966	})
7967	.await?;
7968
7969	let terminal = wait_for_terminal_turn(&manager, &turn.id, Duration::from_secs(2)).await?;
7970	assert_eq!(terminal.status, RuntimeTurnStatus::Completed);
7971	Ok(())
7972	}
7973
7974	#[tokio::test]
7975	async fn steer_turn_on_active_turn_records_item_and_event() -> Result<()> {
7976	let manager = test_manager(test_runtime_dir())?;
7977	let thread = manager
7978	.create_thread(CreateThreadRequest {
7979	model: None,
7980	workspace: None,
7981	mode: None,
7982	allow_shell: None,
7983	trust_mode: None,
7984	auto_approve: None,
7985	archived: false,
7986	system_prompt: None,
7987	task_id: None,
7988	..Default::default()
7989	})
7990	.await?;
7991
7992	let harness = install_mock_engine(&manager, &thread.id).await;
7993	let mut rx_op = harness.rx_op;
7994	let mut rx_steer = harness.rx_steer;
7995	let tx_event = harness.tx_event;
7996	let (steer_seen_tx, steer_seen_rx) = oneshot::channel::<String>();
7997	tokio::spawn(async move {
7998	if matches!(rx_op.recv().await, Some(Op::SendMessage { .. })) {
7999	let _ = tx_event
8000	.send(EngineEvent::TurnStarted {
8001	turn_id: "engine_turn_steer".to_string(),
8002	created_at: chrono::Utc::now(),
8003	route: None,
8004	})
8005	.await;
8006	if let Some(steer) = rx_steer.recv().await {
8007	let _ = steer_seen_tx.send(steer);
8008	}
8009	let _ = tx_event
8010	.send(EngineEvent::MessageStarted { index: 0 })
8011	.await;
8012	let _ = tx_event
8013	.send(EngineEvent::MessageDelta {
8014	index: 0,
8015	content: "steered response".to_string(),
8016	})
8017	.await;
8018	let _ = tx_event
8019	.send(EngineEvent::MessageComplete { index: 0 })
8020	.await;
8021	let _ = tx_event
8022	.send(EngineEvent::TurnComplete {
8023	usage: Usage {
8024	input_tokens: 8,
8025	output_tokens: 9,
8026	..Usage::default()
8027	},
8028	status: TurnOutcomeStatus::Completed,
8029	error: None,
8030	tool_catalog: None,
8031	base_url: None,
8032	})
8033	.await;
8034	}
8035	});
8036
8037	let turn = manager
8038	.start_turn(
8039	&thread.id,
8040	StartTurnRequest {
8041	prompt: "initial".to_string(),
8042	input_summary: None,
8043	model: None,
8044	mode: None,
8045	allow_shell: None,
8046	trust_mode: None,
8047	auto_approve: None,
8048	..Default::default()
8049	},
8050	)
8051	.await?;
8052
8053	let steer_text = "add bullet list".to_string();
8054	let steered_turn = manager
8055	.steer_turn(
8056	&thread.id,
8057	&turn.id,
8058	SteerTurnRequest {
8059	prompt: steer_text.clone(),
8060	},
8061	)
8062	.await?;
8063	assert_eq!(steered_turn.steer_count, 1);
8064	let observed_steer = steer_seen_rx
8065	.await
8066	.context("driver did not receive steer")?;
8067	assert_eq!(observed_steer, steer_text);
8068
8069	let final_turn = wait_for_terminal_turn(&manager, &turn.id, Duration::from_secs(2)).await?;
8070	assert_eq!(final_turn.status, RuntimeTurnStatus::Completed);
8071	assert_eq!(final_turn.steer_count, 1);
8072
8073	let events = manager.events_since(&thread.id, None)?;
8074	assert!(events.iter().any(\|ev\| ev.event == "turn.steered"));
8075	assert!(events.iter().any(\|ev\| {
8076	ev.event == "item.completed"
8077	&& ev
8078	.payload
8079	.get("item")
8080	.and_then(\|item\| item.get("detail"))
8081	.and_then(Value::as_str)
8082	== Some("add bullet list")
8083	}));
8084	Ok(())
8085	}
8086
8087	#[tokio::test]
8088	async fn steer_receipts_outlive_caller_cancellation_after_engine_acceptance() -> Result<()> {
8089	let manager = test_manager(test_runtime_dir())?;
8090	let thread = manager
8091	.create_thread(CreateThreadRequest::default())
8092	.await?;
8093	let harness = install_mock_engine(&manager, &thread.id).await;
8094	let mut rx_op = harness.rx_op;
8095	let mut rx_steer = harness.rx_steer;
8096	let tx_event = harness.tx_event;
8097
8098	let turn = manager
8099	.start_turn(
8100	&thread.id,
8101	StartTurnRequest {
8102	prompt: "initial".to_string(),
8103	..Default::default()
8104	},
8105	)
8106	.await?;
8107	assert!(matches!(rx_op.recv().await, Some(Op::SendMessage { .. })));
8108
8109	// Hold publication after durable persistence and mailbox acceptance so the
8110	// API future can be cancelled while the detached receipt task is pending.
8111	let emit_guard = manager.event_emit.lock().await;
8112	let steer_manager = manager.clone();
8113	let thread_id = thread.id.clone();
8114	let turn_id = turn.id.clone();
8115	let steer_task = tokio::spawn(async move {
8116	steer_manager
8117	.steer_turn(
8118	&thread_id,
8119	&turn_id,
8120	SteerTurnRequest {
8121	prompt: "keep the accepted steer".to_string(),
8122	},
8123	)
8124	.await
8125	});
8126	assert_eq!(
8127	tokio::time::timeout(Duration::from_secs(2), rx_steer.recv()).await?,
8128	Some("keep the accepted steer".to_string())
8129	);
8130	steer_task.abort();
8131	assert!(
8132	steer_task
8133	.await
8134	.expect_err("caller task must be cancelled")
8135	.is_cancelled()
8136	);
8137	drop(emit_guard);
8138
8139	let deadline = Instant::now() + Duration::from_secs(2);
8140	loop {
8141	let events = manager.events_since(&thread.id, None)?;
8142	let steered = events.iter().any(\|event\| event.event == "turn.steered");
8143	let completed = events.iter().any(\|event\| {
8144	event.event == "item.completed"
8145	&& event
8146	.payload
8147	.get("item")
8148	.and_then(\|item\| item.get("detail"))
8149	.and_then(Value::as_str)
8150	== Some("keep the accepted steer")
8151	});
8152	if steered && completed {
8153	break;
8154	}
8155	if Instant::now() >= deadline {
8156	bail!("detached steer receipts were not persisted after caller cancellation");
8157	}
8158	tokio::task::yield_now().await;
8159	}
8160
8161	let persisted_turn = manager.store.load_turn(&turn.id)?;
8162	assert_eq!(persisted_turn.steer_count, 1);
8163	let items = manager.store.list_items_for_turn(&turn.id)?;
8164	let steer_item = items
8165	.iter()
8166	.find(\|item\| item.detail.as_deref() == Some("keep the accepted steer"))
8167	.context("accepted steer item must remain durable")?;
8168	assert!(persisted_turn.item_ids.contains(&steer_item.id));
8169
8170	tx_event
8171	.send(EngineEvent::MessageStarted { index: 0 })
8172	.await?;
8173	tx_event
8174	.send(EngineEvent::MessageDelta {
8175	index: 0,
8176	content: "accepted steer completed".to_string(),
8177	})
8178	.await?;
8179	tx_event
8180	.send(EngineEvent::MessageComplete { index: 0 })
8181	.await?;
8182	tx_event
8183	.send(EngineEvent::TurnComplete {
8184	usage: Usage::default(),
8185	status: TurnOutcomeStatus::Completed,
8186	error: None,
8187	tool_catalog: None,
8188	base_url: None,
8189	})
8190	.await?;
8191	let terminal = wait_for_terminal_turn(&manager, &turn.id, Duration::from_secs(2)).await?;
8192	assert_eq!(terminal.status, RuntimeTurnStatus::Completed);
8193	Ok(())
8194	}
8195
8196	#[tokio::test]
8197	async fn steer_rejects_a_terminal_durable_turn_without_dispatch_or_item() -> Result<()> {
8198	let manager = test_manager(test_runtime_dir())?;
8199	let thread = manager
8200	.create_thread(CreateThreadRequest::default())
8201	.await?;
8202	let harness = install_mock_engine(&manager, &thread.id).await;
8203	let mut rx_op = harness.rx_op;
8204	let mut rx_steer = harness.rx_steer;
8205	let tx_event = harness.tx_event;
8206
8207	let turn = manager
8208	.start_turn(
8209	&thread.id,
8210	StartTurnRequest {
8211	prompt: "initial".to_string(),
8212	..Default::default()
8213	},
8214	)
8215	.await?;
8216	assert!(matches!(rx_op.recv().await, Some(Op::SendMessage { .. })));
8217	let original_item_ids = turn.item_ids.clone();
8218	{
8219	let _turn_mutation = manager.store.turn_mutation.lock();
8220	let mut terminal = manager.store.load_turn(&turn.id)?;
8221	terminal.status = RuntimeTurnStatus::Completed;
8222	terminal.ended_at = Some(Utc::now());
8223	manager.store.save_turn(&terminal)?;
8224	}
8225
8226	let error = manager
8227	.steer_turn(
8228	&thread.id,
8229	&turn.id,
8230	SteerTurnRequest {
8231	prompt: "must be rejected".to_string(),
8232	},
8233	)
8234	.await
8235	.expect_err("terminal turn must reject steering");
8236	assert!(error.to_string().contains("no longer in progress"));
8237	assert!(
8238	tokio::time::timeout(Duration::from_millis(100), rx_steer.recv())
8239	.await
8240	.is_err(),
8241	"rejected terminal steer must not reach the engine"
8242	);
8243	let persisted = manager.store.load_turn(&turn.id)?;
8244	assert_eq!(persisted.steer_count, 0);
8245	assert_eq!(persisted.item_ids, original_item_ids);
8246	assert_eq!(manager.store.list_items_for_turn(&turn.id)?.len(), 1);
8247
8248	// Restore the synthetic record and let the real monitor settle normally.
8249	{
8250	let _turn_mutation = manager.store.turn_mutation.lock();
8251	let mut active = manager.store.load_turn(&turn.id)?;
8252	active.status = RuntimeTurnStatus::InProgress;
8253	active.ended_at = None;
8254	manager.store.save_turn(&active)?;
8255	}
8256	tx_event
8257	.send(EngineEvent::MessageStarted { index: 0 })
8258	.await?;
8259	tx_event
8260	.send(EngineEvent::MessageDelta {
8261	index: 0,
8262	content: "terminal rejection test completed".to_string(),
8263	})
8264	.await?;
8265	tx_event
8266	.send(EngineEvent::MessageComplete { index: 0 })
8267	.await?;
8268	tx_event
8269	.send(EngineEvent::TurnComplete {
8270	usage: Usage::default(),
8271	status: TurnOutcomeStatus::Completed,
8272	error: None,
8273	tool_catalog: None,
8274	base_url: None,
8275	})
8276	.await?;
8277	let terminal = wait_for_terminal_turn(&manager, &turn.id, Duration::from_secs(2)).await?;
8278	assert_eq!(terminal.status, RuntimeTurnStatus::Completed);
8279	Ok(())
8280	}
8281
8282	#[tokio::test]
8283	async fn concurrent_event_publication_keeps_live_and_durable_sequence_order() -> Result<()> {
8284	let manager = test_manager(test_runtime_dir())?;
8285	let thread = manager
8286	.create_thread(CreateThreadRequest::default())
8287	.await?;
8288	let mut live_rx = manager.subscribe_events();
8289
8290	let mut emitters = Vec::new();
8291	for index in 0..24_u64 {
8292	let emitter = manager.clone();
8293	let thread_id = thread.id.clone();
8294	emitters.push(tokio::spawn(async move {
8295	emitter
8296	.emit_event(
8297	&thread_id,
8298	None,
8299	None,
8300	"test.concurrent",
8301	json!({ "index": index }),
8302	)
8303	.await
8304	}));
8305	}
8306	for emitter in emitters {
8307	emitter.await??;
8308	}
8309
8310	let mut live = Vec::new();
8311	for _ in 0..24 {
8312	live.push(tokio::time::timeout(Duration::from_secs(2), live_rx.recv()).await??);
8313	}
8314	assert!(live.windows(2).all(\|pair\| pair[0].seq < pair[1].seq));
8315
8316	let durable: Vec<_> = manager
8317	.events_since(&thread.id, None)?
8318	.into_iter()
8319	.filter(\|event\| event.event == "test.concurrent")
8320	.collect();
8321	assert_eq!(durable.len(), 24);
8322	assert_eq!(
8323	live.iter()
8324	.map(\|event\| (event.seq, event.payload.clone()))
8325	.collect::<Vec<_>>(),
8326	durable
8327	.iter()
8328	.map(\|event\| (event.seq, event.payload.clone()))
8329	.collect::<Vec<_>>(),
8330	"broadcast order must exactly match append order"
8331	);
8332	Ok(())
8333	}
8334
8335	#[tokio::test]
8336	async fn closed_engine_event_stream_fails_turn_items_and_evicts_engine() -> Result<()> {
8337	let manager = test_manager(test_runtime_dir())?;
8338	let thread = manager
8339	.create_thread(CreateThreadRequest::default())
8340	.await?;
8341	let harness = install_mock_engine(&manager, &thread.id).await;
8342	let mut rx_op = harness.rx_op;
8343	let tx_event = harness.tx_event;
8344
8345	let turn = manager
8346	.start_turn(
8347	&thread.id,
8348	StartTurnRequest {
8349	prompt: "engine stream will close".to_string(),
8350	..Default::default()
8351	},
8352	)
8353	.await?;
8354	assert!(matches!(rx_op.recv().await, Some(Op::SendMessage { .. })));
8355	drop(tx_event);
8356
8357	let terminal = wait_for_terminal_turn(&manager, &turn.id, Duration::from_secs(2)).await?;
8358	assert_eq!(terminal.status, RuntimeTurnStatus::Failed);
8359	let terminal_error = terminal.error.as_deref().unwrap_or_default();
8360	assert!(
8361	terminal.error.as_deref().is_some_and(\|error\| {
8362	error.contains("Failed to monitor") \|\| error.contains("without producing any output")
8363	}),
8364	"unexpected terminal error: {terminal_error:?}"
8365	);
8366	assert!(
8367	manager
8368	.store
8369	.list_items_for_turn(&turn.id)?
8370	.iter()
8371	.all(\|item\| !matches!(
8372	item.status,
8373	TurnItemLifecycleStatus::Queued \| TurnItemLifecycleStatus::InProgress
8374	))
8375	);
8376	let deadline = Instant::now() + Duration::from_secs(2);
8377	loop {
8378	if !manager.active.lock().await.engines.contains_key(&thread.id) {
8379	break;
8380	}
8381	if Instant::now() >= deadline {
8382	bail!("failed engine was not evicted");
8383	}
8384	tokio::task::yield_now().await;
8385	}
8386	assert!(matches!(rx_op.recv().await, Some(Op::Shutdown)));
8387	Ok(())
8388	}
8389
8390	#[tokio::test]
8391	async fn failed_turn_cancels_pending_user_input_and_clears_snapshot() -> Result<()> {
8392	let manager = test_manager(test_runtime_dir())?;
8393	let thread = manager
8394	.create_thread(CreateThreadRequest::default())
8395	.await?;
8396	let mut harness = install_mock_engine(&manager, &thread.id).await;
8397	let turn = manager
8398	.start_turn(
8399	&thread.id,
8400	StartTurnRequest {
8401	prompt: "input required, then the engine stream closes".to_string(),
8402	..Default::default()
8403	},
8404	)
8405	.await?;
8406	assert!(matches!(
8407	harness.rx_op.recv().await,
8408	Some(Op::SendMessage { .. })
8409	));
8410	harness
8411	.tx_event
8412	.send(EngineEvent::UserInputRequired {
8413	id: "input_failed_turn".to_string(),
8414	request: crate::tools::user_input::UserInputRequest {
8415	questions: vec![crate::tools::user_input::UserInputQuestion {
8416	header: "Continue".to_string(),
8417	id: "continue".to_string(),
8418	question: "Continue?".to_string(),
8419	options: vec![crate::tools::user_input::UserInputOption {
8420	label: "Yes".to_string(),
8421	description: "Continue now".to_string(),
8422	}],
8423	allow_free_text: false,
8424	multi_select: false,
8425	}],
8426	},
8427	})
8428	.await?;
8429
8430	let deadline = Instant::now() + Duration::from_secs(2);
8431	loop {
8432	if !manager
8433	.get_thread_detail(&thread.id)
8434	.await?
8435	.pending_user_inputs
8436	.is_empty()
8437	{
8438	break;
8439	}
8440	if Instant::now() >= deadline {
8441	bail!("pending user input did not reach the canonical snapshot");
8442	}
8443	sleep(Duration::from_millis(20)).await;
8444	}
8445
8446	// Fail the turn mid-prompt: closing the event stream settles the turn as
8447	// failed through the monitor-failure path.
8448	harness.close_event_stream();
8449
8450	let canceled = tokio::time::timeout(
8451	Duration::from_secs(2),
8452	harness.recv_user_input_cancellation(),
8453	)
8454	.await
8455	.expect("failure-path user-input cancellation timed out");
8456	assert_eq!(canceled.as_deref(), Some("input_failed_turn"));
8457
8458	let terminal = wait_for_terminal_turn(&manager, &turn.id, Duration::from_secs(2)).await?;
8459	assert_eq!(terminal.status, RuntimeTurnStatus::Failed);
8460
8461	let deadline = Instant::now() + Duration::from_secs(2);
8462	loop {
8463	let detail = manager.get_thread_detail(&thread.id).await?;
8464	if detail.pending_user_inputs.is_empty()
8465	&& manager.events_since(&thread.id, None)?.iter().any(\|event\| {
8466	event.event == "user_input.canceled"
8467	&& event.turn_id.as_deref() == Some(turn.id.as_str())
8468	&& event.payload.get("input_id").and_then(Value::as_str)
8469	== Some("input_failed_turn")
8470	&& event.payload.get("terminal").and_then(Value::as_bool) == Some(true)
8471	})
8472	{
8473	break;
8474	}
8475	if Instant::now() >= deadline {
8476	bail!("failed turn left a stale pending user input in the snapshot");
8477	}
8478	sleep(Duration::from_millis(20)).await;
8479	}
8480	Ok(())
8481	}
8482
8483	#[tokio::test]
8484	async fn compaction_lifecycle_emits_item_events_with_compaction_counts() -> Result<()> {
8485	let manager = test_manager(test_runtime_dir())?;
8486	let thread = manager
8487	.create_thread(CreateThreadRequest {
8488	model: None,
8489	workspace: None,
8490	mode: None,
8491	allow_shell: None,
8492	trust_mode: None,
8493	auto_approve: None,
8494	archived: false,
8495	system_prompt: None,
8496	task_id: None,
8497	..Default::default()
8498	})
8499	.await?;
8500
8501	let harness = install_mock_engine(&manager, &thread.id).await;
8502	let mut rx_op = harness.rx_op;
8503	let tx_event = harness.tx_event;
8504	tokio::spawn(async move {
8505	let mut op_count = 0usize;
8506	while let Some(op) = rx_op.recv().await {
8507	match op {
8508	Op::SendMessage { .. } => {
8509	op_count = op_count.saturating_add(1);
8510	let _ = tx_event
8511	.send(EngineEvent::TurnStarted {
8512	turn_id: "engine_turn_auto".to_string(),
8513	created_at: chrono::Utc::now(),
8514	route: None,
8515	})
8516	.await;
8517	let _ = tx_event
8518	.send(EngineEvent::CompactionStarted {
8519	id: "auto_compact_1".to_string(),
8520	auto: true,
8521	message: "auto compact begin".to_string(),
8522	})
8523	.await;
8524	let _ = tx_event
8525	.send(EngineEvent::CompactionCompleted {
8526	id: "auto_compact_1".to_string(),
8527	auto: true,
8528	message: "auto compact done".to_string(),
8529	messages_before: Some(7),
8530	messages_after: Some(3),
8531	summary_prompt: None,
8532	})
8533	.await;
8534	let _ = tx_event
8535	.send(EngineEvent::TurnComplete {
8536	usage: Usage {
8537	input_tokens: 3,
8538	output_tokens: 3,
8539	..Usage::default()
8540	},
8541	status: TurnOutcomeStatus::Completed,
8542	error: None,
8543	tool_catalog: None,
8544	base_url: None,
8545	})
8546	.await;
8547	}
8548	Op::CompactContext { .. } => {
8549	op_count = op_count.saturating_add(1);
8550	let _ = tx_event
8551	.send(EngineEvent::CompactionStarted {
8552	id: "manual_compact_1".to_string(),
8553	auto: false,
8554	message: "manual compact begin".to_string(),
8555	})
8556	.await;
8557	let _ = tx_event
8558	.send(EngineEvent::CompactionCompleted {
8559	id: "manual_compact_1".to_string(),
8560	auto: false,
8561	message: "manual compact done".to_string(),
8562	messages_before: Some(5),
8563	messages_after: Some(2),
8564	summary_prompt: Some(
8565	"## 📋 Conversation Summary (Auto-Generated)\n\nkey facts."
8566	.to_string(),
8567	),
8568	})
8569	.await;
8570	let _ = tx_event
8571	.send(EngineEvent::TurnComplete {
8572	usage: Usage {
8573	input_tokens: 1,
8574	output_tokens: 1,
8575	..Usage::default()
8576	},
8577	status: TurnOutcomeStatus::Completed,
8578	error: None,
8579	tool_catalog: None,
8580	base_url: None,
8581	})
8582	.await;
8583	}
8584	_ => {}
8585	}
8586	if op_count >= 2 {
8587	break;
8588	}
8589	}
8590	});
8591
8592	let auto_turn = manager
8593	.start_turn(
8594	&thread.id,
8595	StartTurnRequest {
8596	prompt: "trigger auto".to_string(),
8597	input_summary: None,
8598	model: None,
8599	mode: None,
8600	allow_shell: None,
8601	trust_mode: None,
8602	auto_approve: None,
8603	..Default::default()
8604	},
8605	)
8606	.await?;
8607	let auto_turn = wait_for_terminal_turn(&manager, &auto_turn.id, Duration::from_secs(2)).await?;
8608	assert_eq!(auto_turn.status, RuntimeTurnStatus::Completed);
8609
8610	let manual_turn = manager
8611	.compact_thread(
8612	&thread.id,
8613	CompactThreadRequest {
8614	reason: Some("manual request".to_string()),
8615	},
8616	)
8617	.await?;
8618	let manual_turn =
8619	wait_for_terminal_turn(&manager, &manual_turn.id, Duration::from_secs(2)).await?;
8620	assert_eq!(manual_turn.status, RuntimeTurnStatus::Completed);
8621
8622	let events = manager.events_since(&thread.id, None)?;
8623	assert!(events.iter().any(\|ev\| {
8624	ev.event == "item.started"
8625	&& ev
8626	.payload
8627	.get("item")
8628	.and_then(\|item\| item.get("kind"))
8629	.and_then(Value::as_str)
8630	== Some("context_compaction")
8631	&& ev.payload.get("auto").and_then(Value::as_bool) == Some(true)
8632	}));
8633	assert!(events.iter().any(\|ev\| {
8634	ev.event == "item.completed"
8635	&& ev
8636	.payload
8637	.get("item")
8638	.and_then(\|item\| item.get("kind"))
8639	.and_then(Value::as_str)
8640	== Some("context_compaction")
8641	&& ev.payload.get("auto").and_then(Value::as_bool) == Some(true)
8642	&& ev.payload.get("messages_before").and_then(Value::as_u64) == Some(7)
8643	&& ev.payload.get("messages_after").and_then(Value::as_u64) == Some(3)
8644	}));
8645	assert!(events.iter().any(\|ev\| {
8646	ev.event == "item.completed"
8647	&& ev
8648	.payload
8649	.get("item")
8650	.and_then(\|item\| item.get("kind"))
8651	.and_then(Value::as_str)
8652	== Some("context_compaction")
8653	&& ev.payload.get("auto").and_then(Value::as_bool) == Some(false)
8654	&& ev.payload.get("messages_before").and_then(Value::as_u64) == Some(5)
8655	&& ev.payload.get("messages_after").and_then(Value::as_u64) == Some(2)
8656	}));
8657
8658	// The manual compact carried a summary_prompt → it must be persisted into
8659	// the thread record so engine reloads restore it. The auto compact carried
8660	// None → exactly one summary section, from the manual pass.
8661	let record = manager.get_thread(&thread.id).await?;
8662	let record_prompt = record.system_prompt.expect("record keeps a system prompt");
8663	assert!(record_prompt.contains(COMPACTION_SUMMARY_BEGIN));
8664	assert!(record_prompt.contains("Conversation Summary (Auto-Generated)"));
8665	assert!(record_prompt.contains("key facts."));
8666	assert_eq!(record_prompt.matches(COMPACTION_SUMMARY_BEGIN).count(), 1);
8667	Ok(())
8668	}
8669
8670	#[test]
8671	fn summarize_text_truncates() {
8672	let out = summarize_text("abcdefghijklmnopqrstuvwxyz", 10);
8673	assert_eq!(out, "abcdefg...");
8674	}
8675
8676	#[test]
8677	fn approval_decision_requires_auto_approve_and_trust_for_full_access() {
8678	assert_eq!(
8679	RuntimeThreadManager::approval_decision(false, false, false),
8680	RuntimeApprovalDecision::DenyTool
8681	);
8682	assert_eq!(
8683	RuntimeThreadManager::approval_decision(false, true, false),
8684	RuntimeApprovalDecision::DenyTool
8685	);
8686	assert_eq!(
8687	RuntimeThreadManager::approval_decision(true, false, false),
8688	RuntimeApprovalDecision::ApproveTool
8689	);
8690	assert_eq!(
8691	RuntimeThreadManager::approval_decision(true, false, true),
8692	RuntimeApprovalDecision::DenyTool
8693	);
8694	assert_eq!(
8695	RuntimeThreadManager::approval_decision(true, true, true),
8696	RuntimeApprovalDecision::RetryWithFullAccess
8697	);
8698	}
8699
8700	#[test]
8701	fn opening_manager_recovers_stale_queued_and_in_progress_work() -> Result<()> {
8702	let data_dir = test_runtime_dir();
8703	let manager = test_manager(data_dir.clone())?;
8704	let started_at = Utc::now() - chrono::Duration::seconds(5);
8705	let created_at = started_at - chrono::Duration::seconds(1);
8706
8707	let thread = ThreadRecord {
8708	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
8709	id: "thr_restart".to_string(),
8710	created_at,
8711	updated_at: created_at,
8712	model: DEFAULT_TEXT_MODEL.to_string(),
8713	model_provider: None,
8714	model_provider_id: None,
8715	workspace: PathBuf::from("."),
8716	mode: "agent".to_string(),
8717	permission_posture: None,
8718	allow_shell: false,
8719	trust_mode: false,
8720	auto_approve: false,
8721	latest_turn_id: Some("turn_in_progress".to_string()),
8722	latest_response_bookmark: None,
8723	archived: false,
8724	system_prompt: None,
8725	task_id: None,
8726	title: None,
8727	session_id: None,
8728	};
8729	manager.store.save_thread(&thread)?;
8730
8731	let completed_item = TurnItemRecord {
8732	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
8733	id: "item_completed".to_string(),
8734	turn_id: "turn_in_progress".to_string(),
8735	kind: TurnItemKind::Status,
8736	status: TurnItemLifecycleStatus::Completed,
8737	summary: "done".to_string(),
8738	detail: None,
8739	metadata: None,
8740	artifact_refs: Vec::new(),
8741	started_at: Some(started_at),
8742	ended_at: Some(started_at + chrono::Duration::seconds(1)),
8743	};
8744	let in_progress_item = TurnItemRecord {
8745	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
8746	id: "item_in_progress".to_string(),
8747	turn_id: "turn_in_progress".to_string(),
8748	kind: TurnItemKind::ToolCall,
8749	status: TurnItemLifecycleStatus::InProgress,
8750	summary: "running".to_string(),
8751	detail: None,
8752	metadata: None,
8753	artifact_refs: Vec::new(),
8754	started_at: Some(started_at),
8755	ended_at: None,
8756	};
8757	let queued_item = TurnItemRecord {
8758	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
8759	id: "item_queued".to_string(),
8760	turn_id: "turn_queued".to_string(),
8761	kind: TurnItemKind::ToolCall,
8762	status: TurnItemLifecycleStatus::Queued,
8763	summary: "queued".to_string(),
8764	detail: None,
8765	metadata: None,
8766	artifact_refs: Vec::new(),
8767	started_at: None,
8768	ended_at: None,
8769	};
8770	manager.store.save_item(&completed_item)?;
8771	manager.store.save_item(&in_progress_item)?;
8772	manager.store.save_item(&queued_item)?;
8773
8774	manager.store.save_turn(&TurnRecord {
8775	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
8776	id: "turn_in_progress".to_string(),
8777	thread_id: thread.id.clone(),
8778	status: RuntimeTurnStatus::InProgress,
8779	input_summary: "hello".to_string(),
8780	created_at,
8781	started_at: Some(started_at),
8782	ended_at: None,
8783	duration_ms: None,
8784	usage: None,
8785	permission_posture: None,
8786	effective_provider: None,
8787	effective_provider_id: None,
8788	effective_billing_surface: None,
8789	effective_endpoint_fingerprint: None,
8790	effective_billing_mode: None,
8791	effective_dispatched_at: None,
8792	effective_model: None,
8793	routed_usage: Vec::new(),
8794	routed_usage_source_ids: Vec::new(),
8795	routed_usage_dropped_records: 0,
8796	error: None,
8797	item_ids: vec![completed_item.id.clone(), in_progress_item.id.clone()],
8798	steer_count: 0,
8799	})?;
8800	manager.store.save_turn(&TurnRecord {
8801	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
8802	id: "turn_queued".to_string(),
8803	thread_id: thread.id.clone(),
8804	status: RuntimeTurnStatus::Queued,
8805	input_summary: "later".to_string(),
8806	created_at,
8807	started_at: None,
8808	ended_at: None,
8809	duration_ms: None,
8810	usage: None,
8811	permission_posture: None,
8812	effective_provider: None,
8813	effective_provider_id: None,
8814	effective_billing_surface: None,
8815	effective_endpoint_fingerprint: None,
8816	effective_billing_mode: None,
8817	effective_dispatched_at: None,
8818	effective_model: None,
8819	routed_usage: Vec::new(),
8820	routed_usage_source_ids: Vec::new(),
8821	routed_usage_dropped_records: 0,
8822	error: None,
8823	item_ids: vec![queued_item.id.clone()],
8824	steer_count: 0,
8825	})?;
8826	drop(manager);
8827
8828	let recovered = test_manager(data_dir)?;
8829
8830	let recovered_thread = recovered.store.load_thread(&thread.id)?;
8831	assert!(recovered_thread.updated_at >= thread.updated_at);
8832
8833	let recovered_in_progress_turn = recovered.store.load_turn("turn_in_progress")?;
8834	assert_eq!(
8835	recovered_in_progress_turn.status,
8836	RuntimeTurnStatus::Interrupted
8837	);
8838	assert_eq!(
8839	recovered_in_progress_turn.error.as_deref(),
8840	Some(RUNTIME_RESTART_REASON)
8841	);
8842	assert!(recovered_in_progress_turn.ended_at.is_some());
8843	assert!(
8844	recovered_in_progress_turn
8845	.duration_ms
8846	.is_some_and(\|duration\| duration >= 5_000)
8847	);
8848
8849	let recovered_queued_turn = recovered.store.load_turn("turn_queued")?;
8850	assert_eq!(recovered_queued_turn.status, RuntimeTurnStatus::Interrupted);
8851	assert_eq!(
8852	recovered_queued_turn.error.as_deref(),
8853	Some(RUNTIME_RESTART_REASON)
8854	);
8855	assert!(recovered_queued_turn.ended_at.is_some());
8856	assert_eq!(recovered_queued_turn.duration_ms, None);
8857
8858	assert_eq!(
8859	recovered.store.load_item(&completed_item.id)?.status,
8860	TurnItemLifecycleStatus::Completed
8861	);
8862	let recovered_in_progress_item = recovered.store.load_item(&in_progress_item.id)?;
8863	assert_eq!(
8864	recovered_in_progress_item.status,
8865	TurnItemLifecycleStatus::Interrupted
8866	);
8867	assert!(recovered_in_progress_item.ended_at.is_some());
8868
8869	let recovered_queued_item = recovered.store.load_item(&queued_item.id)?;
8870	assert_eq!(
8871	recovered_queued_item.status,
8872	TurnItemLifecycleStatus::Interrupted
8873	);
8874	assert!(recovered_queued_item.ended_at.is_some());
8875
8876	Ok(())
8877	}
8878
8879	#[test]
8880	fn parse_mode_defaults_to_agent() {
8881	assert_eq!(parse_mode("unknown"), AppMode::Agent);
8882	assert_eq!(parse_mode("plan"), AppMode::Plan);
8883	}
8884
8885	#[test]
8886	fn mode_only_override_preserves_legacy_full_access_posture() -> Result<()> {
8887	let mut thread = sample_thread("thr_legacy_full_access");
8888	thread.permission_posture = None;
8889	thread.auto_approve = true;
8890
8891	let policy = runtime_policy_with_overrides(&thread, Some("plan"), None, None)?;
8892	assert_eq!(policy.mode, AppMode::Plan);
8893	assert_eq!(policy.permission_wire(), "full_access");
8894
8895	let ask = runtime_policy_with_overrides(&thread, Some("act"), None, Some(false))?;
8896	assert_eq!(ask.permission_wire(), "ask");
8897	Ok(())
8898	}
8899
8900	#[test]
8901	fn parse_mode_opt_resolves_explicit_tokens_and_aliases() {
8902	assert_eq!(parse_mode_opt("agent"), Some(AppMode::Agent));
8903	assert_eq!(parse_mode_opt("1"), Some(AppMode::Agent));
8904	assert_eq!(parse_mode_opt("plan"), Some(AppMode::Plan));
8905	assert_eq!(parse_mode_opt("2"), Some(AppMode::Plan));
8906	assert_eq!(parse_mode_opt("auto"), Some(AppMode::Agent));
8907	assert_eq!(parse_mode_opt("operate"), Some(AppMode::Operate));
8908	assert_eq!(parse_mode_opt("3"), Some(AppMode::Operate));
8909	assert_eq!(parse_mode_opt("yolo"), Some(AppMode::Yolo));
8910	assert_eq!(parse_mode_opt("4"), Some(AppMode::Yolo));
8911	assert_eq!(parse_mode_opt(" PLAN "), Some(AppMode::Plan));
8912	}
8913
8914	#[test]
8915	fn parse_mode_opt_rejects_prompt_fragments() {
8916	for input in [
8917	"plan a trip to Tokyo",
8918	"switch the agent on",
8919	"enter yolo mode",
8920	"agent of chaos",
8921	"mode",
8922	] {
8923	assert_eq!(parse_mode_opt(input), None);
8924	}
8925	}
8926
8927	#[test]
8928	fn parse_mode_wrapper_defaults_and_resolves_numeric_aliases() {
8929	assert_eq!(parse_mode("plan a trip to Tokyo"), AppMode::Agent);
8930	assert_eq!(parse_mode("auto"), AppMode::Agent);
8931	assert_eq!(parse_mode("1"), AppMode::Agent);
8932	assert_eq!(parse_mode("2"), AppMode::Plan);
8933	assert_eq!(parse_mode("3"), AppMode::Operate);
8934	assert_eq!(parse_mode("4"), AppMode::Yolo);
8935	}
8936
8937	fn rebind_event(event: &str, agent_id: &str, seq: u64) -> RuntimeEventRecord {
8938	RuntimeEventRecord {
8939	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
8940	seq,
8941	timestamp: Utc::now(),
8942	thread_id: "thr_test".to_string(),
8943	turn_id: Some("turn_test".to_string()),
8944	item_id: None,
8945	event: event.to_string(),
8946	payload: json!({ "agent_id": agent_id }),
8947	}
8948	}
8949
8950	#[test]
8951	fn collect_agent_rebind_hints_resumes_a_mid_fanout_session() {
8952	// Mirror what runtime_threads persists during a real fanout: three
8953	// workers spawned, two finished, one still running when the session
8954	// was killed. The TUI re-attach must rebuild placeholders for the
8955	// running worker AND the two completed workers (the fanout card
8956	// tracks all of them so the dot-grid stays accurate post-resume).
8957	let events = vec![
8958	rebind_event("agent.spawned", "agent_a", 1),
8959	rebind_event("agent.spawned", "agent_b", 2),
8960	rebind_event("agent.spawned", "agent_c", 3),
8961	rebind_event("agent.progress", "agent_a", 4),
8962	rebind_event("agent.completed", "agent_a", 5),
8963	rebind_event("agent.progress", "agent_b", 6),
8964	rebind_event("agent.completed", "agent_b", 7),
8965	rebind_event("agent.progress", "agent_c", 8),
8966	];
8967	let hints = collect_agent_rebind_hints(&events);
8968	assert_eq!(hints.len(), 3, "every fanout worker must be rebound");
8969	let by_id: std::collections::BTreeMap<&str, AgentRebindStatus> = hints
8970	.iter()
8971	.map(\|h\| (h.agent_id.as_str(), h.status))
8972	.collect();
8973	assert_eq!(by_id.get("agent_a"), Some(&AgentRebindStatus::Completed));
8974	assert_eq!(by_id.get("agent_b"), Some(&AgentRebindStatus::Completed));
8975	assert_eq!(
8976	by_id.get("agent_c"),
8977	Some(&AgentRebindStatus::InProgress),
8978	"in-flight worker must rebind in InProgress, not downgrade"
8979	);
8980	}
8981
8982	#[test]
8983	fn collect_agent_rebind_hints_ignores_unrelated_events() {
8984	// Status / tool events should not produce phantom hints — only the
8985	// agent.* family carries the contract we re-bind from.
8986	let events = vec![
8987	RuntimeEventRecord {
8988	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
8989	seq: 1,
8990	timestamp: Utc::now(),
8991	thread_id: "thr".to_string(),
8992	turn_id: None,
8993	item_id: None,
8994	event: "tool.completed".to_string(),
8995	payload: json!({"name": "read_file"}),
8996	},
8997	rebind_event("agent.spawned", "agent_x", 2),
8998	RuntimeEventRecord {
8999	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
9000	seq: 3,
9001	timestamp: Utc::now(),
9002	thread_id: "thr".to_string(),
9003	turn_id: None,
9004	item_id: None,
9005	event: "compaction.completed".to_string(),
9006	payload: json!({"messages_after": 12}),
9007	},
9008	];
9009	let hints = collect_agent_rebind_hints(&events);
9010	assert_eq!(hints.len(), 1);
9011	assert_eq!(hints[0].agent_id, "agent_x");
9012	}
9013
9014	#[test]
9015	fn collect_agent_rebind_hints_does_not_downgrade_completed_to_in_progress() {
9016	// Out-of-order replay: a stale `agent.progress` arriving after the
9017	// completed event must NOT clobber the terminal status. This matters
9018	// when an event log is concatenated from interrupted segments.
9019	let events = vec![
9020	rebind_event("agent.spawned", "agent_y", 1),
9021	rebind_event("agent.completed", "agent_y", 2),
9022	rebind_event("agent.progress", "agent_y", 3),
9023	];
9024	let hints = collect_agent_rebind_hints(&events);
9025	assert_eq!(hints.len(), 1);
9026	assert_eq!(hints[0].status, AgentRebindStatus::Completed);
9027	}
9028
9029	/// Helper for the `fork_at_user_message` tests: write a sequence of
9030	/// (user, assistant) turns under the given thread id. Each turn gets
9031	/// one UserMessage item carrying `user_text` in `detail` plus one
9032	/// AgentMessage item. Turn `created_at` is monotonically increasing
9033	/// so the chronological sort in `list_turns_for_thread` is stable.
9034	fn seed_turns_with_user_messages(
9035	manager: &RuntimeThreadManager,
9036	thread_id: &str,
9037	user_texts: &[&str],
9038	) -> Result<Vec<String>> {
9039	let mut turn_ids = Vec::new();
9040	let base = Utc::now();
9041	for (offset, text) in user_texts.iter().enumerate() {
9042	let created_at = base + chrono::Duration::milliseconds(offset as i64);
9043	let turn_id = format!("turn_test_{offset}");
9044	let user_item_id = format!("item_user_{offset}");
9045	let asst_item_id = format!("item_asst_{offset}");
9046	manager.store.save_item(&TurnItemRecord {
9047	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
9048	id: user_item_id.clone(),
9049	turn_id: turn_id.clone(),
9050	kind: TurnItemKind::UserMessage,
9051	status: TurnItemLifecycleStatus::Completed,
9052	summary: (*text).to_string(),
9053	detail: Some((*text).to_string()),
9054	metadata: None,
9055	artifact_refs: Vec::new(),
9056	started_at: Some(created_at),
9057	ended_at: Some(created_at),
9058	})?;
9059	manager.store.save_item(&TurnItemRecord {
9060	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
9061	id: asst_item_id.clone(),
9062	turn_id: turn_id.clone(),
9063	kind: TurnItemKind::AgentMessage,
9064	status: TurnItemLifecycleStatus::Completed,
9065	summary: format!("reply {offset}"),
9066	detail: Some(format!("reply {offset}")),
9067	metadata: None,
9068	artifact_refs: Vec::new(),
9069	started_at: Some(created_at),
9070	ended_at: Some(created_at),
9071	})?;
9072	manager.store.save_turn(&TurnRecord {
9073	schema_version: CURRENT_RUNTIME_SCHEMA_VERSION,
9074	id: turn_id.clone(),
9075	thread_id: thread_id.to_string(),
9076	status: RuntimeTurnStatus::Completed,
9077	input_summary: (*text).to_string(),
9078	created_at,
9079	started_at: Some(created_at),
9080	ended_at: Some(created_at),
9081	duration_ms: Some(0),
9082	usage: None,
9083	permission_posture: None,
9084	effective_provider: None,
9085	effective_provider_id: None,
9086	effective_billing_surface: None,
9087	effective_endpoint_fingerprint: None,
9088	effective_billing_mode: None,
9089	effective_dispatched_at: None,
9090	effective_model: None,
9091	routed_usage: Vec::new(),
9092	routed_usage_source_ids: Vec::new(),
9093	routed_usage_dropped_records: 0,
9094	error: None,
9095	item_ids: vec![user_item_id, asst_item_id],
9096	steer_count: 0,
9097	})?;
9098	turn_ids.push(turn_id);
9099	}
9100	Ok(turn_ids)
9101	}
9102
9103	#[tokio::test]
9104	async fn fork_at_user_message_drops_tail_and_returns_user_text() -> Result<()> {
9105	// Seed three completed user/assistant turns. Backtracking with
9106	// depth=0 should drop only the most recent turn ("third") and
9107	// hand back its original text so the caller can refill the
9108	// composer.
9109	let manager = test_manager(test_runtime_dir())?;
9110	let thread = manager
9111	.create_thread(CreateThreadRequest {
9112	model: None,
9113	workspace: None,
9114	mode: None,
9115	allow_shell: None,
9116	trust_mode: None,
9117	auto_approve: None,
9118	archived: false,
9119	system_prompt: None,
9120	task_id: None,
9121	..Default::default()
9122	})
9123	.await?;
9124	seed_turns_with_user_messages(&manager, &thread.id, &["first", "second", "third"])?;
9125
9126	let (forked, original_text) = manager.fork_at_user_message(&thread.id, 0).await?;
9127	assert_eq!(original_text.as_deref(), Some("third"));
9128	assert_ne!(forked.id, thread.id);
9129
9130	let forked_turns = manager.store.list_turns_for_thread(&forked.id)?;
9131	assert_eq!(
9132	forked_turns.len(),
9133	2,
9134	"depth=0 should drop the most recent turn"
9135	);
9136	let summaries: Vec<&str> = forked_turns
9137	.iter()
9138	.map(\|t\| t.input_summary.as_str())
9139	.collect();
9140	assert_eq!(summaries, vec!["first", "second"]);
9141	Ok(())
9142	}
9143
9144	#[tokio::test]
9145	async fn fork_at_user_message_depth_one_drops_two_turns() -> Result<()> {
9146	let manager = test_manager(test_runtime_dir())?;
9147	let thread = manager
9148	.create_thread(CreateThreadRequest {
9149	model: None,
9150	workspace: None,
9151	mode: None,
9152	allow_shell: None,
9153	trust_mode: None,
9154	auto_approve: None,
9155	archived: false,
9156	system_prompt: None,
9157	task_id: None,
9158	..Default::default()
9159	})
9160	.await?;
9161	seed_turns_with_user_messages(&manager, &thread.id, &["a", "b", "c", "d"])?;
9162
9163	let (forked, original_text) = manager.fork_at_user_message(&thread.id, 1).await?;
9164	assert_eq!(original_text.as_deref(), Some("c"));
9165	let forked_turns = manager.store.list_turns_for_thread(&forked.id)?;
9166	let summaries: Vec<&str> = forked_turns
9167	.iter()
9168	.map(\|t\| t.input_summary.as_str())
9169	.collect();
9170	assert_eq!(summaries, vec!["a", "b"]);
9171	Ok(())
9172	}
9173
9174	#[tokio::test]
9175	async fn fork_at_user_message_out_of_range_errors() -> Result<()> {
9176	let manager = test_manager(test_runtime_dir())?;
9177	let thread = manager
9178	.create_thread(CreateThreadRequest {
9179	model: None,
9180	workspace: None,
9181	mode: None,
9182	allow_shell: None,
9183	trust_mode: None,
9184	auto_approve: None,
9185	archived: false,
9186	system_prompt: None,
9187	task_id: None,
9188	..Default::default()
9189	})
9190	.await?;
9191	seed_turns_with_user_messages(&manager, &thread.id, &["only"])?;
9192
9193	let err = manager.fork_at_user_message(&thread.id, 5).await.err();
9194	assert!(err.is_some(), "depth past the end should bail out");
9195	Ok(())
9196	}
9197
9198	#[tokio::test]
9199	async fn fork_at_user_message_does_not_mutate_source() -> Result<()> {
9200	// The source thread must be untouched: turns still present, items
9201	// still present, latest_turn_id still pointing at the original
9202	// tail. Backtrack creates a sibling, never edits in place.
9203	let manager = test_manager(test_runtime_dir())?;
9204	let thread = manager
9205	.create_thread(CreateThreadRequest {
9206	model: None,
9207	workspace: None,
9208	mode: None,
9209	allow_shell: None,
9210	trust_mode: None,
9211	auto_approve: None,
9212	archived: false,
9213	system_prompt: None,
9214	task_id: None,
9215	..Default::default()
9216	})
9217	.await?;
9218	let turn_ids = seed_turns_with_user_messages(&manager, &thread.id, &["x", "y", "z"])?;
9219
9220	let _ = manager.fork_at_user_message(&thread.id, 0).await?;
9221
9222	let source_turns = manager.store.list_turns_for_thread(&thread.id)?;
9223	assert_eq!(
9224	source_turns.len(),
9225	3,
9226	"source thread must still hold every turn after fork"
9227	);
9228	for tid in &turn_ids {
9229	assert!(
9230	manager.store.load_turn(tid).is_ok(),
9231	"turn {tid} must remain on disk"
9232	);
9233	}
9234	Ok(())
9235	}
9236
9237	// ── compaction summary persistence (merge_summary_into_prompt) ──
9238
9239	#[test]
9240	fn summary_merge_appends_section_to_base_prompt() {
9241	let merged = merge_summary_into_prompt(
9242	Some("You are a helpful agent."),
9243	"## 📋 Conversation Summary (Auto-Generated)\n\nUser prefers lists.",
9244	);
9245	assert!(merged.starts_with("You are a helpful agent."));
9246	assert!(merged.contains(COMPACTION_SUMMARY_BEGIN));
9247	assert!(merged.contains("User prefers lists."));
9248	assert!(merged.ends_with(COMPACTION_SUMMARY_END));
9249	// Reload restore keys on the marker: SyncSession maps the record to
9250	// SystemPrompt::Text and extract_compaction_summary_prompt checks
9251	// `contains("Conversation Summary (Auto-Generated)")`.
9252	assert!(merged.contains("Conversation Summary (Auto-Generated)"));
9253	}
9254
9255	#[test]
9256	fn summary_merge_replaces_existing_section_idempotently() {
9257	let first = merge_summary_into_prompt(Some("Base prompt."), "summary v1");
9258	let second = merge_summary_into_prompt(Some(&first), "summary v2");
9259	assert!(second.contains("summary v2"));
9260	assert!(!second.contains("summary v1"));
9261	assert_eq!(
9262	second.matches(COMPACTION_SUMMARY_BEGIN).count(),
9263	1,
9264	"repeated compactions must swap the section, not stack duplicates"
9265	);
9266	assert!(second.starts_with("Base prompt."));
9267	}
9268
9269	#[test]
9270	fn summary_merge_handles_missing_base() {
9271	let merged = merge_summary_into_prompt(None, "only summary");
9272	assert!(merged.starts_with(COMPACTION_SUMMARY_BEGIN));
9273	assert!(merged.contains("only summary"));
9274	let empty_base = merge_summary_into_prompt(Some(""), "only summary");
9275	assert!(empty_base.starts_with(COMPACTION_SUMMARY_BEGIN));
9276	}
9277
9278	#[test]
9279	fn summary_strip_preserves_text_after_section() {
9280	let with_tail = format!(
9281	"Base.\n\n{COMPACTION_SUMMARY_BEGIN}\nold summary\n{COMPACTION_SUMMARY_END}\n\nTrailing rules."
9282	);
9283	let stripped = strip_summary_section(&with_tail);
9284	assert!(stripped.contains("Base."));
9285	assert!(stripped.contains("Trailing rules."));
9286	assert!(!stripped.contains("old summary"));
9287	// Re-merge keeps the tail intact.
9288	let merged = merge_summary_into_prompt(Some(&with_tail), "new summary");
9289	assert!(merged.contains("Trailing rules."));
9290	assert!(merged.contains("new summary"));
9291	}
9292
9293	#[test]
9294	fn summary_strip_handles_missing_end_sentinel() {
9295	let broken = format!("Base.\n\n{COMPACTION_SUMMARY_BEGIN}\ntruncated…");
9296	let stripped = strip_summary_section(&broken);
9297	assert_eq!(stripped, "Base.");
9298	}
9299

9299 lines RUST