返回 CodeWhale
exec_stream_drop_acceptance.rs
根目录 / crates / tui / tests / integration / exec_stream_drop_acceptance.rs
1 //! Process-level acceptance for the v0.9.4 Terminal-Bench P0 exit-path fix.
2 //!
3 //! Benchmark evidence (Terminal-Bench 2.1, codewhale 0.9.4): five tasks were
4 //! forfeited when the DeepSeek stream dropped mid-response ("error decoding
5 //! response body" after partial content). The engine surfaced the warning
6 //! and `codewhale exec` exited 1, and Harbor raised
7 //! `NonZeroAgentExitCodeError`. The fix: headless turns re-issue the request
8 //! after a mid-stream network drop (bounded by MAX_STREAM_RETRIES), and a
9 //! turn that still fails exits `EX_TEMPFAIL` (75) — a retryable
10 //! infrastructure failure the harness can distinguish from a genuine task
11 //! failure (exit 1).
12 //!
13 //! These tests drive the real binary against a raw TCP server that sends a
14 //! partial SSE body and then closes mid-`content-length`, reproducing the
15 //! exact reqwest decode failure from the bench artifacts.
16
17 use std::io::Read;
18 use std::path::{Path, PathBuf};
19 use std::process::{Command, Stdio};
20 use std::sync::atomic::{AtomicUsize, Ordering};
21 use std::sync::{Arc, Mutex};
22 use std::time::Duration;
23
24 use serde_json::{Value, json};
25 use tempfile::TempDir;
26 use wait_timeout::ChildExt;
27
28 #[cfg(all(unix, feature = "long-running-tests"))]
29 #[path = "../support/qa_harness/mod.rs"]
30 mod qa_harness;
31
32 const MODEL: &str = "stream-drop-test";
33 /// The server claims a body far larger than it delivers on a "drop" response,
34 /// so hyper raises `error decoding response body` after the first SSE chunk —
35 /// the exact failure string in the Terminal-Bench crash artifacts.
36 const CLAIMED_DROP_BODY_LEN: usize = 1_048_576;
37
38 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
39 async fn headless_exec_recovers_from_mid_stream_drop() {
40 let workspace = TempDir::new().expect("workspace");
41 let home = TempDir::new().expect("home");
42 let (base_url, chat_posts, _requests, _server) = start_flaky_server(1, Duration::ZERO).await;
43
44 let output = run_exec(workspace.path(), home.path(), &base_url);
45
46 assert!(
47 output.status.success(),
48 "a recovered stream drop must exit 0\nstdout:\n{}\nstderr:\n{}",
49 String::from_utf8_lossy(&output.stdout),
50 String::from_utf8_lossy(&output.stderr)
51 );
52 assert_eq!(
53 chat_posts.load(Ordering::SeqCst),
54 2,
55 "the dropped attempt must be re-issued exactly once"
56 );
57 let stdout = String::from_utf8_lossy(&output.stdout);
58 assert!(
59 stdout.contains("recovered after retry"),
60 "the retried turn's content must stream: {stdout}"
61 );
62 assert!(
63 !stdout.contains(r#""type":"error""#),
64 "a transient drop that the retry recovers must not surface an error event: {stdout}"
65 );
66 let meta = terminal_metadata(&stdout);
67 assert_eq!(
68 meta["meta"]["status"].as_str(),
69 Some("completed"),
70 "recovered run must record a completed terminal receipt: {meta}"
71 );
72 }
73
74 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
75 async fn headless_exec_exits_ex_tempfail_after_drop_budget_exhausted() {
76 let workspace = TempDir::new().expect("workspace");
77 let home = TempDir::new().expect("home");
78 // More drops than the engine can consume: initial attempt +
79 // MAX_STREAM_RETRIES (3) resumes, then the turn must fail.
80 let (base_url, chat_posts, _requests, _server) =
81 start_flaky_server(usize::MAX, Duration::ZERO).await;
82
83 let output = run_exec(workspace.path(), home.path(), &base_url);
84
85 assert_eq!(
86 output.status.code(),
87 Some(75),
88 "retry budget exhausted on a network-class failure must exit EX_TEMPFAIL (75), \
89 not the generic task-failure 1\nstdout:\n{}\nstderr:\n{}",
90 String::from_utf8_lossy(&output.stdout),
91 String::from_utf8_lossy(&output.stderr)
92 );
93 assert_eq!(
94 chat_posts.load(Ordering::SeqCst),
95 4,
96 "initial attempt plus the bounded resume budget (MAX_STREAM_RETRIES = 3)"
97 );
98 let stdout = String::from_utf8_lossy(&output.stdout);
99 let error_events = stdout
100 .lines()
101 .filter(|line| line.contains(r#""type":"error""#))
102 .count();
103 assert_eq!(
104 error_events, 1,
105 "only the final, budget-exhausted attempt may emit an error event: {stdout}"
106 );
107 let error_line = stdout
108 .lines()
109 .find(|line| line.contains(r#""type":"error""#))
110 .expect("terminal error event");
111 assert!(
112 error_line.contains("Provider stream connection dropped"),
113 "the error channel must carry the real failure: {error_line}"
114 );
115 let meta = terminal_metadata(&stdout);
116 assert_eq!(meta["meta"]["status"].as_str(), Some("failed"), "{meta}");
117 assert_eq!(
118 meta["meta"]["error_category"].as_str(),
119 Some("network"),
120 "the terminal receipt must classify the failure as retryable infra: {meta}"
121 );
122 }
123
124 /// #5769's later report: the NEXT manually submitted turn, with no approval,
125 /// must reach the real dispatcher and survive its UI watchdog after a loss.
126 #[cfg(all(unix, feature = "long-running-tests"))]
127 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
128 async fn tui_next_manual_turn_completes_after_exhausted_partial_sse_drop() {
129 use qa_harness::harness::{Harness, make_sealed_workspace};
130
131 const FIRST: &str = "R4_FIRST_MANUAL";
132 const SECOND: &str = "R4_NEXT_MANUAL";
133 // Deliberately cross the current 30-second dispatch watchdog using wall
134 // time. The request is admitted, but its HTTP response is still pending.
135 const HEALTHY_RESPONSE_DELAY: Duration = Duration::from_secs(35);
136 const WAIT: Duration = Duration::from_secs(60);
137 let workspace = make_sealed_workspace().expect("sealed workspace");
138 let (base_url, chat_posts, requests, server) =
139 start_flaky_server(4, HEALTHY_RESPONSE_DELAY).await;
140 let outbox = workspace.home().join("turn-receipts.jsonl");
141 std::fs::write(workspace.home().join(".codewhale/.onboarded"), "").expect("onboarding receipt");
142 let trust = workspace.workspace().join(".deepseek");
143 std::fs::create_dir_all(&trust).expect("trust directory");
144 std::fs::write(trust.join("trusted"), "").expect("workspace trust");
145 let config = format!(
146 r#"provider = "loopback"
147 prompt_suggestion = false
148 allow_shell = false
149 [providers.loopback]
150 kind = "openai-compatible"
151 base_url = {base_url}
152 api_key = "synthetic-loopback-key"
153 model = "{MODEL}"
154 [retry]
155 enabled = false
156 [notifications]
157 method = "off"
158 completion_sound = "off"
159 [lifecycle_outbox]
160 path = {outbox}
161 "#,
162 base_url = json!(base_url),
163 outbox = json!(outbox),
164 );
165 std::fs::write(workspace.home().join(".codewhale/config.toml"), config)
166 .expect("loopback TUI configuration");
167 let mut tui = Harness::builder(Harness::cargo_bin("codewhale-tui"))
168 .cwd(workspace.workspace())
169 .clear_env()
170 .seal_home(workspace.home())
171 .env("CODEWHALE_DISABLE_MODELS_DEV_FETCH", "1")
172 .env("CODEWHALE_NO_UPDATE_CHECK", "1")
173 .env("CODEWHALE_TELEMETRY", "0")
174 .env("NO_ANIMATIONS", "1")
175 .args([
176 "--workspace",
177 workspace.workspace().to_str().expect("workspace UTF-8"),
178 "--no-project-config",
179 "--fresh",
180 ])
181 .size(40, 120)
182 .spawn()
183 .expect("start the real TUI");
184 let pid = tui.pid().expect("TUI PID");
185 tui.wait_for_text("Type a message", WAIT)
186 .expect("ready composer");
187 tui.type_line(FIRST).expect("first manual submission");
188 wait_for_tui_turn_ends(&mut tui, &outbox, 1, WAIT);
189 let first_events = read_tui_outbox(&outbox);
190 let first = first_events
191 .iter()
192 .find(|event| event["event"] == "turn_end")
193 .expect("first terminal UI receipt");
194 assert_eq!(first["kind"], "turn.failed", "{first:#}");
195 assert!(
196 first["payload"]["error"]
197 .as_str()
198 .is_some_and(|error| error.starts_with("Provider stream connection dropped")),
199 "the first UI turn must exhaust an actual partial SSE loss: {first:#}"
200 );
201 // The lifecycle outbox contains a bounded user-facing error, not the
202 // underlying reqwest chain. The server's partial SSE body and the exact
203 // request count prove the real transport loss and exhausted retry budget.
204 assert_eq!(chat_posts.load(Ordering::SeqCst), 4);
205
206 // No restart, continuation, event injection, approval answer, or queue
207 // admission: type into the same running composer after its failed receipt.
208 let submitted = std::time::Instant::now();
209 tui.type_line(SECOND).expect("next manual submission");
210 wait_for_tui_turn_ends(&mut tui, &outbox, 2, WAIT);
211 assert!(submitted.elapsed() >= HEALTHY_RESPONSE_DELAY);
212 tui.wait_for_text("recovered after retry", WAIT)
213 .expect("second answer reaches the actual UI");
214 // Completion is proved by the lifecycle receipt below. The composer
215 // intentionally omits the old transient "turn completed" chrome; prove
216 // that the actual UI is ready and still accepts an unsent edit instead.
217 tui.wait_for(
218 |frame| frame.contains("recovered after retry") && frame.contains("Type a message"),
219 WAIT,
220 )
221 .expect("completed answer and ready composer reach the actual UI");
222 const UNSENT: &str = "R4_UNSENT_RECOVERY_CHECK";
223 tui.paste(UNSENT).expect("edit the recovered composer");
224 tui.wait_for(|frame| frame.row(frame.cursor().0).contains(UNSENT), WAIT)
225 .expect("the same composer remains editable after completion");
226 assert_eq!(tui.pid(), Some(pid));
227 let events = read_tui_outbox(&outbox);
228 let starts = events
229 .iter()
230 .filter(|event| event["event"] == "turn_start")
231 .collect::<Vec<_>>();
232 let ends = events
233 .iter()
234 .filter(|event| event["event"] == "turn_end")
235 .collect::<Vec<_>>();
236 assert_eq!(starts.len(), 2, "two real TUI dispatches: {events:#?}");
237 assert_eq!(ends.len(), 2, "two real TUI completions: {events:#?}");
238 assert_eq!(ends[1]["kind"], "turn.completed");
239 assert!(ends[1]["payload"]["error"].is_null());
240 let session = ends[0]["thread_id"].as_str().expect("TUI session identity");
241 assert!(!session.is_empty());
242 assert!(
243 starts
244 .iter()
245 .chain(&ends)
246 .all(|event| event["thread_id"] == session)
247 );
248 assert!(ends.iter().all(|event| event["turn_id"].is_string()));
249 assert_ne!(ends[0]["turn_id"], ends[1]["turn_id"]);
250 assert!(events.iter().all(|event| {
251 !event["event"].as_str().unwrap().contains("approval")
252 && !event["event"].as_str().unwrap().contains("user_input")
253 }));
254 let requests = requests.lock().unwrap();
255 assert_eq!(requests.len(), 5, "four failed attempts then one new turn");
256 let final_messages = requests.last().unwrap()["messages"].as_array().unwrap();
257 let users = final_messages
258 .iter()
259 .filter(|message| message["role"] == "user")
260 .collect::<Vec<_>>();
261 assert_eq!(users.len(), 2, "no synthetic continuation user turns");
262 assert!(users[0]["content"].to_string().contains(FIRST));
263 assert!(users[1]["content"].to_string().contains(SECOND));
264 assert!(
265 final_messages
266 .iter()
267 .all(|message| message["role"] != "tool")
268 );
269 assert!(requests.iter().all(|request| request["stream"] == true));
270 drop(requests);
271 tui.shutdown();
272 server.abort();
273 }
274
275 #[cfg(all(unix, feature = "long-running-tests"))]
276 fn read_tui_outbox(path: &Path) -> Vec<Value> {
277 let text = match std::fs::read_to_string(path) {
278 Ok(text) => text,
279 Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Vec::new(),
280 Err(error) => panic!("read TUI lifecycle receipt: {error}"),
281 };
282 // A concurrent append can expose an unfinished last line. Only complete
283 // records count as receipts; a malformed complete record is a failure.
284 text.split_inclusive('\n')
285 .filter(|line| line.ends_with('\n'))
286 .map(|line| serde_json::from_str(line).expect("complete TUI outbox record"))
287 .collect()
288 }
289
290 #[cfg(all(unix, feature = "long-running-tests"))]
291 fn wait_for_tui_turn_ends(
292 tui: &mut qa_harness::harness::Harness,
293 outbox: &Path,
294 count: usize,
295 timeout: Duration,
296 ) {
297 tui.wait_for(
298 |frame| {
299 for unexpected in [
300 "Turn dispatch timed out",
301 "engine may have stopped",
302 "Approval required",
303 "engine session id diverged",
304 ] {
305 assert!(
306 !frame.contains(unexpected),
307 "{unexpected}: {}",
308 frame.text()
309 );
310 }
311 read_tui_outbox(outbox)
312 .iter()
313 .filter(|event| event["event"] == "turn_end")
314 .count()
315 >= count
316 },
317 timeout,
318 )
319 .unwrap_or_else(|error| {
320 panic!(
321 "UI did not finish turn {count}: {error}\n{}",
322 tui.diagnostics()
323 )
324 });
325 }
326
327 /// Extract the terminal `metadata` event from the stream-json stdout.
328 fn terminal_metadata(stdout: &str) -> Value {
329 let line = stdout
330 .lines()
331 .rev()
332 .find(|line| line.contains(r#""type":"metadata""#))
333 .unwrap_or_else(|| panic!("stream-json metadata event missing: {stdout}"));
334 serde_json::from_str(line).expect("metadata event is valid JSON")
335 }
336
337 /// Spawn a raw HTTP server that answers `GET /v1/models` and, for every POST
338 /// to the chat endpoint, either truncates the SSE body mid-stream (the first
339 /// `drops` requests) or completes a normal text turn. Returns the base URL
340 /// and a counter of chat-completion POSTs.
341 async fn start_flaky_server(
342 drops: usize,
343 healthy_response_delay: Duration,
344 ) -> (
345 String,
346 Arc<AtomicUsize>,
347 Arc<Mutex<Vec<Value>>>,
348 tokio::task::JoinHandle<()>,
349 ) {
350 let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
351 .await
352 .expect("bind flaky server");
353 let addr = listener.local_addr().expect("flaky server addr");
354 let chat_posts = Arc::new(AtomicUsize::new(0));
355 let server_posts = Arc::clone(&chat_posts);
356 let requests = Arc::new(Mutex::new(Vec::new()));
357 let captured = Arc::clone(&requests);
358 let task = tokio::spawn(async move {
359 loop {
360 let (mut socket, _) = match listener.accept().await {
361 Ok(pair) => pair,
362 Err(_) => break,
363 };
364 let request = match read_http_request(&mut socket).await {
365 Some(request) => request,
366 None => continue,
367 };
368 if request.starts_with("GET ") {
369 write_all(
370 &mut socket,
371 &http_response(
372 "application/json",
373 &json!({"object":"list","data":[{"id":MODEL,"object":"model"}]})
374 .to_string(),
375 ),
376 )
377 .await;
378 continue;
379 }
380 let body = request.split_once("\r\n\r\n").expect("request body").1;
381 captured
382 .lock()
383 .unwrap()
384 .push(serde_json::from_str(body).expect("chat request JSON"));
385 let call = server_posts.fetch_add(1, Ordering::SeqCst) + 1;
386 if call <= drops {
387 // Partial SSE (one real content chunk), then the socket
388 // closes with the declared content-length unmet — the
389 // production "Provider stream connection dropped" failure.
390 let partial = sse_chunk(
391 json!({"id":"drop","object":"chat.completion.chunk","model":MODEL,"choices":[{"index":0,"delta":{"content":"partial answer that must be discarded"},"finish_reason":null}]}),
392 );
393 let head = format!(
394 "HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\ncontent-length: {CLAIMED_DROP_BODY_LEN}\r\nconnection: close\r\n\r\n"
395 );
396 write_all(&mut socket, &format!("{head}{partial}")).await;
397 } else {
398 tokio::time::sleep(healthy_response_delay).await;
399 let body = [
400 sse_chunk(json!({"id":"final","object":"chat.completion.chunk","model":MODEL,"choices":[{"index":0,"delta":{"content":"recovered after retry"},"finish_reason":null}]})),
401 sse_chunk(json!({"id":"final","object":"chat.completion.chunk","model":MODEL,"choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":20,"completion_tokens":2,"total_tokens":22}})),
402 "data: [DONE]\n\n".to_string(),
403 ]
404 .join("");
405 write_all(&mut socket, &http_response("text/event-stream", &body)).await;
406 }
407 }
408 });
409 (format!("http://{addr}/v1"), chat_posts, requests, task)
410 }
411
412 /// Read one HTTP request (headers plus the content-length body, when any).
413 async fn read_http_request(socket: &mut tokio::net::TcpStream) -> Option<String> {
414 use tokio::io::AsyncReadExt;
415 let mut buffer = Vec::new();
416 let mut chunk = [0u8; 4096];
417 let header_end = loop {
418 if let Some(pos) = find_subslice(&buffer, b"\r\n\r\n") {
419 break pos + 4;
420 }
421 let read = socket.read(&mut chunk).await.ok()?;
422 if read == 0 {
423 return None;
424 }
425 buffer.extend_from_slice(&chunk[..read]);
426 if buffer.len() > 1 << 20 {
427 return None;
428 }
429 };
430 let headers = String::from_utf8_lossy(&buffer[..header_end]).to_string();
431 let content_length = headers
432 .lines()
433 .find_map(|line| {
434 line.to_ascii_lowercase()
435 .strip_prefix("content-length:")
436 .and_then(|value| value.trim().parse::<usize>().ok())
437 })
438 .unwrap_or(0);
439 let request_len = header_end.checked_add(content_length)?;
440 if request_len > 1 << 20 {
441 return None;
442 }
443 while buffer.len() < request_len {
444 let read = socket.read(&mut chunk).await.ok()?;
445 if read == 0 {
446 return None;
447 }
448 buffer.extend_from_slice(&chunk[..read]);
449 }
450 buffer.truncate(request_len);
451 String::from_utf8(buffer).ok()
452 }
453
454 fn find_subslice(haystack: &[u8], needle: &[u8]) -> Option<usize> {
455 haystack
456 .windows(needle.len())
457 .position(|window| window == needle)
458 }
459
460 async fn write_all(socket: &mut tokio::net::TcpStream, bytes: &str) {
461 use tokio::io::AsyncWriteExt;
462 let _ = socket.write_all(bytes.as_bytes()).await;
463 let _ = socket.shutdown().await;
464 }
465
466 fn http_response(content_type: &str, body: &str) -> String {
467 format!(
468 "HTTP/1.1 200 OK\r\ncontent-type: {content_type}\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
469 body.len()
470 )
471 }
472
473 fn sse_chunk(value: Value) -> String {
474 format!(
475 "data: {}\n\n",
476 serde_json::to_string(&value).expect("SSE JSON")
477 )
478 }
479
480 fn run_exec(workspace: &Path, home: &Path, base_url: &str) -> std::process::Output {
481 std::fs::create_dir_all(home.join(".codewhale")).expect("config directory");
482 std::fs::create_dir_all(home.join(".deepseek")).expect("legacy config directory");
483 std::fs::write(
484 home.join(".codewhale/config.toml"),
485 "allow_shell = true\n\n[retry]\nenabled = false\n",
486 )
487 .expect("headless test config");
488 let mut command = Command::new(binary());
489 preserve_host_env(&mut command);
490 command
491 .current_dir(workspace)
492 .args(["--workspace", workspace.to_str().expect("workspace utf8")])
493 .arg("--no-project-config")
494 .args([
495 "exec",
496 "--auto",
497 "--model",
498 MODEL,
499 "--output-format",
500 "stream-json",
501 ])
502 .arg("answer briefly")
503 .env("HOME", home)
504 .env("USERPROFILE", home)
505 .env("XDG_CONFIG_HOME", home.join(".config"))
506 .env("XDG_DATA_HOME", home.join(".local/share"))
507 .env("XDG_CACHE_HOME", home.join(".cache"))
508 .env("CODEWHALE_CONFIG_PATH", home.join(".codewhale/config.toml"))
509 .env("DEEPSEEK_CONFIG_PATH", home.join(".deepseek/config.toml"))
510 .env("DEEPSEEK_API_KEY", "ci-test-key-not-real")
511 .env("DEEPSEEK_BASE_URL", base_url)
512 .env("CODEWHALE_BASE_URL", base_url)
513 .env("DEEPSEEK_MODEL", MODEL)
514 .env("CODEWHALE_MODEL", MODEL)
515 .env("RUST_LOG", "warn")
516 .stdout(Stdio::piped())
517 .stderr(Stdio::piped());
518 run_with_timeout(command, Duration::from_secs(45))
519 }
520
521 fn binary() -> PathBuf {
522 std::env::var_os("CARGO_BIN_EXE_codewhale-tui")
523 .map(PathBuf::from)
524 .unwrap_or_else(|| {
525 PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../target/debug/codewhale-tui")
526 })
527 }
528
529 fn preserve_host_env(command: &mut Command) {
530 command.env_clear();
531 for key in [
532 "PATH",
533 "PATHEXT",
534 "SystemRoot",
535 "SystemDrive",
536 "WINDIR",
537 "COMSPEC",
538 "TEMP",
539 "TMP",
540 "TERM",
541 "LANG",
542 "LC_ALL",
543 ] {
544 if let Some(value) = std::env::var_os(key) {
545 command.env(key, value);
546 }
547 }
548 }
549
550 fn run_with_timeout(mut command: Command, timeout: Duration) -> std::process::Output {
551 let mut child = command.spawn().expect("spawn codewhale exec");
552 let stdout = read_in_background(child.stdout.take().expect("stdout"));
553 let stderr = read_in_background(child.stderr.take().expect("stderr"));
554 let status = child
555 .wait_timeout(timeout)
556 .expect("wait")
557 .unwrap_or_else(|| {
558 let _ = child.kill();
559 let _ = child.wait();
560 panic!("codewhale exec timed out")
561 });
562 std::process::Output {
563 status,
564 stdout: stdout.join().expect("stdout thread").expect("read stdout"),
565 stderr: stderr.join().expect("stderr thread").expect("read stderr"),
566 }
567 }
568
569 fn read_in_background<R: Read + Send + 'static>(
570 mut reader: R,
571 ) -> std::thread::JoinHandle<std::io::Result<Vec<u8>>> {
572 std::thread::spawn(move || {
573 let mut bytes = Vec::new();
574 reader.read_to_end(&mut bytes).map(|_| bytes)
575 })
576 }
577
577 lines RUST