返回 CodeWhale
adaptive_evidence_acceptance.rs
根目录 / crates / tui / tests / integration / adaptive_evidence_acceptance.rs
1 //! Process-level acceptance for adaptive exact-evidence routing (#4619).
2 //!
3 //! "Exact" begins at the common engine routing seam: tool adapters such as
4 //! Bash intentionally bound their own operating-system stream and annotate
5 //! that truncation before returning a `ToolResult`. Adaptive evidence binds
6 //! every byte of that returned result. Root streaming, sequential/deferred
7 //! completion, and MCP all converge on the same engine seam; sub-agents have a
8 //! separate call site covered by `tools::subagent::tests`.
9
10 use std::io::Read;
11 use std::path::{Path, PathBuf};
12 use std::process::{Command, Stdio};
13 use std::sync::Arc;
14 use std::sync::atomic::{AtomicUsize, Ordering};
15 use std::time::Duration;
16
17 use serde_json::{Value, json};
18 use sha2::{Digest, Sha256};
19 use tempfile::TempDir;
20 use wait_timeout::ChildExt;
21 use wiremock::matchers::{method, path};
22 use wiremock::{Mock, MockServer, Request, Respond, ResponseTemplate};
23
24 const MODEL: &str = "adaptive-evidence-test";
25 const SUCCESS_CALL_ID: &str = "call_bash_success";
26 const FAILURE_CALL_ID: &str = "call_bash_failure";
27 const RETRIEVE_CALL_ID: &str = "call_retrieve_omitted_range";
28 const SUCCESS_SENTINEL: &str = "DEEP_SUCCESS_EVIDENCE_SENTINEL_4619";
29 const FAILURE_SENTINEL: &str = "DEEP_FAILURE_EVIDENCE_SENTINEL_4619";
30
31 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
32 async fn headless_bash_success_and_failure_are_distinct_bounded_exact_evidence() {
33 let workspace = TempDir::new().expect("workspace");
34 let home = TempDir::new().expect("home");
35
36 let server = mock_llm().await;
37 let output = run_exec(workspace.path(), home.path(), &server);
38 assert!(
39 output.status.success(),
40 "exec failed\nstdout:\n{}\nstderr:\n{}",
41 String::from_utf8_lossy(&output.stdout),
42 String::from_utf8_lossy(&output.stderr)
43 );
44
45 let requests = server.received_requests().await.expect("recorded requests");
46 let success_receipt =
47 receipt_for(&requests, SUCCESS_CALL_ID).expect("model-visible Bash success receipt");
48 let failure_receipt =
49 receipt_for(&requests, FAILURE_CALL_ID).expect("model-visible Bash failure receipt");
50 for (receipt, sentinel) in [
51 (&success_receipt, SUCCESS_SENTINEL),
52 (&failure_receipt, FAILURE_SENTINEL),
53 ] {
54 assert!(
55 receipt.contains("of output omitted"),
56 "model-facing truncation must state how much was omitted"
57 );
58 assert!(
59 receipt.contains("full output at"),
60 "model-facing truncation must name the recovery path"
61 );
62 assert!(
63 // The footer prints the artifact directory with the platform's
64 // path separator; compare on a normalized view so Windows
65 // backslashes don't fail an otherwise-correct footer.
66 receipt.replace('\\', "/").contains("/artifacts/"),
67 "the footer names where the omitted bytes live on disk"
68 );
69 // The receipt must name a route the model can take *from this
70 // receipt*. It previously asserted the opposite — that the footer must
71 // NOT name `retrieve_tool_result` — which came from #5018's
72 // "no storage language" pass, not from any shell-vs-tool-result
73 // distinction: #4619 shipped the footer naming
74 // `retrieve_tool_result ref=art_<call>`, #5018 replaced the whole
75 // recovery line with "view full output in the tool details view" (a
76 // view the model cannot open) and froze that removal as a negative
77 // assertion, and #5212 restored the artifact path but left the stale
78 // negative in place. The follow-up probe below settles it empirically
79 // for *this* receipt: the scripted model reads the ref out of the
80 // receipt text it was handed, calls `retrieve_tool_result` with it,
81 // and gets back the exact line the receipt omitted.
82 assert!(
83 receipt.contains("retrieve_tool_result"),
84 "the footer must name the recovery route the model can actually take"
85 );
86 assert!(!receipt.contains("[Exact evidence retained"));
87 assert!(!receipt.contains(sentinel));
88 assert!(
89 receipt.len() <= 42_000,
90 "bounded preview must stay within the hybrid 32 KiB head + 8 KiB tail receipt budget, got {} bytes",
91 receipt.len()
92 );
93 }
94 assert_ne!(success_receipt, failure_receipt);
95
96 // The model followed the `ref=` the failure receipt named and got the byte
97 // range the receipt omitted. The mock parsed that ref out of the receipt
98 // text itself, so this only passes when the footer hands over a ref the
99 // retrieval tool can resolve in the origin session.
100 let retrieve_receipt = receipt_for(&requests, RETRIEVE_CALL_ID)
101 .expect("model-visible retrieve_tool_result receipt");
102 assert!(
103 retrieve_receipt.contains(FAILURE_SENTINEL),
104 "retrieve_tool_result must return the range the receipt omitted, got: {retrieve_receipt}"
105 );
106
107 let artifact_dir = find_artifact_dir(home.path()).expect("origin-session artifacts");
108 let payloads = std::fs::read_dir(&artifact_dir)
109 .expect("artifact directory")
110 .filter_map(Result::ok)
111 .map(|entry| entry.path())
112 .filter(|path| path.extension().and_then(|ext| ext.to_str()) == Some("txt"))
113 .count();
114 assert_eq!(payloads, 2, "exactly one evidence payload per result");
115
116 let success = assert_exact_artifact(&artifact_dir, SUCCESS_CALL_ID, SUCCESS_SENTINEL, "Bash");
117 let failure = assert_exact_artifact(&artifact_dir, FAILURE_CALL_ID, FAILURE_SENTINEL, "Bash");
118 assert_ne!(
119 success, failure,
120 "success and failure bytes must stay distinct"
121 );
122 }
123
124 fn assert_exact_artifact(
125 artifact_dir: &Path,
126 call_id: &str,
127 sentinel: &str,
128 tool_name: &str,
129 ) -> Vec<u8> {
130 let handle = format!("art_{call_id}");
131 let exact =
132 std::fs::read(artifact_dir.join(format!("{handle}.txt"))).expect("exact evidence bytes");
133 assert!(
134 String::from_utf8_lossy(&exact).contains(sentinel),
135 "deep content omitted from context must remain retrievable"
136 );
137 let metadata: Value = serde_json::from_slice(
138 &std::fs::read(artifact_dir.join(format!("{handle}.evidence.json")))
139 .expect("evidence metadata"),
140 )
141 .expect("valid evidence metadata");
142 let digest = Sha256::digest(&exact)
143 .iter()
144 .map(|byte| format!("{byte:02x}"))
145 .collect::<String>();
146 assert_eq!(metadata["handle"], handle);
147 assert_eq!(metadata["call_id"], call_id);
148 assert_eq!(metadata["tool_name"], tool_name);
149 assert_eq!(metadata["digest"], digest);
150 assert_eq!(metadata["size_bytes"], exact.len() as u64);
151 assert_eq!(metadata["generation"], 1);
152 assert_eq!(metadata["redacted"], false);
153 assert_eq!(metadata["encoding"], "utf-8");
154 assert_eq!(metadata["retention_state"], "live");
155 assert!(
156 metadata["origin_session"]
157 .as_str()
158 .is_some_and(|id| !id.is_empty())
159 );
160 exact
161 }
162
163 async fn mock_llm() -> MockServer {
164 let server = MockServer::start().await;
165 Mock::given(method("GET"))
166 .and(path("/v1/models"))
167 .respond_with(json_response(json!({
168 "object": "list",
169 "data": [{"id": MODEL, "object": "model"}]
170 })))
171 .mount(&server)
172 .await;
173 Mock::given(method("POST"))
174 .and(path("/v1/chat/completions"))
175 .respond_with(EvidenceScenario {
176 requests: Arc::new(AtomicUsize::new(0)),
177 })
178 .mount(&server)
179 .await;
180 server
181 }
182
183 #[derive(Clone)]
184 struct EvidenceScenario {
185 requests: Arc<AtomicUsize>,
186 }
187
188 /// Scripted four-turn scenario. Turn 3 is the empirical half of the design
189 /// question this test settles: rather than asserting from the outside which
190 /// recovery route a Bash receipt *should* name, the scripted model reads the
191 /// route out of the receipt it was actually handed and takes it, so the
192 /// assertion in the test body observes whether that route returns the bytes
193 /// the receipt omitted.
194 impl Respond for EvidenceScenario {
195 fn respond(&self, request: &Request) -> ResponseTemplate {
196 let sequence = self.requests.fetch_add(1, Ordering::SeqCst);
197 let body = request.body_json::<Value>().unwrap_or(Value::Null);
198 let response = match sequence {
199 0 => bash_tool_sse(SUCCESS_CALL_ID, true),
200 1 => bash_tool_sse(FAILURE_CALL_ID, false),
201 2 => {
202 let receipt = tool_result_content_for(&body, FAILURE_CALL_ID)
203 .expect("Bash failure receipt on the third request");
204 let reference = quoted_after(receipt, "ref=\"")
205 .expect("failure receipt must hand over a retrieval ref");
206 retrieve_tool_sse(reference, FAILURE_SENTINEL)
207 }
208 _ => final_sse(),
209 };
210 assert!(
211 sequence < 4,
212 "unexpected extra model request #{sequence}: {body}"
213 );
214 sse_response(response)
215 }
216 }
217
218 /// Read the recovery ref the truncation footer hands the model, e.g. the
219 /// `art_call_bash_failure` inside `… call retrieve_tool_result with ref="…"`.
220 fn quoted_after<'a>(receipt: &'a str, marker: &str) -> Option<&'a str> {
221 let rest = &receipt[receipt.find(marker)? + marker.len()..];
222 rest.get(..rest.find('"')?)
223 }
224
225 fn run_exec(workspace: &Path, home: &Path, server: &MockServer) -> std::process::Output {
226 std::fs::create_dir_all(home.join(".codewhale")).expect("config directory");
227 std::fs::create_dir_all(home.join(".deepseek")).expect("legacy config directory");
228 std::fs::write(
229 home.join(".codewhale/config.toml"),
230 "allow_shell = true\n\n[retry]\nenabled = false\n",
231 )
232 .expect("headless test config");
233 let mut command = Command::new(binary());
234 preserve_host_env(&mut command);
235 command
236 .current_dir(workspace)
237 .args(["--workspace", workspace.to_str().expect("workspace utf8")])
238 .arg("--no-project-config")
239 .args([
240 "exec",
241 "--auto",
242 "--model",
243 MODEL,
244 "--output-format",
245 "stream-json",
246 ])
247 .arg("run both provider-fixtured Bash evidence probes")
248 .env("HOME", home)
249 .env("USERPROFILE", home)
250 .env("XDG_CONFIG_HOME", home.join(".config"))
251 .env("XDG_DATA_HOME", home.join(".local/share"))
252 .env("XDG_CACHE_HOME", home.join(".cache"))
253 .env("CODEWHALE_CONFIG_PATH", home.join(".codewhale/config.toml"))
254 .env("DEEPSEEK_CONFIG_PATH", home.join(".deepseek/config.toml"))
255 .env("DEEPSEEK_API_KEY", "ci-test-key-not-real")
256 .env("DEEPSEEK_BASE_URL", server.uri())
257 .env("CODEWHALE_BASE_URL", server.uri())
258 .env("DEEPSEEK_MODEL", MODEL)
259 .env("CODEWHALE_MODEL", MODEL)
260 // The adaptive evidence lane is an explicit opt-in; this test exists
261 // to prove it end to end, so the spawned binary runs opted in.
262 .env("CODEWHALE_ADAPTIVE_OUTPUT_ROUTING", "1")
263 .env("RUST_LOG", "warn")
264 .stdout(Stdio::piped())
265 .stderr(Stdio::piped());
266 run_with_timeout(command, Duration::from_secs(45))
267 }
268
269 fn find_artifact_dir(home: &Path) -> Option<PathBuf> {
270 let sessions = home.join(".codewhale/sessions");
271 std::fs::read_dir(sessions)
272 .ok()?
273 .filter_map(Result::ok)
274 .find_map(|entry| {
275 let path = entry.path().join("artifacts");
276 path.is_dir().then_some(path)
277 })
278 }
279
280 fn receipt_for(requests: &[Request], call_id: &str) -> Option<String> {
281 requests
282 .iter()
283 .filter_map(|request| request.body_json::<Value>().ok())
284 .find_map(|body| tool_result_content_for(&body, call_id).map(str::to_owned))
285 }
286
287 fn tool_result_content_for<'a>(body: &'a Value, call_id: &str) -> Option<&'a str> {
288 body.get("messages")?
289 .as_array()?
290 .iter()
291 .find(|message| {
292 message.get("role").and_then(Value::as_str) == Some("tool")
293 && message.get("tool_call_id").and_then(Value::as_str) == Some(call_id)
294 })?
295 .get("content")?
296 .as_str()
297 }
298
299 fn bash_tool_sse(call_id: &str, success: bool) -> String {
300 let (sentinel, prefix) = if success {
301 (SUCCESS_SENTINEL, "BASH-SUCCESS")
302 } else {
303 (FAILURE_SENTINEL, "BASH-FAILURE")
304 };
305 let command = probe_command(sentinel, prefix, success);
306 tool_call_sse(
307 call_id,
308 "Bash",
309 &json!({"action": "run", "command": command, "timeout_ms": 30_000}),
310 )
311 }
312
313 /// Take the recovery route the receipt named, asking for the omitted range by
314 /// the sentinel that rides in it.
315 fn retrieve_tool_sse(reference: &str, query: &str) -> String {
316 tool_call_sse(
317 RETRIEVE_CALL_ID,
318 "retrieve_tool_result",
319 &json!({"ref": reference, "mode": "query", "query": query}),
320 )
321 }
322
323 fn tool_call_sse(call_id: &str, name: &str, arguments: &Value) -> String {
324 let arguments = serde_json::to_string(arguments).expect("tool arguments");
325 [
326 chunk(json!({"id":"tool","object":"chat.completion.chunk","model":MODEL,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":call_id,"type":"function","function":{"name":name,"arguments":arguments}}]},"finish_reason":null}]})),
327 chunk(json!({"id":"tool","object":"chat.completion.chunk","model":MODEL,"choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":10,"completion_tokens":2,"total_tokens":12}})),
328 "data: [DONE]\n\n".to_string(),
329 ].join("")
330 }
331
332 /// Stderr filler line the sentinel rides on, which has to land in a narrow
333 /// window. `shell_output` bounds each stream to `TRUNCATED_HEAD_BYTES` =
334 /// 30_000/5 = 6_000 bytes of head plus 24_000 of tail, so anything past ~line
335 /// 68 of stderr never reaches the artifact at all. Below that, the bounded
336 /// stdout section (~30.1 KB: head + notice + tail) plus the `STDERR:`
337 /// separator puts stderr filler line *n* at roughly 30_110 + 86n bytes of the
338 /// result, so anything before ~line 31 is still inside the preview's 32 KiB
339 /// head and the receipt would show it. Line 50 sits near the middle of
340 /// [31, 68] with ~1.6 KB of slack on each side.
341 ///
342 /// #5212 wrote 100 here against a "22 KB head bound" that does not exist: the
343 /// sentinel landed in the stream's own omitted middle, so the artifact never
344 /// carried it and this test's retrievability assertion failed. It went
345 /// unnoticed because an earlier assertion in the same loop failed first.
346 const SENTINEL_LINE: usize = 50;
347
348 /// Shell fixture that emits enough bytes to force exact-evidence routing under
349 /// the 32_768-token default threshold. The Bash adapter self-bounds each
350 /// stream to ~30 KB, so a single stream would now fit inside the hybrid
351 /// 32 KiB + 8 KiB preview budget; the probe therefore fills stdout AND stderr
352 /// (~60 KB combined) so the envelope still omits a middle range, with the
353 /// sentinel at [`SENTINEL_LINE`] of stderr. The probe executes through the
354 /// platform shell — bash on Unix, `cmd /C` on Windows (#1691) — so each
355 /// platform needs native syntax to exercise the same routing path.
356 #[cfg(not(windows))]
357 fn probe_command(sentinel: &str, prefix: &str, success: bool) -> String {
358 let trailer = if success { "" } else { "; exit 7" };
359 let stdout_loop = format!(
360 "i=0; while [ \"$i\" -lt 2800 ]; do printf '{prefix}-%04d-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\\n' \"$i\"; i=$((i + 1)); done"
361 );
362 let stderr_loop = format!(
363 "j=0; while [ \"$j\" -lt 2800 ]; do if [ \"$j\" -eq {SENTINEL_LINE} ]; then printf '%s\\n' '{sentinel}'; fi; printf '{prefix}-ERR-%04d-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\\n' \"$j\"; j=$((j + 1)); done"
364 );
365 format!("{stdout_loop}; {{ {stderr_loop}; }} >&2{trailer}")
366 }
367
368 /// PowerShell syntax: on Windows the shell dispatcher prefers `pwsh.exe`,
369 /// then in-box `powershell.exe`, only falling back to `cmd.exe` when no
370 /// PowerShell exists at all. Single quotes only — the payload is passed to
371 /// `-Command` as one argv string, and four or more double quotes would push
372 /// it onto the temp-`-File` path for no benefit. The failure variant mirrors
373 /// the Unix `{ ...; } >&2; exit 7` shape by writing every line to the OS
374 /// stderr handle and exiting 7 after the loop.
375 #[cfg(windows)]
376 fn probe_command(sentinel: &str, prefix: &str, success: bool) -> String {
377 let stdout_line = format!(
378 "'{prefix}-{{0}}-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'"
379 );
380 let stderr_line = format!(
381 "'{prefix}-ERR-{{0}}-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'"
382 );
383 let stdout_loop = format!("0..2799 | ForEach-Object {{ Write-Output ({stdout_line} -f $_) }}");
384 let stderr_loop = format!(
385 "0..2799 | ForEach-Object {{ if ($_ -eq {SENTINEL_LINE}) {{ [Console]::Error.WriteLine('{sentinel}') }}; [Console]::Error.WriteLine(({stderr_line} -f $_)) }}"
386 );
387 let trailer = if success { "" } else { "; exit 7" };
388 format!("{stdout_loop}; {stderr_loop}{trailer}")
389 }
390
391 fn final_sse() -> String {
392 [
393 chunk(json!({"id":"final","object":"chat.completion.chunk","model":MODEL,"choices":[{"index":0,"delta":{"content":"evidence retained"},"finish_reason":null}]})),
394 chunk(json!({"id":"final","object":"chat.completion.chunk","model":MODEL,"choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":20,"completion_tokens":2,"total_tokens":22}})),
395 "data: [DONE]\n\n".to_string(),
396 ].join("")
397 }
398
399 fn chunk(value: Value) -> String {
400 format!(
401 "data: {}\n\n",
402 serde_json::to_string(&value).expect("SSE JSON")
403 )
404 }
405
406 fn sse_response(body: String) -> ResponseTemplate {
407 ResponseTemplate::new(200)
408 .insert_header("content-type", "text/event-stream")
409 .set_body_string(body)
410 }
411
412 fn json_response(value: Value) -> ResponseTemplate {
413 ResponseTemplate::new(200).set_body_json(value)
414 }
415
416 fn binary() -> PathBuf {
417 std::env::var_os("CARGO_BIN_EXE_codewhale-tui")
418 .map(PathBuf::from)
419 .unwrap_or_else(|| {
420 PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../target/debug/codewhale-tui")
421 })
422 }
423
424 fn preserve_host_env(command: &mut Command) {
425 command.env_clear();
426 for key in [
427 "PATH",
428 "PATHEXT",
429 "SystemRoot",
430 "SystemDrive",
431 "WINDIR",
432 "COMSPEC",
433 "TEMP",
434 "TMP",
435 "TERM",
436 "LANG",
437 "LC_ALL",
438 ] {
439 if let Some(value) = std::env::var_os(key) {
440 command.env(key, value);
441 }
442 }
443 }
444
445 fn run_with_timeout(mut command: Command, timeout: Duration) -> std::process::Output {
446 let mut child = command.spawn().expect("spawn codewhale exec");
447 let stdout = read_in_background(child.stdout.take().expect("stdout"));
448 let stderr = read_in_background(child.stderr.take().expect("stderr"));
449 let status = child
450 .wait_timeout(timeout)
451 .expect("wait")
452 .unwrap_or_else(|| {
453 let _ = child.kill();
454 let _ = child.wait();
455 panic!("codewhale exec timed out")
456 });
457 std::process::Output {
458 status,
459 stdout: stdout.join().expect("stdout thread").expect("read stdout"),
460 stderr: stderr.join().expect("stderr thread").expect("read stderr"),
461 }
462 }
463
464 fn read_in_background<R: Read + Send + 'static>(
465 mut reader: R,
466 ) -> std::thread::JoinHandle<std::io::Result<Vec<u8>>> {
467 std::thread::spawn(move || {
468 let mut bytes = Vec::new();
469 reader.read_to_end(&mut bytes).map(|_| bytes)
470 })
471 }
472
472 lines RUST