返回 CodeWhale
adaptive_evidence_acceptance.rs
根目录 / crates / tui / tests / adaptive_evidence_acceptance.rs
1 //! Process-level acceptance for adaptive exact-evidence routing (#4619).
2 //!
3 //! "Exact" begins at the common engine routing seam: tool adapters such as
4 //! Bash intentionally bound their own operating-system stream and annotate
5 //! that truncation before returning a `ToolResult`. Adaptive evidence binds
6 //! every byte of that returned result. Root streaming, sequential/deferred
7 //! completion, and MCP all converge on the same engine seam; sub-agents have a
8 //! separate call site covered by `tools::subagent::tests`.
9
10 use std::io::Read;
11 use std::path::{Path, PathBuf};
12 use std::process::{Command, Stdio};
13 use std::sync::Arc;
14 use std::sync::atomic::{AtomicUsize, Ordering};
15 use std::time::Duration;
16
17 use serde_json::{Value, json};
18 use sha2::{Digest, Sha256};
19 use tempfile::TempDir;
20 use wait_timeout::ChildExt;
21 use wiremock::matchers::{method, path};
22 use wiremock::{Mock, MockServer, Request, Respond, ResponseTemplate};
23
24 const MODEL: &str = "adaptive-evidence-test";
25 const SUCCESS_CALL_ID: &str = "call_bash_success";
26 const FAILURE_CALL_ID: &str = "call_bash_failure";
27 const RETRIEVE_CALL_ID: &str = "call_retrieve_omitted_range";
28 const SUCCESS_SENTINEL: &str = "DEEP_SUCCESS_EVIDENCE_SENTINEL_4619";
29 const FAILURE_SENTINEL: &str = "DEEP_FAILURE_EVIDENCE_SENTINEL_4619";
30
31 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
32 async fn headless_bash_success_and_failure_are_distinct_bounded_exact_evidence() {
33 let workspace = TempDir::new().expect("workspace");
34 let home = TempDir::new().expect("home");
35
36 let server = mock_llm().await;
37 let output = run_exec(workspace.path(), home.path(), &server);
38 assert!(
39 output.status.success(),
40 "exec failed\nstdout:\n{}\nstderr:\n{}",
41 String::from_utf8_lossy(&output.stdout),
42 String::from_utf8_lossy(&output.stderr)
43 );
44
45 let requests = server.received_requests().await.expect("recorded requests");
46 let success_receipt =
47 receipt_for(&requests, SUCCESS_CALL_ID).expect("model-visible Bash success receipt");
48 let failure_receipt =
49 receipt_for(&requests, FAILURE_CALL_ID).expect("model-visible Bash failure receipt");
50 for (receipt, sentinel) in [
51 (&success_receipt, SUCCESS_SENTINEL),
52 (&failure_receipt, FAILURE_SENTINEL),
53 ] {
54 assert!(
55 receipt.contains("of output omitted"),
56 "model-facing truncation must state how much was omitted"
57 );
58 assert!(
59 receipt.contains("full output at"),
60 "model-facing truncation must name the recovery path"
61 );
62 assert!(
63 receipt.contains("/artifacts/"),
64 "the footer names where the omitted bytes live on disk"
65 );
66 // The receipt must name a route the model can take *from this
67 // receipt*. It previously asserted the opposite — that the footer must
68 // NOT name `retrieve_tool_result` — which came from #5018's
69 // "no storage language" pass, not from any shell-vs-tool-result
70 // distinction: #4619 shipped the footer naming
71 // `retrieve_tool_result ref=art_<call>`, #5018 replaced the whole
72 // recovery line with "view full output in the tool details view" (a
73 // view the model cannot open) and froze that removal as a negative
74 // assertion, and #5212 restored the artifact path but left the stale
75 // negative in place. The follow-up probe below settles it empirically
76 // for *this* receipt: the scripted model reads the ref out of the
77 // receipt text it was handed, calls `retrieve_tool_result` with it,
78 // and gets back the exact line the receipt omitted.
79 assert!(
80 receipt.contains("retrieve_tool_result"),
81 "the footer must name the recovery route the model can actually take"
82 );
83 assert!(!receipt.contains("[Exact evidence retained"));
84 assert!(!receipt.contains(sentinel));
85 assert!(
86 receipt.len() <= 42_000,
87 "bounded preview must stay within the hybrid 32 KiB head + 8 KiB tail receipt budget, got {} bytes",
88 receipt.len()
89 );
90 }
91 assert_ne!(success_receipt, failure_receipt);
92
93 // The model followed the `ref=` the failure receipt named and got the byte
94 // range the receipt omitted. The mock parsed that ref out of the receipt
95 // text itself, so this only passes when the footer hands over a ref the
96 // retrieval tool can resolve in the origin session.
97 let retrieve_receipt = receipt_for(&requests, RETRIEVE_CALL_ID)
98 .expect("model-visible retrieve_tool_result receipt");
99 assert!(
100 retrieve_receipt.contains(FAILURE_SENTINEL),
101 "retrieve_tool_result must return the range the receipt omitted, got: {retrieve_receipt}"
102 );
103
104 let artifact_dir = find_artifact_dir(home.path()).expect("origin-session artifacts");
105 let payloads = std::fs::read_dir(&artifact_dir)
106 .expect("artifact directory")
107 .filter_map(Result::ok)
108 .map(|entry| entry.path())
109 .filter(|path| path.extension().and_then(|ext| ext.to_str()) == Some("txt"))
110 .count();
111 assert_eq!(payloads, 2, "exactly one evidence payload per result");
112
113 let success = assert_exact_artifact(&artifact_dir, SUCCESS_CALL_ID, SUCCESS_SENTINEL, "Bash");
114 let failure = assert_exact_artifact(&artifact_dir, FAILURE_CALL_ID, FAILURE_SENTINEL, "Bash");
115 assert_ne!(
116 success, failure,
117 "success and failure bytes must stay distinct"
118 );
119 }
120
121 fn assert_exact_artifact(
122 artifact_dir: &Path,
123 call_id: &str,
124 sentinel: &str,
125 tool_name: &str,
126 ) -> Vec<u8> {
127 let handle = format!("art_{call_id}");
128 let exact =
129 std::fs::read(artifact_dir.join(format!("{handle}.txt"))).expect("exact evidence bytes");
130 assert!(
131 String::from_utf8_lossy(&exact).contains(sentinel),
132 "deep content omitted from context must remain retrievable"
133 );
134 let metadata: Value = serde_json::from_slice(
135 &std::fs::read(artifact_dir.join(format!("{handle}.evidence.json")))
136 .expect("evidence metadata"),
137 )
138 .expect("valid evidence metadata");
139 let digest = Sha256::digest(&exact)
140 .iter()
141 .map(|byte| format!("{byte:02x}"))
142 .collect::<String>();
143 assert_eq!(metadata["handle"], handle);
144 assert_eq!(metadata["call_id"], call_id);
145 assert_eq!(metadata["tool_name"], tool_name);
146 assert_eq!(metadata["digest"], digest);
147 assert_eq!(metadata["size_bytes"], exact.len() as u64);
148 assert_eq!(metadata["generation"], 1);
149 assert_eq!(metadata["redacted"], false);
150 assert_eq!(metadata["encoding"], "utf-8");
151 assert_eq!(metadata["retention_state"], "live");
152 assert!(
153 metadata["origin_session"]
154 .as_str()
155 .is_some_and(|id| !id.is_empty())
156 );
157 exact
158 }
159
160 async fn mock_llm() -> MockServer {
161 let server = MockServer::start().await;
162 Mock::given(method("GET"))
163 .and(path("/v1/models"))
164 .respond_with(json_response(json!({
165 "object": "list",
166 "data": [{"id": MODEL, "object": "model"}]
167 })))
168 .mount(&server)
169 .await;
170 Mock::given(method("POST"))
171 .and(path("/v1/chat/completions"))
172 .respond_with(EvidenceScenario {
173 requests: Arc::new(AtomicUsize::new(0)),
174 })
175 .mount(&server)
176 .await;
177 server
178 }
179
180 #[derive(Clone)]
181 struct EvidenceScenario {
182 requests: Arc<AtomicUsize>,
183 }
184
185 /// Scripted four-turn scenario. Turn 3 is the empirical half of the design
186 /// question this test settles: rather than asserting from the outside which
187 /// recovery route a Bash receipt *should* name, the scripted model reads the
188 /// route out of the receipt it was actually handed and takes it, so the
189 /// assertion in the test body observes whether that route returns the bytes
190 /// the receipt omitted.
191 impl Respond for EvidenceScenario {
192 fn respond(&self, request: &Request) -> ResponseTemplate {
193 let sequence = self.requests.fetch_add(1, Ordering::SeqCst);
194 let body = request.body_json::<Value>().unwrap_or(Value::Null);
195 let response = match sequence {
196 0 => bash_tool_sse(SUCCESS_CALL_ID, true),
197 1 => bash_tool_sse(FAILURE_CALL_ID, false),
198 2 => {
199 let receipt = tool_result_content_for(&body, FAILURE_CALL_ID)
200 .expect("Bash failure receipt on the third request");
201 let reference = quoted_after(receipt, "ref=\"")
202 .expect("failure receipt must hand over a retrieval ref");
203 retrieve_tool_sse(reference, FAILURE_SENTINEL)
204 }
205 _ => final_sse(),
206 };
207 assert!(
208 sequence < 4,
209 "unexpected extra model request #{sequence}: {body}"
210 );
211 sse_response(response)
212 }
213 }
214
215 /// Read the recovery ref the truncation footer hands the model, e.g. the
216 /// `art_call_bash_failure` inside `… call retrieve_tool_result with ref="…"`.
217 fn quoted_after<'a>(receipt: &'a str, marker: &str) -> Option<&'a str> {
218 let rest = &receipt[receipt.find(marker)? + marker.len()..];
219 rest.get(..rest.find('"')?)
220 }
221
222 fn run_exec(workspace: &Path, home: &Path, server: &MockServer) -> std::process::Output {
223 std::fs::create_dir_all(home.join(".codewhale")).expect("config directory");
224 std::fs::create_dir_all(home.join(".deepseek")).expect("legacy config directory");
225 std::fs::write(
226 home.join(".codewhale/config.toml"),
227 "allow_shell = true\n\n[retry]\nenabled = false\n",
228 )
229 .expect("headless test config");
230 let mut command = Command::new(binary());
231 preserve_host_env(&mut command);
232 command
233 .current_dir(workspace)
234 .args(["--workspace", workspace.to_str().expect("workspace utf8")])
235 .arg("--no-project-config")
236 .args([
237 "exec",
238 "--auto",
239 "--model",
240 MODEL,
241 "--output-format",
242 "stream-json",
243 ])
244 .arg("run both provider-fixtured Bash evidence probes")
245 .env("HOME", home)
246 .env("USERPROFILE", home)
247 .env("XDG_CONFIG_HOME", home.join(".config"))
248 .env("XDG_DATA_HOME", home.join(".local/share"))
249 .env("XDG_CACHE_HOME", home.join(".cache"))
250 .env("CODEWHALE_CONFIG_PATH", home.join(".codewhale/config.toml"))
251 .env("DEEPSEEK_CONFIG_PATH", home.join(".deepseek/config.toml"))
252 .env("DEEPSEEK_API_KEY", "ci-test-key-not-real")
253 .env("DEEPSEEK_BASE_URL", server.uri())
254 .env("CODEWHALE_BASE_URL", server.uri())
255 .env("DEEPSEEK_MODEL", MODEL)
256 .env("CODEWHALE_MODEL", MODEL)
257 .env("RUST_LOG", "warn")
258 .stdout(Stdio::piped())
259 .stderr(Stdio::piped());
260 run_with_timeout(command, Duration::from_secs(45))
261 }
262
263 fn find_artifact_dir(home: &Path) -> Option<PathBuf> {
264 let sessions = home.join(".codewhale/sessions");
265 std::fs::read_dir(sessions)
266 .ok()?
267 .filter_map(Result::ok)
268 .find_map(|entry| {
269 let path = entry.path().join("artifacts");
270 path.is_dir().then_some(path)
271 })
272 }
273
274 fn receipt_for(requests: &[Request], call_id: &str) -> Option<String> {
275 requests
276 .iter()
277 .filter_map(|request| request.body_json::<Value>().ok())
278 .find_map(|body| tool_result_content_for(&body, call_id).map(str::to_owned))
279 }
280
281 fn tool_result_content_for<'a>(body: &'a Value, call_id: &str) -> Option<&'a str> {
282 body.get("messages")?
283 .as_array()?
284 .iter()
285 .find(|message| {
286 message.get("role").and_then(Value::as_str) == Some("tool")
287 && message.get("tool_call_id").and_then(Value::as_str) == Some(call_id)
288 })?
289 .get("content")?
290 .as_str()
291 }
292
293 fn bash_tool_sse(call_id: &str, success: bool) -> String {
294 let (sentinel, prefix) = if success {
295 (SUCCESS_SENTINEL, "BASH-SUCCESS")
296 } else {
297 (FAILURE_SENTINEL, "BASH-FAILURE")
298 };
299 let command = probe_command(sentinel, prefix, success);
300 tool_call_sse(
301 call_id,
302 "Bash",
303 &json!({"action": "run", "command": command, "timeout_ms": 30_000}),
304 )
305 }
306
307 /// Take the recovery route the receipt named, asking for the omitted range by
308 /// the sentinel that rides in it.
309 fn retrieve_tool_sse(reference: &str, query: &str) -> String {
310 tool_call_sse(
311 RETRIEVE_CALL_ID,
312 "retrieve_tool_result",
313 &json!({"ref": reference, "mode": "query", "query": query}),
314 )
315 }
316
317 fn tool_call_sse(call_id: &str, name: &str, arguments: &Value) -> String {
318 let arguments = serde_json::to_string(arguments).expect("tool arguments");
319 [
320 chunk(json!({"id":"tool","object":"chat.completion.chunk","model":MODEL,"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":call_id,"type":"function","function":{"name":name,"arguments":arguments}}]},"finish_reason":null}]})),
321 chunk(json!({"id":"tool","object":"chat.completion.chunk","model":MODEL,"choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":10,"completion_tokens":2,"total_tokens":12}})),
322 "data: [DONE]\n\n".to_string(),
323 ].join("")
324 }
325
326 /// Stderr filler line the sentinel rides on, which has to land in a narrow
327 /// window. `shell_output` bounds each stream to `TRUNCATED_HEAD_BYTES` =
328 /// 30_000/5 = 6_000 bytes of head plus 24_000 of tail, so anything past ~line
329 /// 68 of stderr never reaches the artifact at all. Below that, the bounded
330 /// stdout section (~30.1 KB: head + notice + tail) plus the `STDERR:`
331 /// separator puts stderr filler line *n* at roughly 30_110 + 86n bytes of the
332 /// result, so anything before ~line 31 is still inside the preview's 32 KiB
333 /// head and the receipt would show it. Line 50 sits near the middle of
334 /// [31, 68] with ~1.6 KB of slack on each side.
335 ///
336 /// #5212 wrote 100 here against a "22 KB head bound" that does not exist: the
337 /// sentinel landed in the stream's own omitted middle, so the artifact never
338 /// carried it and this test's retrievability assertion failed. It went
339 /// unnoticed because an earlier assertion in the same loop failed first.
340 const SENTINEL_LINE: usize = 50;
341
342 /// Shell fixture that emits enough bytes to force exact-evidence routing under
343 /// the 32_768-token default threshold. The Bash adapter self-bounds each
344 /// stream to ~30 KB, so a single stream would now fit inside the hybrid
345 /// 32 KiB + 8 KiB preview budget; the probe therefore fills stdout AND stderr
346 /// (~60 KB combined) so the envelope still omits a middle range, with the
347 /// sentinel at [`SENTINEL_LINE`] of stderr. The probe executes through the
348 /// platform shell — bash on Unix, `cmd /C` on Windows (#1691) — so each
349 /// platform needs native syntax to exercise the same routing path.
350 #[cfg(not(windows))]
351 fn probe_command(sentinel: &str, prefix: &str, success: bool) -> String {
352 let trailer = if success { "" } else { "; exit 7" };
353 let stdout_loop = format!(
354 "i=0; while [ \"$i\" -lt 2800 ]; do printf '{prefix}-%04d-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\\n' \"$i\"; i=$((i + 1)); done"
355 );
356 let stderr_loop = format!(
357 "j=0; while [ \"$j\" -lt 2800 ]; do if [ \"$j\" -eq {SENTINEL_LINE} ]; then printf '%s\\n' '{sentinel}'; fi; printf '{prefix}-ERR-%04d-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\\n' \"$j\"; j=$((j + 1)); done"
358 );
359 format!("{stdout_loop}; {{ {stderr_loop}; }} >&2{trailer}")
360 }
361
362 /// PowerShell syntax: on Windows the shell dispatcher prefers `pwsh.exe`,
363 /// then in-box `powershell.exe`, only falling back to `cmd.exe` when no
364 /// PowerShell exists at all. Single quotes only — the payload is passed to
365 /// `-Command` as one argv string, and four or more double quotes would push
366 /// it onto the temp-`-File` path for no benefit. The failure variant mirrors
367 /// the Unix `{ ...; } >&2; exit 7` shape by writing every line to the OS
368 /// stderr handle and exiting 7 after the loop.
369 #[cfg(windows)]
370 fn probe_command(sentinel: &str, prefix: &str, success: bool) -> String {
371 let stdout_line = format!(
372 "'{prefix}-{{0}}-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'"
373 );
374 let stderr_line = format!(
375 "'{prefix}-ERR-{{0}}-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'"
376 );
377 let stdout_loop = format!("0..2799 | ForEach-Object {{ Write-Output ({stdout_line} -f $_) }}");
378 let stderr_loop = format!(
379 "0..2799 | ForEach-Object {{ if ($_ -eq {SENTINEL_LINE}) {{ [Console]::Error.WriteLine('{sentinel}') }}; [Console]::Error.WriteLine(({stderr_line} -f $_)) }}"
380 );
381 let trailer = if success { "" } else { "; exit 7" };
382 format!("{stdout_loop}; {stderr_loop}{trailer}")
383 }
384
385 fn final_sse() -> String {
386 [
387 chunk(json!({"id":"final","object":"chat.completion.chunk","model":MODEL,"choices":[{"index":0,"delta":{"content":"evidence retained"},"finish_reason":null}]})),
388 chunk(json!({"id":"final","object":"chat.completion.chunk","model":MODEL,"choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":20,"completion_tokens":2,"total_tokens":22}})),
389 "data: [DONE]\n\n".to_string(),
390 ].join("")
391 }
392
393 fn chunk(value: Value) -> String {
394 format!(
395 "data: {}\n\n",
396 serde_json::to_string(&value).expect("SSE JSON")
397 )
398 }
399
400 fn sse_response(body: String) -> ResponseTemplate {
401 ResponseTemplate::new(200)
402 .insert_header("content-type", "text/event-stream")
403 .set_body_string(body)
404 }
405
406 fn json_response(value: Value) -> ResponseTemplate {
407 ResponseTemplate::new(200).set_body_json(value)
408 }
409
410 fn binary() -> PathBuf {
411 std::env::var_os("CARGO_BIN_EXE_codewhale-tui")
412 .map(PathBuf::from)
413 .unwrap_or_else(|| {
414 PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../target/debug/codewhale-tui")
415 })
416 }
417
418 fn preserve_host_env(command: &mut Command) {
419 command.env_clear();
420 for key in [
421 "PATH",
422 "PATHEXT",
423 "SystemRoot",
424 "SystemDrive",
425 "WINDIR",
426 "COMSPEC",
427 "TEMP",
428 "TMP",
429 "TERM",
430 "LANG",
431 "LC_ALL",
432 ] {
433 if let Some(value) = std::env::var_os(key) {
434 command.env(key, value);
435 }
436 }
437 }
438
439 fn run_with_timeout(mut command: Command, timeout: Duration) -> std::process::Output {
440 let mut child = command.spawn().expect("spawn codewhale exec");
441 let stdout = read_in_background(child.stdout.take().expect("stdout"));
442 let stderr = read_in_background(child.stderr.take().expect("stderr"));
443 let status = child
444 .wait_timeout(timeout)
445 .expect("wait")
446 .unwrap_or_else(|| {
447 let _ = child.kill();
448 let _ = child.wait();
449 panic!("codewhale exec timed out")
450 });
451 std::process::Output {
452 status,
453 stdout: stdout.join().expect("stdout thread").expect("read stdout"),
454 stderr: stderr.join().expect("stderr thread").expect("read stderr"),
455 }
456 }
457
458 fn read_in_background<R: Read + Send + 'static>(
459 mut reader: R,
460 ) -> std::thread::JoinHandle<std::io::Result<Vec<u8>>> {
461 std::thread::spawn(move || {
462 let mut bytes = Vec::new();
463 reader.read_to_end(&mut bytes).map(|_| bytes)
464 })
465 }
466
466 lines RUST