返回 CodeWhale
integration_mock_llm.rs
根目录 / crates / tui / tests / integration / integration_mock_llm.rs
1 //! Integration tests for the [`MockLlmClient`](mock::MockLlmClient).
2 //!
3 //! These tests exercise the [`LlmClient`](llm_client::LlmClient) trait surface
4 //! directly. They verify that the mock client itself behaves correctly under
5 //! the patterns the runtime relies on:
6 //!
7 //! - **Streaming turn loop** — events arrive in order, `MessageStop` terminates
8 //! the stream.
9 //! - **Reasoning replay** (issue #69 / V4 §5.1.1) — when the runtime sends a
10 //! second turn after a tool round, it MUST replay prior `reasoning_content`.
11 //! Catches the HTTP 400 path that broke v0.4.9-v0.5.1.
12 //! - **Tool-call round-trip** — assistant emits `tool_calls`, runtime executes,
13 //! tool result is appended, next turn streams text.
14 //! - **Multiple tool calls in one round** — assistant returns N tool_calls;
15 //! the request payload preserves their ordering.
16 //! - **Compaction-style non-streaming call** — `create_message` returns a
17 //! queued `MessageResponse` without going through the streaming path.
18 //! - **Sub-agent style turn** — child mailbox receives a parent prompt and
19 //! replies; trait boundary is the same.
20 //! - **Capacity-gate observation** — runtime can probe estimated request size
21 //! and decline to dispatch; the mock surfaces capture-side hooks for that.
22 //!
23 //! # Why trait-level (not engine-level)
24 //!
25 //! As of v0.6.7 the engine (`crates/tui/src/core/engine.rs`) holds a concrete
26 //! `Option<CodewhaleClient>` — the [`LlmClient`] trait is implemented but no
27 //! consumer takes `Arc<dyn LlmClient>` or generic `<C: LlmClient>`. Wiring the
28 //! mock into a full engine turn-loop therefore requires a separate refactor:
29 //! every `Option<CodewhaleClient>` consumer (engine, registry, rlm, review,
30 //! cycle_manager, compaction, subagent) must move to `Arc<dyn LlmClient>`.
31 //!
32 //! Per the v0.7.0 mock-LLM issue (the parent of this file): "If the engine's
33 //! API surfaces are too tangled to mock cleanly … document that as BLOCKED with
34 //! what wiring needs to change. In that case still commit any partial work
35 //! that lands cleanly." Full engine integration coverage remains blocked on
36 //! that seam; this file keeps the blocker documented instead of carrying
37 //! ignored placeholder tests.
38 //!
39 //! Once `Arc<dyn LlmClient>` lands, add engine-level tests that reuse this mock.
40
41 use futures_util::StreamExt;
42
43 // Bring in the production model types verbatim — no other crate sources are
44 // needed because the mock is self-contained against `models.rs`.
45
46 // Mirror the real `llm_client` module hierarchy so that `mock.rs`'s
47 // `super::{LlmClient, StreamEventBox}` paths resolve. We re-declare a local
48 // `LlmClient` trait + `StreamEventBox` alias that match the production shape
49 // 1:1 (the public surface that ships in the binary). The mock implements
50 // this local trait, which is structurally identical to the production trait.
51 //
52 // The helper file lives under `tests/support/` so cargo does not try to
53 // compile it as its own test binary.
54
55 use crate::llm_client::LlmClient;
56 use crate::llm_client::mock::{MockLlmClient, canned};
57 use codewhale_models::Role;
58 use codewhale_models::{ContentBlock, Delta, Message, MessageRequest, StreamEvent, Usage};
59
60 // === Helpers ===============================================================
61
62 fn user_message(text: &str) -> Message {
63 Message {
64 role: Role::User,
65 content: vec![ContentBlock::Text {
66 text: text.to_string(),
67 cache_control: None,
68 }],
69 }
70 }
71
72 fn assistant_thinking(thinking: &str, text: &str) -> Message {
73 Message {
74 role: Role::Assistant,
75 content: vec![
76 ContentBlock::Thinking {
77 thinking: thinking.to_string(),
78 signature: None,
79 state: None,
80 },
81 ContentBlock::Text {
82 text: text.to_string(),
83 cache_control: None,
84 },
85 ],
86 }
87 }
88
89 fn assistant_tool_call(id: &str, name: &str, input: serde_json::Value) -> Message {
90 Message {
91 role: Role::Assistant,
92 content: vec![ContentBlock::ToolUse {
93 id: id.to_string(),
94 name: name.to_string(),
95 input,
96 caller: None,
97 thought_signature: None,
98 }],
99 }
100 }
101
102 fn tool_result_message(tool_use_id: &str, content: &str) -> Message {
103 Message {
104 role: Role::User,
105 content: vec![ContentBlock::ToolResult {
106 tool_use_id: tool_use_id.to_string(),
107 content: content.to_string(),
108 is_error: None,
109 content_blocks: None,
110 }],
111 }
112 }
113
114 fn make_request(messages: Vec<Message>) -> MessageRequest {
115 MessageRequest {
116 model: "deepseek-v4-pro".to_string(),
117 messages,
118 max_tokens: 4096,
119 system: None,
120 tools: None,
121 tool_choice: None,
122 metadata: None,
123 thinking: None,
124 reasoning_effort: Some("high".to_string()),
125 stream: Some(true),
126 temperature: None,
127 top_p: None,
128 }
129 }
130
131 async fn drain_stream_text(
132 mock: &MockLlmClient,
133 request: MessageRequest,
134 ) -> (String, Option<String>) {
135 let mut stream = mock
136 .create_message_stream(request)
137 .await
138 .expect("stream open");
139 let mut text = String::new();
140 let mut stop_reason: Option<String> = None;
141 while let Some(ev) = stream.next().await {
142 match ev.expect("event") {
143 StreamEvent::ContentBlockDelta {
144 delta: Delta::TextDelta { text: t },
145 ..
146 } => text.push_str(&t),
147 StreamEvent::MessageDelta { delta, .. } => {
148 stop_reason = delta.stop_reason;
149 }
150 StreamEvent::MessageStop => break,
151 _ => {}
152 }
153 }
154 (text, stop_reason)
155 }
156
157 // === 1. Full turn loop with streaming =======================================
158
159 #[tokio::test]
160 async fn full_turn_loop_streams_text_chunks() {
161 // Two text deltas + finish reason — exercises the canonical streaming
162 // turn-loop path the engine drives.
163 let turn = vec![
164 canned::message_start("msg_1"),
165 canned::text_block_start(0),
166 canned::text_delta(0, "Hello, "),
167 canned::text_delta(0, "world!"),
168 canned::block_stop(0),
169 canned::message_delta("end_turn", Some(Usage::default())),
170 canned::message_stop(),
171 ];
172 let mock = MockLlmClient::new(vec![turn]);
173
174 let request = make_request(vec![user_message("greet me")]);
175 let (text, stop) = drain_stream_text(&mock, request).await;
176
177 assert_eq!(text, "Hello, world!");
178 assert_eq!(stop.as_deref(), Some("end_turn"));
179 assert_eq!(mock.call_count(), 1);
180 assert_eq!(mock.captured_requests().len(), 1);
181 }
182
183 // === 2. Reasoning replay (V4 thinking-mode HTTP-400 regression) =============
184
185 #[tokio::test]
186 async fn reasoning_replay_required_on_subsequent_turn() {
187 // Turn 1: assistant emits thinking + tool_call. Turn 2: text reply.
188 let turn1 = vec![
189 canned::message_start("r1"),
190 canned::thinking_delta(0, "I should call list_dir."),
191 canned::tool_use_block_start(1, "call_a", "list_dir"),
192 canned::tool_input_delta(1, r#"{"path":"/tmp"}"#),
193 canned::block_stop(1),
194 canned::message_delta("tool_use", None),
195 canned::message_stop(),
196 ];
197 let turn2 = vec![
198 canned::message_start("r2"),
199 canned::text_block_start(0),
200 canned::text_delta(0, "I see /tmp."),
201 canned::block_stop(0),
202 canned::message_delta("end_turn", None),
203 canned::message_stop(),
204 ];
205 let mock = MockLlmClient::new(vec![turn1, turn2]);
206
207 // === Round 1: user prompt -> assistant tool_call ===
208 let req1 = make_request(vec![user_message("list /tmp")]);
209 let _ = mock.create_message_stream(req1).await.unwrap().next().await;
210 // (we don't drain — capture is what matters here)
211
212 // === Round 2: runtime composes the next request including the prior
213 // assistant turn's reasoning_content. The mock can verify that any
214 // ContentBlock::Thinking the runtime preserves is present in the next
215 // outgoing request — the very payload shape that broke v0.4.9-v0.5.1.
216 let next_messages = vec![
217 user_message("list /tmp"),
218 assistant_thinking("I should call list_dir.", ""),
219 assistant_tool_call("call_a", "list_dir", serde_json::json!({ "path": "/tmp" })),
220 tool_result_message("call_a", "/tmp/file1\n/tmp/file2"),
221 ];
222 let req2 = make_request(next_messages);
223 let _ = mock.create_message_stream(req2).await.unwrap();
224
225 // The mock captured both requests. Assert the SECOND request preserves
226 // the prior assistant message's Thinking block — i.e. the runtime did
227 // not strip reasoning_content before re-sending. (V4 thinking-mode tool
228 // turns reject HTTP 400 if reasoning_content is missing.)
229 let captured = mock.captured_requests();
230 assert_eq!(captured.len(), 2);
231
232 let req2 = &captured[1];
233 let assistant_with_thinking = req2
234 .messages
235 .iter()
236 .find(|m| {
237 m.role == "assistant"
238 && m.content
239 .iter()
240 .any(|b| matches!(b, ContentBlock::Thinking { .. }))
241 })
242 .expect("turn 2 request must replay assistant Thinking content");
243
244 let thinking_text = assistant_with_thinking
245 .content
246 .iter()
247 .find_map(|b| match b {
248 ContentBlock::Thinking { thinking, .. } => Some(thinking.clone()),
249 _ => None,
250 })
251 .expect("Thinking block present");
252 assert_eq!(
253 thinking_text, "I should call list_dir.",
254 "reasoning_content must be replayed verbatim across tool-call rounds"
255 );
256 }
257
258 // === 3. Tool-call round-trip ================================================
259
260 #[tokio::test]
261 async fn tool_call_round_trip_streams_args_then_continues() {
262 // Turn 1 emits a tool_use block with chunked input JSON.
263 let turn1 = vec![
264 canned::message_start("rt1"),
265 canned::tool_use_block_start(0, "call_x", "read_file"),
266 canned::tool_input_delta(0, r#"{"path":"#),
267 canned::tool_input_delta(0, r#""README.md"}"#),
268 canned::block_stop(0),
269 canned::message_delta("tool_use", None),
270 canned::message_stop(),
271 ];
272 let turn2 = vec![
273 canned::message_start("rt2"),
274 canned::text_block_start(0),
275 canned::text_delta(0, "README starts with: # deepseek-tui"),
276 canned::block_stop(0),
277 canned::message_delta("end_turn", None),
278 canned::message_stop(),
279 ];
280 let mock = MockLlmClient::new(vec![turn1, turn2]);
281
282 // Round 1
283 let mut s1 = mock
284 .create_message_stream(make_request(vec![user_message("read README.md")]))
285 .await
286 .unwrap();
287
288 let mut tool_use_seen = false;
289 let mut json_seen = String::new();
290 while let Some(ev) = s1.next().await {
291 match ev.unwrap() {
292 StreamEvent::ContentBlockStart { content_block, .. } => {
293 use codewhale_models::ContentBlockStart;
294 if let ContentBlockStart::ToolUse { name, .. } = content_block {
295 assert_eq!(name, "read_file");
296 tool_use_seen = true;
297 }
298 }
299 StreamEvent::ContentBlockDelta {
300 delta: Delta::InputJsonDelta { partial_json },
301 ..
302 } => json_seen.push_str(&partial_json),
303 StreamEvent::MessageStop => break,
304 _ => {}
305 }
306 }
307 assert!(tool_use_seen);
308 let parsed: serde_json::Value =
309 serde_json::from_str(&json_seen).expect("valid JSON after concat");
310 assert_eq!(parsed["path"], "README.md");
311
312 // Round 2 — runtime sends back a tool_result and the mock replies with
313 // the final assistant text turn.
314 let req2 = make_request(vec![
315 user_message("read README.md"),
316 assistant_tool_call(
317 "call_x",
318 "read_file",
319 serde_json::json!({ "path": "README.md" }),
320 ),
321 tool_result_message("call_x", "# deepseek-tui\n..."),
322 ]);
323 let (text, stop) = drain_stream_text(&mock, req2).await;
324 assert!(text.contains("# deepseek-tui"));
325 assert_eq!(stop.as_deref(), Some("end_turn"));
326 }
327
328 // === 4. Multiple tool calls in one round (parallel ordering) ================
329
330 #[tokio::test]
331 async fn parallel_tool_calls_preserve_ordering_in_turn_payload() {
332 // Assistant returns two tool_calls in a single turn (indices 0 and 1).
333 // The runtime is free to execute them in parallel; this test asserts that
334 // the canonical event ordering survives a single-turn replay.
335 let turn = vec![
336 canned::message_start("p1"),
337 canned::tool_use_block_start(0, "call_one", "list_dir"),
338 canned::tool_input_delta(0, r#"{"path":"a"}"#),
339 canned::block_stop(0),
340 canned::tool_use_block_start(1, "call_two", "list_dir"),
341 canned::tool_input_delta(1, r#"{"path":"b"}"#),
342 canned::block_stop(1),
343 canned::message_delta("tool_use", None),
344 canned::message_stop(),
345 ];
346 let mock = MockLlmClient::new(vec![turn]);
347
348 let mut stream = mock
349 .create_message_stream(make_request(vec![user_message("list both")]))
350 .await
351 .unwrap();
352
353 let mut starts: Vec<(u32, String)> = Vec::new();
354 while let Some(ev) = stream.next().await {
355 if let StreamEvent::ContentBlockStart {
356 index,
357 content_block,
358 } = ev.unwrap()
359 {
360 use codewhale_models::ContentBlockStart;
361 if let ContentBlockStart::ToolUse { id, .. } = content_block {
362 starts.push((index, id));
363 }
364 }
365 }
366
367 assert_eq!(starts.len(), 2);
368 assert_eq!(starts[0], (0, "call_one".to_string()));
369 assert_eq!(starts[1], (1, "call_two".to_string()));
370 }
371
372 // === 5. Compaction-style non-streaming call =================================
373
374 #[tokio::test]
375 async fn compaction_non_streaming_returns_queued_message_response() {
376 use codewhale_models::MessageResponse;
377
378 let mock = MockLlmClient::new(vec![]);
379 mock.push_message_response(MessageResponse {
380 id: "compact_msg".to_string(),
381 r#type: "message".to_string(),
382 role: "assistant".to_string(),
383 content: vec![ContentBlock::Text {
384 text: "## Summary\n- Step 1\n- Step 2".to_string(),
385 cache_control: None,
386 }],
387 model: "deepseek-v4-pro".to_string(),
388 stop_reason: Some("end_turn".to_string()),
389 stop_sequence: None,
390 container: None,
391 usage: Usage::default(),
392 });
393
394 // The runtime's compaction path uses create_message (not stream).
395 let req = MessageRequest {
396 stream: Some(false),
397 ..make_request(vec![user_message("summarize")])
398 };
399 let resp = mock.create_message(req).await.unwrap();
400
401 let text = match &resp.content[0] {
402 ContentBlock::Text { text, .. } => text.clone(),
403 _ => panic!("expected text content"),
404 };
405 assert!(text.contains("Summary"));
406 assert_eq!(resp.id, "compact_msg");
407 assert_eq!(mock.call_count(), 1);
408 }
409
410 // === 6. Sub-agent style turn ================================================
411 //
412 // The next turn after an `agent` summary must re-verify the claimed
413 // side effect before reporting success.
414
415 #[tokio::test]
416 async fn v4_parent_reverifies_subagent_file_self_report_before_claiming_success() {
417 let tmp = tempfile::tempdir().expect("tempdir");
418 let missing = tmp.path().join("child-claimed-write.txt");
419 assert!(!missing.exists(), "fixture path must start missing");
420 let missing_path = missing.display().to_string();
421
422 let parent = MockLlmClient::new(vec![vec![
423 canned::message_start("parent_verify"),
424 canned::thinking_delta(0, "Verify the child's file-write self-report first."),
425 canned::tool_use_block_start(1, "verify_file", "read_file"),
426 canned::tool_input_delta(1, &serde_json::json!({ "path": &missing_path }).to_string()),
427 canned::block_stop(1),
428 canned::message_delta("tool_use", None),
429 canned::message_stop(),
430 ]])
431 .with_model("deepseek-v4-pro");
432 let tool_summary = format!(
433 "[sub-agent result summarized for parent context]\n\
434 Child results are self-reports; verify side effects with tools like read_file or list_dir before claiming success.\n\
435 - agent_filecheck (implementer) status=Completed\n result: Wrote {missing_path} successfully."
436 );
437
438 let mut stream = parent
439 .create_message_stream(make_request(vec![
440 user_message("Use a child to create the file, then report back."),
441 assistant_tool_call(
442 "agent_call",
443 "agent",
444 serde_json::json!({
445 "prompt": "Create the requested file and report the result.",
446 "role": "implementer"
447 }),
448 ),
449 tool_result_message("agent_call", &tool_summary),
450 ]))
451 .await
452 .unwrap();
453
454 let mut text_before_verification = String::new();
455 let mut tool_name = None;
456 let mut tool_input = String::new();
457 while let Some(ev) = stream.next().await {
458 match ev.unwrap() {
459 StreamEvent::ContentBlockStart { content_block, .. } => {
460 use codewhale_models::ContentBlockStart;
461 if let ContentBlockStart::ToolUse { name, .. } = content_block {
462 tool_name = Some(name);
463 }
464 }
465 StreamEvent::ContentBlockDelta { delta, .. } => match delta {
466 Delta::InputJsonDelta { partial_json } => tool_input.push_str(&partial_json),
467 Delta::TextDelta { text } => text_before_verification.push_str(&text),
468 _ => {}
469 },
470 StreamEvent::MessageStop => break,
471 _ => {}
472 }
473 }
474
475 assert_eq!(text_before_verification, "");
476 assert_eq!(tool_name.as_deref(), Some("read_file"));
477 let parsed: serde_json::Value = serde_json::from_str(&tool_input).expect("tool input JSON");
478 assert_eq!(parsed["path"], missing_path);
479 }
480
481 // === 7. Request capture observation =========================================
482 //
483 // The mock surfaces request captures BEFORE the response stream is opened, so
484 // trait-level tests can verify that captured requests are observable per-call
485 // rather than buffered across calls.
486
487 #[tokio::test]
488 async fn capacity_gate_can_observe_request_before_response_streams() {
489 let turn = vec![canned::simple_text_turn("ok")];
490 let mock = MockLlmClient::new(turn);
491
492 // Build a "near-limit" request — many user messages.
493 let mut messages = Vec::new();
494 for i in 0..200 {
495 messages.push(user_message(&format!("m{i}")));
496 }
497 let req = make_request(messages);
498
499 // BEFORE the runtime drains the stream, the mock has already captured
500 // the request. The capacity controller can inspect this and short-circuit
501 // the dispatch if the estimated token cost exceeds the soft cap.
502 let stream_future = mock.create_message_stream(req);
503 let mut stream = stream_future.await.unwrap();
504
505 assert_eq!(mock.captured_requests().len(), 1);
506 let captured = mock.last_request().unwrap();
507 assert_eq!(captured.messages.len(), 200);
508 // Verify the capacity gate could compute a "should defer" decision based
509 // on raw message count + payload size of the captured request.
510 let total_chars: usize = captured
511 .messages
512 .iter()
513 .flat_map(|m| m.content.iter())
514 .map(|b| match b {
515 ContentBlock::Text { text, .. } => text.len(),
516 _ => 0,
517 })
518 .sum();
519 assert!(
520 total_chars > 100,
521 "synthetic over-cap request should have non-trivial size"
522 );
523
524 // Drain to keep the mock state consistent.
525 while stream.next().await.is_some() {}
526 }
527
528 // === 8. Compaction defaults (#402 P0) ======================================
529
530 #[test]
531 fn compaction_config_defaults_are_enabled_for_session_survivability() {
532 // The production CompactionConfig is gated behind a `#[path = ...]` module
533 // that isn't wired here, but we can test the principle: the
534 // `should_compact` function and `CompactionConfig` live in the same crate.
535 // Re-import from the production module to verify the default.
536 //
537 // We test via the mock pathway: the non-streaming compaction call (test 5
538 // above) already exercises `create_message` with `stream: Some(false)`,
539 // which is the code path `compact_messages` uses. Combined with the
540 // capacity controller's `TargetedContextRefresh`, the enabled-by-default
541 // compaction config means long sessions auto-compact before hitting the
542 // context window limit.
543 //
544 // This test is a smoke check that the defaults compile and are correct.
545 // The production `CompactionConfig::default()` is exercised by
546 // `compaction::tests::should_compact_respects_enabled_flag` etc.
547 let config =
548 codewhale_models::compaction_threshold_for_model_at_percent("deepseek-v4-pro", 80.0);
549 // Verify the threshold is reasonable (> 0 and < context window).
550 assert!(config > 0, "compaction threshold must be positive");
551 assert!(config < 1_000_000, "compaction threshold must be below 1M");
552 }
553
553 lines RUST