| 1 | //! Integration tests for the offline evaluation harness. |
| 2 | |
| 3 | use std::fs; |
| 4 | |
| 5 | use tempfile::tempdir; |
| 6 | |
| 7 | use crate::eval::{EvalHarness, EvalHarnessConfig, FixtureRecord, ScenarioStepKind}; |
| 8 | |
| 9 | const HAPPY_PATH_TOOL_LOOP: [ScenarioStepKind; 6] = [ |
| 10 | ScenarioStepKind::List, |
| 11 | ScenarioStepKind::Read, |
| 12 | ScenarioStepKind::Search, |
| 13 | ScenarioStepKind::Edit, |
| 14 | ScenarioStepKind::ApplyPatch, |
| 15 | ScenarioStepKind::Bash, |
| 16 | ]; |
| 17 | |
| 18 | #[test] |
| 19 | fn runs_offline_tool_loop_successfully() { |
| 20 | let harness = EvalHarness::default(); |
| 21 | let run = harness.run().expect("eval harness run should succeed"); |
| 22 | assert_eq!( |
| 23 | ScenarioStepKind::parse("patch"), |
| 24 | Some(ScenarioStepKind::ApplyPatch) |
| 25 | ); |
| 26 | |
| 27 | assert!(run.metrics.success, "expected success metrics: {run:#?}"); |
| 28 | assert_eq!(run.metrics.tool_errors, 0); |
| 29 | assert_eq!(run.metrics.steps, 6); |
| 30 | // Nanosecond granularity: the offline loop performs real fs/tempdir work, |
| 31 | // so a sub-millisecond total is legitimate; `as_millis() > 0` flaked under |
| 32 | // saturated test runs (pre-existing timing flake, fixed for FEAT-018). |
| 33 | assert!(run.metrics.duration.as_nanos() > 0); |
| 34 | assert!(!run.scenario_name.is_empty()); |
| 35 | assert!(run.workspace_summary.file_count >= 3); |
| 36 | |
| 37 | for kind in HAPPY_PATH_TOOL_LOOP { |
| 38 | let stats = run |
| 39 | .metrics |
| 40 | .per_tool |
| 41 | .get(&kind) |
| 42 | .expect("missing per-tool stats"); |
| 43 | assert_eq!(stats.invocations, 1, "unexpected invocations for {kind:?}"); |
| 44 | assert_eq!(stats.errors, 0, "unexpected errors for {kind:?}"); |
| 45 | assert!(stats.total_duration.as_nanos() > 0); |
| 46 | } |
| 47 | |
| 48 | let notes_path = run.workspace_root().join("notes.txt"); |
| 49 | let notes = fs::read_to_string(¬es_path).expect("notes.txt should exist"); |
| 50 | assert!(notes.contains("edited = true")); |
| 51 | assert!(notes.contains("todo: offline metrics (patched)")); |
| 52 | |
| 53 | let report = run.to_report(); |
| 54 | assert_eq!(report.metrics.success, run.metrics.success); |
| 55 | } |
| 56 | |
| 57 | #[test] |
| 58 | fn acceptance_happy_path_records_simulated_llm_tool_plan() { |
| 59 | let record_dir = tempdir().expect("tempdir"); |
| 60 | let scenario_name = "issue-2791-happy-path-tool-loop"; |
| 61 | let config = EvalHarnessConfig { |
| 62 | scenario_name: scenario_name.to_string(), |
| 63 | record_dir: Some(record_dir.path().to_path_buf()), |
| 64 | ..EvalHarnessConfig::default() |
| 65 | }; |
| 66 | let harness = EvalHarness::new(config); |
| 67 | |
| 68 | let run = harness.run().expect("happy-path acceptance run"); |
| 69 | |
| 70 | assert!(run.metrics.success, "expected success metrics: {run:#?}"); |
| 71 | assert_eq!(run.metrics.tool_errors, 0); |
| 72 | assert_eq!(run.metrics.steps, HAPPY_PATH_TOOL_LOOP.len()); |
| 73 | |
| 74 | let actual_tool_names: Vec<&str> = run.steps.iter().map(|step| step.tool_name).collect(); |
| 75 | let expected_tool_names: Vec<&str> = HAPPY_PATH_TOOL_LOOP |
| 76 | .iter() |
| 77 | .map(|kind| kind.tool_name()) |
| 78 | .collect(); |
| 79 | assert_eq!(actual_tool_names, expected_tool_names); |
| 80 | |
| 81 | let scenario_file = record_dir.path().join(format!("{scenario_name}.jsonl")); |
| 82 | let records = read_fixture_records(&scenario_file); |
| 83 | assert_eq!(records.len(), HAPPY_PATH_TOOL_LOOP.len()); |
| 84 | |
| 85 | for (record, kind) in records.iter().zip(HAPPY_PATH_TOOL_LOOP) { |
| 86 | assert_eq!( |
| 87 | record.request.get("tool").and_then(|value| value.as_str()), |
| 88 | Some(kind.tool_name()) |
| 89 | ); |
| 90 | assert_eq!( |
| 91 | record |
| 92 | .request |
| 93 | .get("action") |
| 94 | .and_then(|value| value.as_str()), |
| 95 | kind.action() |
| 96 | ); |
| 97 | |
| 98 | let expected_kind = format!("{kind:?}"); |
| 99 | assert_eq!( |
| 100 | record.request.get("kind").and_then(|value| value.as_str()), |
| 101 | Some(expected_kind.as_str()) |
| 102 | ); |
| 103 | |
| 104 | let event = record |
| 105 | .response_events |
| 106 | .first() |
| 107 | .expect("simulated LLM fixture should include a response event"); |
| 108 | assert_eq!( |
| 109 | event.get("type").and_then(|value| value.as_str()), |
| 110 | Some("ok") |
| 111 | ); |
| 112 | assert!( |
| 113 | event |
| 114 | .get("output") |
| 115 | .and_then(|value| value.as_str()) |
| 116 | .is_some_and(|output| !output.is_empty()), |
| 117 | "fixture event should include non-empty tool output" |
| 118 | ); |
| 119 | } |
| 120 | |
| 121 | let notes_path = run.workspace_root().join("notes.txt"); |
| 122 | let notes = fs::read_to_string(¬es_path).expect("notes.txt should exist"); |
| 123 | assert!(notes.contains("edited = true")); |
| 124 | assert!(notes.contains("todo: offline metrics (patched)")); |
| 125 | } |
| 126 | |
| 127 | fn read_fixture_records(path: &std::path::Path) -> Vec<FixtureRecord> { |
| 128 | fs::read_to_string(path) |
| 129 | .expect("read fixture records") |
| 130 | .lines() |
| 131 | .filter(|line| !line.trim().is_empty()) |
| 132 | .map(|line| serde_json::from_str(line).expect("fixture line should parse")) |
| 133 | .collect() |
| 134 | } |
| 135 | |
| 136 | #[test] |
| 137 | fn records_tool_errors_when_step_fails() { |
| 138 | let config = EvalHarnessConfig { |
| 139 | fail_step: Some(ScenarioStepKind::ApplyPatch), |
| 140 | ..EvalHarnessConfig::default() |
| 141 | }; |
| 142 | let harness = EvalHarness::new(config); |
| 143 | |
| 144 | let run = harness |
| 145 | .run() |
| 146 | .expect("eval harness should return metrics even when a step fails"); |
| 147 | |
| 148 | assert!(!run.metrics.success); |
| 149 | assert!(run.metrics.tool_errors >= 1); |
| 150 | |
| 151 | let patch_stats = run |
| 152 | .metrics |
| 153 | .per_tool |
| 154 | .get(&ScenarioStepKind::ApplyPatch) |
| 155 | .expect("missing apply_patch stats"); |
| 156 | assert_eq!(patch_stats.invocations, 1); |
| 157 | assert_eq!(patch_stats.errors, 1); |
| 158 | |
| 159 | let patch_step = run |
| 160 | .steps |
| 161 | .iter() |
| 162 | .find(|step| step.kind == ScenarioStepKind::ApplyPatch) |
| 163 | .expect("missing apply_patch step"); |
| 164 | assert!(!patch_step.success); |
| 165 | assert!(patch_step.error.as_deref().is_some_and(|e| !e.is_empty())); |
| 166 | } |
| 167 | |
| 168 | #[test] |
| 169 | fn validation_can_fail_without_tool_errors() { |
| 170 | let config = EvalHarnessConfig { |
| 171 | shell_expect_token: "definitely-not-in-output".to_string(), |
| 172 | ..EvalHarnessConfig::default() |
| 173 | }; |
| 174 | let harness = EvalHarness::new(config); |
| 175 | |
| 176 | let run = harness.run().expect("eval harness run should complete"); |
| 177 | |
| 178 | assert_eq!(run.metrics.tool_errors, 0); |
| 179 | assert!( |
| 180 | !run.metrics.success, |
| 181 | "validation should fail due to shell token" |
| 182 | ); |
| 183 | } |
| 184 | |
| 185 | #[test] |
| 186 | fn record_flag_writes_one_jsonl_line_per_step() { |
| 187 | let dir = tempdir().expect("tempdir"); |
| 188 | let config = EvalHarnessConfig { |
| 189 | record_dir: Some(dir.path().to_path_buf()), |
| 190 | ..EvalHarnessConfig::default() |
| 191 | }; |
| 192 | let harness = EvalHarness::new(config); |
| 193 | let run = harness.run().expect("eval harness run should succeed"); |
| 194 | |
| 195 | let scenario_file = dir.path().join("offline-tool-loop.jsonl"); |
| 196 | assert!( |
| 197 | scenario_file.exists(), |
| 198 | "record_dir should contain {}", |
| 199 | scenario_file |
| 200 | .file_name() |
| 201 | .map(|n| n.to_string_lossy().into_owned()) |
| 202 | .unwrap_or_default(), |
| 203 | ); |
| 204 | |
| 205 | let contents = fs::read_to_string(&scenario_file).expect("read jsonl"); |
| 206 | let lines: Vec<&str> = contents.lines().filter(|l| !l.trim().is_empty()).collect(); |
| 207 | assert_eq!( |
| 208 | lines.len(), |
| 209 | run.metrics.steps, |
| 210 | "one JSONL line per step expected" |
| 211 | ); |
| 212 | |
| 213 | // Each line is a self-contained JSON object with the documented schema. |
| 214 | for line in lines { |
| 215 | let parsed: serde_json::Value = |
| 216 | serde_json::from_str(line).expect("each fixture line is valid JSON"); |
| 217 | assert!(parsed.get("request").is_some(), "missing request"); |
| 218 | let events = parsed |
| 219 | .get("response_events") |
| 220 | .and_then(|v| v.as_array()) |
| 221 | .expect("response_events must be an array"); |
| 222 | assert!(!events.is_empty(), "every fixture must have ≥1 event"); |
| 223 | } |
| 224 | } |
| 225 |