| 1 | //! Integration tests for the offline evaluation harness. |
| 2 | |
| 3 | use std::fs; |
| 4 | |
| 5 | #[path = "../src/eval.rs"] |
| 6 | mod eval; |
| 7 | |
| 8 | use eval::{EvalHarness, EvalHarnessConfig, ScenarioStepKind}; |
| 9 | use tempfile::tempdir; |
| 10 | |
| 11 | #[test] |
| 12 | fn runs_offline_tool_loop_successfully() { |
| 13 | let harness = EvalHarness::default(); |
| 14 | let run = harness.run().expect("eval harness run should succeed"); |
| 15 | assert_eq!( |
| 16 | ScenarioStepKind::parse("patch"), |
| 17 | Some(ScenarioStepKind::ApplyPatch) |
| 18 | ); |
| 19 | |
| 20 | assert!(run.metrics.success, "expected success metrics: {run:#?}"); |
| 21 | assert_eq!(run.metrics.tool_errors, 0); |
| 22 | assert_eq!(run.metrics.steps, 6); |
| 23 | assert!(run.metrics.duration.as_millis() > 0); |
| 24 | assert!(!run.scenario_name.is_empty()); |
| 25 | assert!(run.workspace_summary.file_count >= 3); |
| 26 | |
| 27 | for kind in [ |
| 28 | ScenarioStepKind::List, |
| 29 | ScenarioStepKind::Read, |
| 30 | ScenarioStepKind::Search, |
| 31 | ScenarioStepKind::Edit, |
| 32 | ScenarioStepKind::ApplyPatch, |
| 33 | ScenarioStepKind::ExecShell, |
| 34 | ] { |
| 35 | let stats = run |
| 36 | .metrics |
| 37 | .per_tool |
| 38 | .get(&kind) |
| 39 | .expect("missing per-tool stats"); |
| 40 | assert_eq!(stats.invocations, 1, "unexpected invocations for {kind:?}"); |
| 41 | assert_eq!(stats.errors, 0, "unexpected errors for {kind:?}"); |
| 42 | assert!(stats.total_duration.as_nanos() > 0); |
| 43 | } |
| 44 | |
| 45 | let notes_path = run.workspace_root().join("notes.txt"); |
| 46 | let notes = fs::read_to_string(¬es_path).expect("notes.txt should exist"); |
| 47 | assert!(notes.contains("edited = true")); |
| 48 | assert!(notes.contains("todo: offline metrics (patched)")); |
| 49 | |
| 50 | let report = run.to_report(); |
| 51 | assert_eq!(report.metrics.success, run.metrics.success); |
| 52 | } |
| 53 | |
| 54 | #[test] |
| 55 | fn records_tool_errors_when_step_fails() { |
| 56 | let config = EvalHarnessConfig { |
| 57 | fail_step: Some(ScenarioStepKind::ApplyPatch), |
| 58 | ..EvalHarnessConfig::default() |
| 59 | }; |
| 60 | let harness = EvalHarness::new(config); |
| 61 | |
| 62 | let run = harness |
| 63 | .run() |
| 64 | .expect("eval harness should return metrics even when a step fails"); |
| 65 | |
| 66 | assert!(!run.metrics.success); |
| 67 | assert!(run.metrics.tool_errors >= 1); |
| 68 | |
| 69 | let patch_stats = run |
| 70 | .metrics |
| 71 | .per_tool |
| 72 | .get(&ScenarioStepKind::ApplyPatch) |
| 73 | .expect("missing apply_patch stats"); |
| 74 | assert_eq!(patch_stats.invocations, 1); |
| 75 | assert_eq!(patch_stats.errors, 1); |
| 76 | |
| 77 | let patch_step = run |
| 78 | .steps |
| 79 | .iter() |
| 80 | .find(|step| step.kind == ScenarioStepKind::ApplyPatch) |
| 81 | .expect("missing apply_patch step"); |
| 82 | assert!(!patch_step.success); |
| 83 | assert!(patch_step.error.as_deref().is_some_and(|e| !e.is_empty())); |
| 84 | } |
| 85 | |
| 86 | #[test] |
| 87 | fn validation_can_fail_without_tool_errors() { |
| 88 | let config = EvalHarnessConfig { |
| 89 | shell_expect_token: "definitely-not-in-output".to_string(), |
| 90 | ..EvalHarnessConfig::default() |
| 91 | }; |
| 92 | let harness = EvalHarness::new(config); |
| 93 | |
| 94 | let run = harness.run().expect("eval harness run should complete"); |
| 95 | |
| 96 | assert_eq!(run.metrics.tool_errors, 0); |
| 97 | assert!( |
| 98 | !run.metrics.success, |
| 99 | "validation should fail due to shell token" |
| 100 | ); |
| 101 | } |
| 102 | |
| 103 | #[test] |
| 104 | fn record_flag_writes_one_jsonl_line_per_step() { |
| 105 | let dir = tempdir().expect("tempdir"); |
| 106 | let config = EvalHarnessConfig { |
| 107 | record_dir: Some(dir.path().to_path_buf()), |
| 108 | ..EvalHarnessConfig::default() |
| 109 | }; |
| 110 | let harness = EvalHarness::new(config); |
| 111 | let run = harness.run().expect("eval harness run should succeed"); |
| 112 | |
| 113 | let scenario_file = dir.path().join("offline-tool-loop.jsonl"); |
| 114 | assert!( |
| 115 | scenario_file.exists(), |
| 116 | "record_dir should contain {}", |
| 117 | scenario_file |
| 118 | .file_name() |
| 119 | .map(|n| n.to_string_lossy().into_owned()) |
| 120 | .unwrap_or_default(), |
| 121 | ); |
| 122 | |
| 123 | let contents = fs::read_to_string(&scenario_file).expect("read jsonl"); |
| 124 | let lines: Vec<&str> = contents.lines().filter(|l| !l.trim().is_empty()).collect(); |
| 125 | assert_eq!( |
| 126 | lines.len(), |
| 127 | run.metrics.steps, |
| 128 | "one JSONL line per step expected" |
| 129 | ); |
| 130 | |
| 131 | // Each line is a self-contained JSON object with the documented schema. |
| 132 | for line in lines { |
| 133 | let parsed: serde_json::Value = |
| 134 | serde_json::from_str(line).expect("each fixture line is valid JSON"); |
| 135 | assert!(parsed.get("request").is_some(), "missing request"); |
| 136 | let events = parsed |
| 137 | .get("response_events") |
| 138 | .and_then(|v| v.as_array()) |
| 139 | .expect("response_events must be an array"); |
| 140 | assert!(!events.is_empty(), "every fixture must have ≥1 event"); |
| 141 | } |
| 142 | } |
| 143 |