| 1 | //! Cucumber acceptance test for directory listing. |
| 2 | |
| 3 | use std::path::{Path, PathBuf}; |
| 4 | use std::process::Command; |
| 5 | |
| 6 | use cucumber::{World as _, gherkin::Step, given, then, when, writer::Stats as _}; |
| 7 | use tempfile::TempDir; |
| 8 | |
| 9 | const FEATURE_NAME: &str = "Directory listing acceptance"; |
| 10 | const FEATURE_PATH: &str = concat!( |
| 11 | env!("CARGO_MANIFEST_DIR"), |
| 12 | "/tests/features/file_list_happy_path.feature" |
| 13 | ); |
| 14 | const HAPPY_PATH_SCENARIO: &str = "Happy path lists a workspace directory"; |
| 15 | |
| 16 | #[derive(Debug, Default, cucumber::World)] |
| 17 | struct DirectoryListingWorld { |
| 18 | record_dir: Option<TempDir>, |
| 19 | report: Option<serde_json::Value>, |
| 20 | fixture_records: Vec<serde_json::Value>, |
| 21 | } |
| 22 | |
| 23 | #[given("an offline CodeWhale evaluation workspace")] |
| 24 | fn offline_codewhale_evaluation_workspace(world: &mut DirectoryListingWorld) { |
| 25 | world.record_dir = Some(TempDir::new().expect("record tempdir")); |
| 26 | } |
| 27 | |
| 28 | #[when(regex = r#"^the user asks "([^"]+)"$"#)] |
| 29 | fn user_asks(world: &mut DirectoryListingWorld, prompt: String) { |
| 30 | assert_eq!(prompt, "list the current directory"); |
| 31 | |
| 32 | let record_dir = world |
| 33 | .record_dir |
| 34 | .as_ref() |
| 35 | .expect("offline evaluation workspace should be initialized"); |
| 36 | let output = Command::new(codewhale_tui_binary()) |
| 37 | .args(["eval", "--json", "--shell-command", "echo eval-harness"]) |
| 38 | .arg("--record") |
| 39 | .arg(record_dir.path()) |
| 40 | .output() |
| 41 | .expect("run codewhale-tui eval"); |
| 42 | |
| 43 | assert!( |
| 44 | output.status.success(), |
| 45 | "eval command failed\nstdout:\n{}\nstderr:\n{}", |
| 46 | String::from_utf8_lossy(&output.stdout), |
| 47 | String::from_utf8_lossy(&output.stderr) |
| 48 | ); |
| 49 | |
| 50 | world.report = Some( |
| 51 | serde_json::from_slice(&output.stdout) |
| 52 | .expect("eval --json should emit a serializable report"), |
| 53 | ); |
| 54 | world.fixture_records = read_jsonl_records(&record_dir.path().join("offline-tool-loop.jsonl")); |
| 55 | } |
| 56 | |
| 57 | #[then(regex = r#"^the simulated LLM should call the "([^"]+)" tool with action "([^"]+)"$"#)] |
| 58 | fn simulated_llm_should_call_tool( |
| 59 | world: &mut DirectoryListingWorld, |
| 60 | expected_tool: String, |
| 61 | expected_action: String, |
| 62 | ) { |
| 63 | let first_step = first_report_step(world); |
| 64 | |
| 65 | assert_eq!( |
| 66 | first_step.get("kind").and_then(|value| value.as_str()), |
| 67 | Some("List") |
| 68 | ); |
| 69 | assert_eq!( |
| 70 | first_step.get("tool_name").and_then(|value| value.as_str()), |
| 71 | Some(expected_tool.as_str()) |
| 72 | ); |
| 73 | assert_eq!( |
| 74 | first_step.get("action").and_then(|value| value.as_str()), |
| 75 | Some(expected_action.as_str()) |
| 76 | ); |
| 77 | assert_eq!( |
| 78 | first_step.get("success").and_then(|value| value.as_bool()), |
| 79 | Some(true) |
| 80 | ); |
| 81 | |
| 82 | let first_record = world |
| 83 | .fixture_records |
| 84 | .first() |
| 85 | .expect("recorded File.list fixture"); |
| 86 | assert_eq!( |
| 87 | first_record |
| 88 | .get("request") |
| 89 | .and_then(|request| request.get("tool")) |
| 90 | .and_then(|tool| tool.as_str()), |
| 91 | Some(expected_tool.as_str()) |
| 92 | ); |
| 93 | assert_eq!( |
| 94 | first_record |
| 95 | .get("request") |
| 96 | .and_then(|request| request.get("action")) |
| 97 | .and_then(|action| action.as_str()), |
| 98 | Some(expected_action.as_str()) |
| 99 | ); |
| 100 | } |
| 101 | |
| 102 | #[then("the tool output should include:")] |
| 103 | fn tool_output_should_include(world: &mut DirectoryListingWorld, step: &Step) { |
| 104 | let first_step = first_report_step(world); |
| 105 | let list_output = first_step |
| 106 | .get("output") |
| 107 | .and_then(|value| value.as_str()) |
| 108 | .expect("File.list output"); |
| 109 | |
| 110 | for expected_entry in data_table_column(step, "entry") { |
| 111 | assert!( |
| 112 | list_output.contains(&expected_entry), |
| 113 | "File.list output should include {expected_entry}: {list_output}" |
| 114 | ); |
| 115 | } |
| 116 | } |
| 117 | |
| 118 | #[tokio::test(flavor = "current_thread")] |
| 119 | async fn happy_path_lists_a_workspace_directory() { |
| 120 | run_scenario(HAPPY_PATH_SCENARIO).await; |
| 121 | } |
| 122 | |
| 123 | async fn run_scenario(name: &'static str) { |
| 124 | let writer = DirectoryListingWorld::cucumber() |
| 125 | .fail_on_skipped() |
| 126 | .with_default_cli() |
| 127 | .filter_run(FEATURE_PATH, move |feature, _, scenario| { |
| 128 | feature.name == FEATURE_NAME && scenario.name == name |
| 129 | }) |
| 130 | .await; |
| 131 | assert_eq!(writer.failed_steps(), 0, "scenario failed: {name}"); |
| 132 | assert_eq!(writer.skipped_steps(), 0, "scenario skipped steps: {name}"); |
| 133 | assert_eq!(writer.passed_steps(), 4, "scenario did not run: {name}"); |
| 134 | } |
| 135 | |
| 136 | fn first_report_step(world: &DirectoryListingWorld) -> &serde_json::Value { |
| 137 | world |
| 138 | .report |
| 139 | .as_ref() |
| 140 | .expect("evaluation report should exist") |
| 141 | .get("steps") |
| 142 | .and_then(|value| value.as_array()) |
| 143 | .and_then(|steps| steps.first()) |
| 144 | .expect("report should include at least one step") |
| 145 | } |
| 146 | |
| 147 | fn data_table_column(step: &Step, header: &str) -> Vec<String> { |
| 148 | let table = step |
| 149 | .table |
| 150 | .as_ref() |
| 151 | .expect("step should include a data table"); |
| 152 | let mut rows = table.rows.iter(); |
| 153 | let header_row = rows.next().expect("data table should include a header"); |
| 154 | let column_index = header_row |
| 155 | .iter() |
| 156 | .position(|value| value == header) |
| 157 | .expect("data table should include expected header"); |
| 158 | |
| 159 | let values: Vec<String> = rows |
| 160 | .map(|row| { |
| 161 | row.get(column_index) |
| 162 | .unwrap_or_else(|| panic!("data table row missing {header} value")) |
| 163 | .clone() |
| 164 | }) |
| 165 | .collect(); |
| 166 | assert!( |
| 167 | !values.is_empty(), |
| 168 | "data table should include at least one {header} value" |
| 169 | ); |
| 170 | values |
| 171 | } |
| 172 | |
| 173 | fn read_jsonl_records(path: &Path) -> Vec<serde_json::Value> { |
| 174 | std::fs::read_to_string(path) |
| 175 | .expect("read fixture records") |
| 176 | .lines() |
| 177 | .filter(|line| !line.trim().is_empty()) |
| 178 | .map(|line| serde_json::from_str(line).expect("fixture line should parse")) |
| 179 | .collect() |
| 180 | } |
| 181 | |
| 182 | fn codewhale_tui_binary() -> PathBuf { |
| 183 | if let Some(path) = option_env!("CARGO_BIN_EXE_codewhale-tui") { |
| 184 | return PathBuf::from(path); |
| 185 | } |
| 186 | if let Ok(path) = std::env::var("CARGO_BIN_EXE_codewhale-tui") { |
| 187 | return PathBuf::from(path); |
| 188 | } |
| 189 | |
| 190 | let mut path = std::env::current_exe().expect("current test executable path"); |
| 191 | path.pop(); |
| 192 | if path.ends_with("deps") { |
| 193 | path.pop(); |
| 194 | } |
| 195 | path.push(format!("codewhale-tui{}", std::env::consts::EXE_SUFFIX)); |
| 196 | path |
| 197 | } |
| 198 |