| 1 | //! End-to-end tests for the Workflow JS runtime against a fake driver. |
| 2 | |
| 3 | use std::sync::Arc; |
| 4 | use std::time::Duration; |
| 5 | |
| 6 | use codewhale_workflow_js::testing::{FakeDriver, FakeReply}; |
| 7 | use codewhale_workflow_js::{ |
| 8 | ProgressEvent, WORKFLOW_LIFETIME_CAP, WorkflowJsError, WorkflowRunCancel, WorkflowVm, |
| 9 | }; |
| 10 | use serde_json::json; |
| 11 | |
| 12 | async fn run( |
| 13 | driver: &Arc<FakeDriver>, |
| 14 | source: &str, |
| 15 | args: serde_json::Value, |
| 16 | ) -> Result<serde_json::Value, WorkflowJsError> { |
| 17 | WorkflowVm::new() |
| 18 | .run_script( |
| 19 | source, |
| 20 | args, |
| 21 | driver.clone() as Arc<dyn codewhale_workflow_js::WorkflowDriver>, |
| 22 | ) |
| 23 | .await |
| 24 | } |
| 25 | |
| 26 | fn script_message(result: Result<serde_json::Value, WorkflowJsError>) -> String { |
| 27 | match result { |
| 28 | Err(WorkflowJsError::Script(message)) => message, |
| 29 | other => panic!("expected script error, got {other:?}"), |
| 30 | } |
| 31 | } |
| 32 | |
| 33 | #[tokio::test] |
| 34 | async fn plain_return_value_round_trips() { |
| 35 | let driver = Arc::new(FakeDriver::new()); |
| 36 | let value = run(&driver, "return 1 + 1;", json!(null)).await.unwrap(); |
| 37 | assert_eq!(value, json!(2)); |
| 38 | } |
| 39 | |
| 40 | #[tokio::test] |
| 41 | async fn undefined_return_becomes_null() { |
| 42 | let driver = Arc::new(FakeDriver::new()); |
| 43 | let value = run(&driver, "const x = 1;", json!(null)).await.unwrap(); |
| 44 | assert_eq!(value, json!(null)); |
| 45 | } |
| 46 | |
| 47 | #[tokio::test] |
| 48 | async fn args_global_is_the_invocation_input() { |
| 49 | let driver = Arc::new(FakeDriver::new()); |
| 50 | let value = run( |
| 51 | &driver, |
| 52 | "return { sum: args.x + 1, tag: args.tags[0] };", |
| 53 | json!({"x": 41, "tags": ["release"]}), |
| 54 | ) |
| 55 | .await |
| 56 | .unwrap(); |
| 57 | assert_eq!(value, json!({"sum": 42, "tag": "release"})); |
| 58 | } |
| 59 | |
| 60 | #[tokio::test] |
| 61 | async fn checked_in_best_of_n_search_recipe_runs_with_structured_receipts() { |
| 62 | let driver = Arc::new(FakeDriver::new()); |
| 63 | for index in 1..=2 { |
| 64 | driver.on( |
| 65 | // Rules match the driver-visible `TaskRequest.description`, which |
| 66 | // is the full instruction text (the VM's `prompt` alias wins over |
| 67 | // a short label). Match the unique per-candidate suffix line. |
| 68 | &format!("candidate_id=cand_{index:03} of 2."), |
| 69 | FakeReply::Complete( |
| 70 | json!({ |
| 71 | "candidate_id": format!("cand_{index:03}"), |
| 72 | "hypothesis": "bounded fixture", |
| 73 | "modified_paths": ["src/lib.rs"], |
| 74 | "commands_run": ["cargo test --locked"], |
| 75 | "self_verdict": "pass", |
| 76 | "known_risks": [], |
| 77 | "artifact_refs": [format!("patch:cand_{index:03}")] |
| 78 | }) |
| 79 | .to_string(), |
| 80 | ), |
| 81 | ); |
| 82 | } |
| 83 | driver.on( |
| 84 | "read-only tournament judge", |
| 85 | FakeReply::Complete( |
| 86 | json!({ |
| 87 | "winner_id": "cand_001", |
| 88 | "ranking": ["cand_001", "cand_002"], |
| 89 | "verification_required": true, |
| 90 | "reasons": ["fixture score"] |
| 91 | }) |
| 92 | .to_string(), |
| 93 | ), |
| 94 | ); |
| 95 | |
| 96 | let value = run( |
| 97 | &driver, |
| 98 | include_str!("../../../workflows/operate_best_of_n.workflow.js"), |
| 99 | json!({ |
| 100 | "brief": "Implement the fixture", |
| 101 | "strategy": "search", |
| 102 | "n": 2, |
| 103 | "writeRoots": ["src"], |
| 104 | "model": "deepseek-v4-flash", |
| 105 | "thinking": "max" |
| 106 | }), |
| 107 | ) |
| 108 | .await |
| 109 | .expect("checked-in search recipe should execute"); |
| 110 | |
| 111 | assert_eq!(value["scenario"], "operate-search"); |
| 112 | assert_eq!(value["review"]["winner_id"], "cand_001"); |
| 113 | assert_eq!(driver.spawn_count(), 3); |
| 114 | let requests = driver.requests(); |
| 115 | assert_eq!(requests[0].model.as_deref(), Some("deepseek-v4-flash")); |
| 116 | assert_eq!(requests[0].thinking.as_deref(), Some("max")); |
| 117 | assert_eq!(requests[0].write_roots, ["src"]); |
| 118 | assert_eq!(requests[2].write_authority.as_deref(), Some("read_only")); |
| 119 | // Regression: the driver-visible description is the full instruction text, |
| 120 | // so reply rules must target text that actually reaches the driver. If a |
| 121 | // future recipe reintroduces a separate short `description` next to a long |
| 122 | // `prompt`, these needles stop matching, the FakeDriver falls back to its |
| 123 | // non-JSON "done:..." reply, and the structured receipts fail loudly. |
| 124 | assert!( |
| 125 | requests[0] |
| 126 | .description |
| 127 | .starts_with("You are one independent candidate") |
| 128 | ); |
| 129 | assert!( |
| 130 | requests[0] |
| 131 | .description |
| 132 | .contains("CANDIDATE-SPECIFIC INSTRUCTION: candidate_id=cand_001 of 2.") |
| 133 | ); |
| 134 | assert!( |
| 135 | requests[2] |
| 136 | .description |
| 137 | .starts_with("You are the read-only tournament judge") |
| 138 | ); |
| 139 | } |
| 140 | |
| 141 | #[tokio::test] |
| 142 | async fn task_prompt_wins_over_description_as_driver_visible_text() { |
| 143 | let driver = Arc::new(FakeDriver::new()); |
| 144 | let value = run( |
| 145 | &driver, |
| 146 | r#" |
| 147 | return await task({ |
| 148 | description: "short progress label", |
| 149 | prompt: "the real instruction", |
| 150 | }); |
| 151 | "#, |
| 152 | json!(null), |
| 153 | ) |
| 154 | .await |
| 155 | .unwrap(); |
| 156 | |
| 157 | // No rules were registered, so the FakeDriver fallback echoes the |
| 158 | // driver-visible description. The reply text proves the driver received |
| 159 | // the prompt, not the short label. |
| 160 | assert_eq!(value, json!("done:the real instruction")); |
| 161 | let requests = driver.requests(); |
| 162 | assert_eq!(requests.len(), 1); |
| 163 | assert_eq!(requests[0].description, "the real instruction"); |
| 164 | assert_ne!(requests[0].description, "short progress label"); |
| 165 | } |
| 166 | |
| 167 | #[tokio::test] |
| 168 | async fn task_round_trip_carries_all_options_and_normalizes_profile() { |
| 169 | let driver = Arc::new(FakeDriver::new()); |
| 170 | let value = run( |
| 171 | &driver, |
| 172 | r#" |
| 173 | return await task({ |
| 174 | description: "implement the bounded change", |
| 175 | subagentType: "implementer", |
| 176 | profile: " ALpha-1 ", |
| 177 | model: "deepseek-chat", |
| 178 | modelStrength: "faster", |
| 179 | thinking: "low", |
| 180 | cwd: "repo-a", |
| 181 | worktree: true, |
| 182 | writeAuthority: "worktree_write", |
| 183 | writeRoots: ["crates/tui/src"], |
| 184 | exactFiles: ["Cargo.toml"], |
| 185 | coordinationContracts: ["public-api"], |
| 186 | dependencies: ["issue-4619"], |
| 187 | acceptance: ["locked tests pass"], |
| 188 | allowedTools: ["read", "grep"], |
| 189 | maxDepth: 2, |
| 190 | tokenBudget: 5000, |
| 191 | maxSteps: 4, |
| 192 | wallTimeSecs: 90, |
| 193 | label: "L1", |
| 194 | phase: "P1", |
| 195 | }); |
| 196 | "#, |
| 197 | json!(null), |
| 198 | ) |
| 199 | .await |
| 200 | .unwrap(); |
| 201 | assert_eq!(value, json!("done:implement the bounded change")); |
| 202 | |
| 203 | let requests = driver.requests(); |
| 204 | assert_eq!(requests.len(), 1); |
| 205 | let request = &requests[0]; |
| 206 | assert_eq!(request.description, "implement the bounded change"); |
| 207 | assert_eq!(request.subagent_type.as_deref(), Some("implementer")); |
| 208 | assert_eq!(request.profile.as_deref(), Some("alpha-1")); |
| 209 | assert_eq!(request.model.as_deref(), Some("deepseek-chat")); |
| 210 | assert_eq!(request.model_strength.as_deref(), Some("faster")); |
| 211 | assert_eq!(request.thinking.as_deref(), Some("low")); |
| 212 | assert_eq!(request.cwd.as_deref(), Some("repo-a")); |
| 213 | assert!(request.worktree); |
| 214 | assert_eq!(request.write_authority.as_deref(), Some("worktree_write")); |
| 215 | assert_eq!(request.write_roots, ["crates/tui/src"]); |
| 216 | assert_eq!(request.exact_files, ["Cargo.toml"]); |
| 217 | assert_eq!(request.coordination_contracts, ["public-api"]); |
| 218 | assert_eq!(request.dependencies, ["issue-4619"]); |
| 219 | assert_eq!(request.acceptance, ["locked tests pass"]); |
| 220 | assert_eq!( |
| 221 | request.allowed_tools.as_deref(), |
| 222 | Some(["read".to_string(), "grep".to_string()].as_slice()) |
| 223 | ); |
| 224 | assert_eq!(request.max_depth, Some(2)); |
| 225 | assert_eq!(request.token_budget, Some(5000)); |
| 226 | assert_eq!(request.max_steps, Some(4)); |
| 227 | assert_eq!(request.wall_time_secs, Some(90)); |
| 228 | assert_eq!(request.response_schema, None); |
| 229 | assert_eq!(request.label.as_deref(), Some("L1")); |
| 230 | assert_eq!(request.phase.as_deref(), Some("P1")); |
| 231 | } |
| 232 | |
| 233 | #[tokio::test] |
| 234 | async fn task_write_authority_requires_bounded_coordination_scope() { |
| 235 | let driver = Arc::new(FakeDriver::new()); |
| 236 | let error = run( |
| 237 | &driver, |
| 238 | r#" |
| 239 | return await task({ |
| 240 | prompt: "edit without a claim", |
| 241 | type: "implementer", |
| 242 | writeAuthority: "workspace_write", |
| 243 | }); |
| 244 | "#, |
| 245 | json!(null), |
| 246 | ) |
| 247 | .await |
| 248 | .expect_err("unscoped Workflow writer must fail before driver dispatch") |
| 249 | .to_string(); |
| 250 | assert!(error.contains("requires writeRoots"), "{error}"); |
| 251 | assert!(driver.requests().is_empty()); |
| 252 | } |
| 253 | |
| 254 | #[tokio::test] |
| 255 | async fn task_coordination_lists_deduplicate_with_hard_count_bounds() { |
| 256 | let driver = Arc::new(FakeDriver::new()); |
| 257 | run( |
| 258 | &driver, |
| 259 | r#" |
| 260 | return await task({ |
| 261 | prompt: "bounded edit", |
| 262 | type: "implementer", |
| 263 | writeAuthority: "workspace_write", |
| 264 | exactFiles: ["src/a.rs", "src/a.rs"], |
| 265 | dependencies: ["A", "A"], |
| 266 | acceptance: ["tests pass", "tests pass"], |
| 267 | }); |
| 268 | "#, |
| 269 | json!(null), |
| 270 | ) |
| 271 | .await |
| 272 | .expect("bounded unique coordination values"); |
| 273 | let request = driver.requests().pop().expect("request"); |
| 274 | assert_eq!(request.exact_files, ["src/a.rs"]); |
| 275 | assert_eq!(request.dependencies, ["A"]); |
| 276 | assert_eq!(request.acceptance, ["tests pass"]); |
| 277 | } |
| 278 | |
| 279 | #[tokio::test] |
| 280 | async fn task_write_paths_normalize_and_reject_escape_spellings() { |
| 281 | let driver = Arc::new(FakeDriver::new()); |
| 282 | run( |
| 283 | &driver, |
| 284 | r#"return await task({ |
| 285 | prompt: "bounded edit", |
| 286 | type: "implementer", |
| 287 | writeRoots: ["./src//", "src"], |
| 288 | exactFiles: ["src\\lib.rs"] |
| 289 | });"#, |
| 290 | json!(null), |
| 291 | ) |
| 292 | .await |
| 293 | .expect("normalized repo-relative paths"); |
| 294 | let request = driver.requests().pop().expect("request"); |
| 295 | assert_eq!(request.write_roots, ["src"]); |
| 296 | assert_eq!(request.exact_files, ["src/lib.rs"]); |
| 297 | |
| 298 | for path in [ |
| 299 | "../outside", |
| 300 | "/tmp/outside", |
| 301 | "C:\\outside", |
| 302 | "src/../../outside", |
| 303 | ] { |
| 304 | let driver = Arc::new(FakeDriver::new()); |
| 305 | let source = format!( |
| 306 | "return await task({{ prompt: 'escape', type: 'implementer', writeRoots: [{}] }});", |
| 307 | serde_json::to_string(path).expect("path json") |
| 308 | ); |
| 309 | let message = script_message(run(&driver, &source, json!(null)).await); |
| 310 | assert!( |
| 311 | message.contains("repo-relative") || message.contains("traversal"), |
| 312 | "{path}: {message}" |
| 313 | ); |
| 314 | assert!(driver.requests().is_empty()); |
| 315 | } |
| 316 | } |
| 317 | |
| 318 | #[tokio::test] |
| 319 | async fn task_explicit_write_roles_fail_closed_without_scope_and_reject_write_escalation() { |
| 320 | for source in [ |
| 321 | r#"return await task({prompt: "no scope", type: "implementer"});"#, |
| 322 | r#"return await task({prompt: "no scope", type: "builder"});"#, |
| 323 | r#"return await task({prompt: "no scope", type: "general"});"#, |
| 324 | r#"return await task({prompt: "no scope", profile: "release-lead"});"#, |
| 325 | r#"return await task({prompt: "wrong authority", type: "reviewer", writeAuthority: "workspace_write", writeRoots: ["src"]});"#, |
| 326 | r#"return await task({prompt: "wrong authority", type: "scout", writeAuthority: "workspace_write", writeRoots: ["src"]});"#, |
| 327 | r#"return await task({prompt: "role conflict", type: "implementer", role: "reviewer", writeRoots: ["src"]});"#, |
| 328 | ] { |
| 329 | let driver = Arc::new(FakeDriver::new()); |
| 330 | let message = script_message(run(&driver, source, json!(null)).await); |
| 331 | assert!( |
| 332 | message.contains("require") |
| 333 | || message.contains("cannot") |
| 334 | || message.contains("contradictory"), |
| 335 | "{message}" |
| 336 | ); |
| 337 | assert!(driver.requests().is_empty()); |
| 338 | } |
| 339 | } |
| 340 | |
| 341 | #[tokio::test] |
| 342 | async fn task_implementer_identity_can_be_narrowed_to_read_only_authority() { |
| 343 | let driver = Arc::new(FakeDriver::new()); |
| 344 | let value = run( |
| 345 | &driver, |
| 346 | r#"return await task({prompt: "verification-only plan", type: "implementer", writeAuthority: "read_only"});"#, |
| 347 | json!(null), |
| 348 | ) |
| 349 | .await |
| 350 | .expect("read-only authority must safely narrow an implementer identity"); |
| 351 | assert_eq!(value, json!("done:verification-only plan")); |
| 352 | let request = driver.requests().pop().expect("request"); |
| 353 | assert_eq!(request.subagent_type.as_deref(), Some("implementer")); |
| 354 | assert_eq!(request.write_authority.as_deref(), Some("read_only")); |
| 355 | assert!(request.write_roots.is_empty()); |
| 356 | } |
| 357 | |
| 358 | #[tokio::test] |
| 359 | async fn task_accepts_prompt_and_type_aliases() { |
| 360 | let driver = Arc::new(FakeDriver::new()); |
| 361 | run( |
| 362 | &driver, |
| 363 | r#"return await task({ prompt: "aliased", type: "verifier" });"#, |
| 364 | json!(null), |
| 365 | ) |
| 366 | .await |
| 367 | .unwrap(); |
| 368 | let request = &driver.requests()[0]; |
| 369 | assert_eq!(request.description, "aliased"); |
| 370 | assert_eq!(request.subagent_type.as_deref(), Some("verifier")); |
| 371 | } |
| 372 | |
| 373 | #[tokio::test] |
| 374 | async fn task_title_alias_routes_to_description() { |
| 375 | let driver = Arc::new(FakeDriver::new()); |
| 376 | run( |
| 377 | &driver, |
| 378 | r#"return await task({ title: "inspect the release candidate", type: "verifier" });"#, |
| 379 | json!(null), |
| 380 | ) |
| 381 | .await |
| 382 | .expect("title is accepted as the task description"); |
| 383 | |
| 384 | let request = &driver.requests()[0]; |
| 385 | assert_eq!(request.description, "inspect the release candidate"); |
| 386 | assert_eq!(request.subagent_type.as_deref(), Some("verifier")); |
| 387 | } |
| 388 | |
| 389 | #[tokio::test] |
| 390 | async fn task_prompt_takes_precedence_over_short_description() { |
| 391 | let driver = Arc::new(FakeDriver::new()); |
| 392 | run( |
| 393 | &driver, |
| 394 | r#"return await task({ |
| 395 | description: "Short progress summary", |
| 396 | prompt: "Detailed child instructions", |
| 397 | label: "fixture-compatible" |
| 398 | });"#, |
| 399 | json!(null), |
| 400 | ) |
| 401 | .await |
| 402 | .unwrap(); |
| 403 | let request = &driver.requests()[0]; |
| 404 | assert_eq!(request.description, "Detailed child instructions"); |
| 405 | assert_eq!(request.label.as_deref(), Some("fixture-compatible")); |
| 406 | } |
| 407 | |
| 408 | #[tokio::test] |
| 409 | async fn task_rejects_invalid_profile_tokens() { |
| 410 | for bad in ["two words", "a=b", "a\"b", "a`b", " "] { |
| 411 | let driver = Arc::new(FakeDriver::new()); |
| 412 | let source = format!( |
| 413 | "return await task({{ description: \"x\", profile: {} }});", |
| 414 | serde_json::Value::String(bad.to_string()) |
| 415 | ); |
| 416 | let message = script_message(run(&driver, &source, json!(null)).await); |
| 417 | assert!(message.contains("profile"), "profile {bad:?}: {message}"); |
| 418 | assert_eq!(driver.spawn_count(), 0, "invalid profile must not spawn"); |
| 419 | } |
| 420 | } |
| 421 | |
| 422 | #[tokio::test] |
| 423 | async fn task_requires_a_description() { |
| 424 | let driver = Arc::new(FakeDriver::new()); |
| 425 | let message = script_message(run(&driver, "return await task({});", json!(null)).await); |
| 426 | assert!(message.contains("description"), "{message}"); |
| 427 | assert_eq!(driver.spawn_count(), 0); |
| 428 | } |
| 429 | |
| 430 | #[tokio::test] |
| 431 | async fn task_rejects_unknown_option_names() { |
| 432 | let driver = Arc::new(FakeDriver::new()); |
| 433 | let message = script_message( |
| 434 | run( |
| 435 | &driver, |
| 436 | r#"return await task({ description: "x", responseschema: {} });"#, |
| 437 | json!(null), |
| 438 | ) |
| 439 | .await, |
| 440 | ); |
| 441 | assert!(message.contains("invalid options"), "{message}"); |
| 442 | assert_eq!(driver.spawn_count(), 0); |
| 443 | } |
| 444 | |
| 445 | #[tokio::test] |
| 446 | async fn driver_rejection_is_catchable_in_script() { |
| 447 | let driver = Arc::new(FakeDriver::new()); |
| 448 | driver.on("bad", FakeReply::Reject("admission cap".to_string())); |
| 449 | let value = run( |
| 450 | &driver, |
| 451 | r#" |
| 452 | try { |
| 453 | await task({ description: "bad idea" }); |
| 454 | return "no-throw"; |
| 455 | } catch (err) { |
| 456 | return String(err); |
| 457 | } |
| 458 | "#, |
| 459 | json!(null), |
| 460 | ) |
| 461 | .await |
| 462 | .unwrap(); |
| 463 | let text = value.as_str().unwrap(); |
| 464 | assert!(text.contains("admission cap"), "{text}"); |
| 465 | } |
| 466 | |
| 467 | #[tokio::test] |
| 468 | async fn parallel_fan_out_maps_one_failure_to_null_slot() { |
| 469 | let driver = Arc::new(FakeDriver::new()); |
| 470 | driver.on("beta", FakeReply::Fail("boom".to_string())); |
| 471 | let value = run( |
| 472 | &driver, |
| 473 | r#" |
| 474 | return await parallel([ |
| 475 | () => task({ description: "alpha" }), |
| 476 | () => task({ description: "beta" }), |
| 477 | () => task({ description: "gamma" }), |
| 478 | ]); |
| 479 | "#, |
| 480 | json!(null), |
| 481 | ) |
| 482 | .await |
| 483 | .unwrap(); |
| 484 | assert_eq!(value, json!(["done:alpha", null, "done:gamma"])); |
| 485 | assert_eq!(driver.spawn_count(), 3); |
| 486 | } |
| 487 | |
| 488 | #[tokio::test] |
| 489 | async fn parallel_logs_a_breadcrumb_when_a_slot_is_dropped_to_null() { |
| 490 | // #dogfood 0.8.67: a fan-out slot that fails for a non-schema reason still |
| 491 | // resolves to null (documented resilience), but must leave a breadcrumb in |
| 492 | // the run log so an operator can see why a slot came back null / nothing |
| 493 | // spawned — instead of a silent "completed" with no explanation. |
| 494 | let driver = Arc::new(FakeDriver::new()); |
| 495 | driver.on("beta", FakeReply::Fail("boom".to_string())); |
| 496 | let value = run( |
| 497 | &driver, |
| 498 | r#" |
| 499 | return await parallel([ |
| 500 | () => task({ description: "alpha" }), |
| 501 | () => task({ description: "beta" }), |
| 502 | ]); |
| 503 | "#, |
| 504 | json!(null), |
| 505 | ) |
| 506 | .await |
| 507 | .unwrap(); |
| 508 | assert_eq!(value, json!(["done:alpha", null])); |
| 509 | assert!( |
| 510 | driver.events().iter().any(|event| matches!( |
| 511 | event, |
| 512 | ProgressEvent::Log { message } if message.contains("dropped a failed slot") |
| 513 | )), |
| 514 | "a dropped parallel slot should leave a breadcrumb in the run log" |
| 515 | ); |
| 516 | } |
| 517 | |
| 518 | #[tokio::test] |
| 519 | async fn parallel_surfaces_response_schema_errors_instead_of_null() { |
| 520 | let driver = Arc::new(FakeDriver::new()); |
| 521 | driver.on( |
| 522 | "bad schema", |
| 523 | FakeReply::Complete(r#"{"refuted":"yes"}"#.to_string()), |
| 524 | ); |
| 525 | |
| 526 | let message = script_message( |
| 527 | run( |
| 528 | &driver, |
| 529 | r#" |
| 530 | return await parallel([ |
| 531 | () => task({ |
| 532 | description: "bad schema", |
| 533 | responseSchema: { |
| 534 | type: "object", |
| 535 | properties: { refuted: { type: "boolean" } }, |
| 536 | required: ["refuted"], |
| 537 | }, |
| 538 | }), |
| 539 | ]); |
| 540 | "#, |
| 541 | json!(null), |
| 542 | ) |
| 543 | .await, |
| 544 | ); |
| 545 | |
| 546 | // The default bounded repair (#5583) re-asks once — the fake's rule |
| 547 | // matches the repair too, so it fails identically and the run still |
| 548 | // fails loud instead of degrading to a null slot. |
| 549 | assert!(message.contains("responseSchema validation"), "{message}"); |
| 550 | assert_eq!( |
| 551 | driver.spawn_count(), |
| 552 | 2, |
| 553 | "default repair re-asks exactly once" |
| 554 | ); |
| 555 | assert!( |
| 556 | driver.events().iter().any(|event| matches!( |
| 557 | event, |
| 558 | ProgressEvent::TaskSchemaRepairAttempted { attempt: 1, raw, .. } |
| 559 | if raw.contains("yes") |
| 560 | )), |
| 561 | "the failed first attempt should be receipted before the repair" |
| 562 | ); |
| 563 | assert!( |
| 564 | driver.events().iter().any(|event| matches!( |
| 565 | event, |
| 566 | ProgressEvent::TaskSchemaValidationFailed { message, attempt: 2, .. } |
| 567 | if message.contains("responseSchema validation") |
| 568 | )), |
| 569 | "schema validation error should be emitted as workflow progress" |
| 570 | ); |
| 571 | } |
| 572 | |
| 573 | #[tokio::test] |
| 574 | async fn parallel_partial_mode_keeps_schema_failures_as_structured_slots() { |
| 575 | let driver = Arc::new(FakeDriver::new()); |
| 576 | // Repair is disabled per-task so each slot fails terminally on its own |
| 577 | // reply; the mixed fan-out then exercises partial mode directly. |
| 578 | driver.on( |
| 579 | "good slot", |
| 580 | FakeReply::Complete(r#"{"refuted": true}"#.to_string()), |
| 581 | ); |
| 582 | driver.on( |
| 583 | "bad slot", |
| 584 | FakeReply::Complete("not json at all".to_string()), |
| 585 | ); |
| 586 | driver.on("dead slot", FakeReply::Fail("boom".to_string())); |
| 587 | |
| 588 | let value = run( |
| 589 | &driver, |
| 590 | r#" |
| 591 | const results = await parallel([ |
| 592 | () => task({ |
| 593 | description: "good slot", |
| 594 | responseSchema: { "type": "object" }, |
| 595 | }), |
| 596 | () => task({ |
| 597 | description: "bad slot", |
| 598 | schemaRepairAttempts: 0, |
| 599 | responseSchema: { "type": "object" }, |
| 600 | }), |
| 601 | () => task({ description: "dead slot" }), |
| 602 | ], { mode: "partial" }); |
| 603 | return results.map((slot) => |
| 604 | slot && typeof slot === "object" && slot.__taskError !== undefined |
| 605 | ? "error:" + slot.__taskError.kind |
| 606 | : slot === null |
| 607 | ? "null" |
| 608 | : "value:" + JSON.stringify(slot) |
| 609 | ); |
| 610 | "#, |
| 611 | json!(null), |
| 612 | ) |
| 613 | .await |
| 614 | .expect("partial mode completes the fan-out"); |
| 615 | |
| 616 | assert_eq!( |
| 617 | value, |
| 618 | json!([ |
| 619 | "value:{\"refuted\":true}", |
| 620 | // The JS-level kind is the fatal "schema"; the finer decode kind |
| 621 | // (json_parse) lives on the receipt events, asserted below. |
| 622 | "error:schema", |
| 623 | // R9 behavior change: partial mode used to drop a dead subagent |
| 624 | // to `null` — indistinguishable from a slot that legitimately |
| 625 | // returned nothing. It is now a typed, inspectable failure. |
| 626 | "error:agent" |
| 627 | ]) |
| 628 | ); |
| 629 | // Every failed slot still leaves its terminal receipt. |
| 630 | assert!( |
| 631 | driver.events().iter().any(|event| matches!( |
| 632 | event, |
| 633 | ProgressEvent::TaskSchemaValidationFailed { kind, .. } if kind == "json_parse" |
| 634 | )), |
| 635 | "partial mode must not swallow the schema-failure receipt" |
| 636 | ); |
| 637 | } |
| 638 | |
| 639 | #[tokio::test] |
| 640 | async fn parallel_partial_mode_still_fails_the_run_on_cancellation() { |
| 641 | let driver = Arc::new(FakeDriver::new()); |
| 642 | driver.on("hang", FakeReply::Never); |
| 643 | let cancel = WorkflowRunCancel::new(); |
| 644 | let run_cancel = cancel.clone(); |
| 645 | let run_driver = driver.clone(); |
| 646 | let handle = tokio::spawn(async move { |
| 647 | WorkflowVm::new() |
| 648 | .run_script_with_cancel( |
| 649 | r#" |
| 650 | await parallel([ |
| 651 | () => task({ description: "hang", responseSchema: { "type": "object" } }), |
| 652 | ], { mode: "partial" }); |
| 653 | "#, |
| 654 | json!(null), |
| 655 | run_driver as Arc<dyn codewhale_workflow_js::WorkflowDriver>, |
| 656 | run_cancel, |
| 657 | ) |
| 658 | .await |
| 659 | }); |
| 660 | |
| 661 | tokio::time::timeout(Duration::from_secs(2), async { |
| 662 | while driver.spawn_count() == 0 { |
| 663 | tokio::task::yield_now().await; |
| 664 | } |
| 665 | }) |
| 666 | .await |
| 667 | .expect("task should start"); |
| 668 | cancel.cancel(); |
| 669 | |
| 670 | let result = handle.await.expect("VM task should join"); |
| 671 | assert!( |
| 672 | matches!(result, Err(WorkflowJsError::Cancelled)), |
| 673 | "partial mode must not downgrade cancellation into a slot value: {result:?}" |
| 674 | ); |
| 675 | } |
| 676 | |
| 677 | #[tokio::test] |
| 678 | async fn pipeline_surfaces_response_schema_errors_instead_of_null() { |
| 679 | let driver = Arc::new(FakeDriver::new()); |
| 680 | driver.on( |
| 681 | "bad schema", |
| 682 | FakeReply::Complete("not json at all".to_string()), |
| 683 | ); |
| 684 | |
| 685 | let message = script_message( |
| 686 | run( |
| 687 | &driver, |
| 688 | r#" |
| 689 | return await pipeline( |
| 690 | ["bad schema"], |
| 691 | (description) => task({ |
| 692 | description, |
| 693 | schemaRepairAttempts: 0, |
| 694 | responseSchema: { |
| 695 | type: "object", |
| 696 | properties: { refuted: { type: "boolean" } }, |
| 697 | required: ["refuted"], |
| 698 | }, |
| 699 | }), |
| 700 | ); |
| 701 | "#, |
| 702 | json!(null), |
| 703 | ) |
| 704 | .await, |
| 705 | ); |
| 706 | |
| 707 | // Repair disabled: the first decode failure is terminal. |
| 708 | assert!(message.contains("not valid JSON"), "{message}"); |
| 709 | assert_eq!(driver.spawn_count(), 1); |
| 710 | assert!( |
| 711 | driver.events().iter().any(|event| matches!( |
| 712 | event, |
| 713 | ProgressEvent::TaskSchemaValidationFailed { kind, attempt: 1, .. } |
| 714 | if kind == "json_parse" |
| 715 | )), |
| 716 | "a disabled repair must fail terminally on attempt 1 with the parse kind" |
| 717 | ); |
| 718 | } |
| 719 | |
| 720 | #[tokio::test] |
| 721 | async fn prose_wrapped_json_repairs_in_one_attempt() { |
| 722 | let driver = Arc::new(FakeDriver::new()); |
| 723 | // First match wins: the repair spawn's description carries the |
| 724 | // "[schema repair 2]" marker, the first attempt's does not. |
| 725 | driver.on( |
| 726 | "[schema repair", |
| 727 | FakeReply::Complete(r#"{"refuted": true}"#.to_string()), |
| 728 | ); |
| 729 | driver.on( |
| 730 | "score the claim", |
| 731 | FakeReply::Complete( |
| 732 | "Sure! Happy to help. Here is my verdict:\n\ |
| 733 | ```json\n{\"refuted\": true}\n```\n\ |
| 734 | Let me know if you need anything else." |
| 735 | .to_string(), |
| 736 | ), |
| 737 | ); |
| 738 | |
| 739 | let value = run( |
| 740 | &driver, |
| 741 | r#" |
| 742 | return await task({ |
| 743 | description: "score the claim", |
| 744 | responseSchema: { |
| 745 | type: "object", |
| 746 | properties: { refuted: { type: "boolean" } }, |
| 747 | required: ["refuted"], |
| 748 | }, |
| 749 | }); |
| 750 | "#, |
| 751 | json!(null), |
| 752 | ) |
| 753 | .await |
| 754 | .expect("prose-wrapped JSON should repair in one attempt"); |
| 755 | |
| 756 | assert_eq!(value, json!({ "refuted": true })); |
| 757 | assert_eq!(driver.spawn_count(), 2); |
| 758 | let requests = driver.requests(); |
| 759 | assert_eq!(requests[0].response_schema, requests[1].response_schema); |
| 760 | assert!( |
| 761 | requests[1].description.starts_with("[schema repair 2]"), |
| 762 | "the repair spawn must identify itself: {}", |
| 763 | requests[1].description |
| 764 | ); |
| 765 | assert!( |
| 766 | requests[1].description.contains("score the claim"), |
| 767 | "the repair prompt must embed the original task" |
| 768 | ); |
| 769 | assert!( |
| 770 | driver.events().iter().any(|event| matches!( |
| 771 | event, |
| 772 | ProgressEvent::TaskSchemaRepairAttempted { |
| 773 | kind, attempt: 1, raw, raw_truncated: false, .. |
| 774 | } if kind == "json_parse" && raw.contains("Happy to help") |
| 775 | )), |
| 776 | "the prose failure should be receipted with the parse kind" |
| 777 | ); |
| 778 | assert!( |
| 779 | !driver |
| 780 | .events() |
| 781 | .iter() |
| 782 | .any(|event| matches!(event, ProgressEvent::TaskSchemaValidationFailed { .. })), |
| 783 | "a successful repair must not leave a terminal schema-failure receipt" |
| 784 | ); |
| 785 | } |
| 786 | |
| 787 | #[tokio::test] |
| 788 | async fn schema_violation_receipt_names_the_validation_kind() { |
| 789 | let driver = Arc::new(FakeDriver::new()); |
| 790 | driver.on( |
| 791 | "[schema repair", |
| 792 | FakeReply::Complete(r#"{"refuted": false}"#.to_string()), |
| 793 | ); |
| 794 | driver.on( |
| 795 | "check the gate", |
| 796 | FakeReply::Complete(r#"{"refuted":"no"}"#.to_string()), |
| 797 | ); |
| 798 | |
| 799 | let value = run( |
| 800 | &driver, |
| 801 | r#" |
| 802 | return await task({ |
| 803 | description: "check the gate", |
| 804 | responseSchema: { |
| 805 | type: "object", |
| 806 | properties: { refuted: { type: "boolean" } }, |
| 807 | required: ["refuted"], |
| 808 | }, |
| 809 | }); |
| 810 | "#, |
| 811 | json!(null), |
| 812 | ) |
| 813 | .await |
| 814 | .expect("valid JSON of the wrong shape should repair"); |
| 815 | |
| 816 | assert_eq!(value, json!({ "refuted": false })); |
| 817 | assert!( |
| 818 | driver.events().iter().any(|event| matches!( |
| 819 | event, |
| 820 | ProgressEvent::TaskSchemaRepairAttempted { kind, message, .. } |
| 821 | if kind == "schema_validation" |
| 822 | && message.contains("responseSchema validation") |
| 823 | )), |
| 824 | "a parsed-but-invalid reply must receipt as schema_validation, not json_parse" |
| 825 | ); |
| 826 | } |
| 827 | |
| 828 | #[tokio::test] |
| 829 | async fn schema_repair_attempts_is_bounded_at_the_parse_gate() { |
| 830 | let driver = Arc::new(FakeDriver::new()); |
| 831 | let message = script_message( |
| 832 | run( |
| 833 | &driver, |
| 834 | r#" |
| 835 | return await task({ |
| 836 | description: "bound me", |
| 837 | schemaRepairAttempts: 4, |
| 838 | responseSchema: { "type": "object" }, |
| 839 | }); |
| 840 | "#, |
| 841 | json!(null), |
| 842 | ) |
| 843 | .await, |
| 844 | ); |
| 845 | assert!(message.contains("bounded to 3"), "{message}"); |
| 846 | assert_eq!( |
| 847 | driver.spawn_count(), |
| 848 | 0, |
| 849 | "no child may spawn for a bad option" |
| 850 | ); |
| 851 | } |
| 852 | |
| 853 | #[tokio::test] |
| 854 | async fn repair_is_refused_when_the_shared_budget_is_exhausted() { |
| 855 | let driver = Arc::new(FakeDriver::new()); |
| 856 | // Attempt 1 is admitted with an empty pool and debits it fully at spawn. |
| 857 | driver.set_budget(Some(100), 100); |
| 858 | driver.on( |
| 859 | "spend it all", |
| 860 | FakeReply::Complete("sure thing, no JSON here".to_string()), |
| 861 | ); |
| 862 | |
| 863 | let message = script_message( |
| 864 | run( |
| 865 | &driver, |
| 866 | r#" |
| 867 | return await task({ |
| 868 | description: "spend it all", |
| 869 | responseSchema: { "type": "object" }, |
| 870 | }); |
| 871 | "#, |
| 872 | json!(null), |
| 873 | ) |
| 874 | .await, |
| 875 | ); |
| 876 | |
| 877 | assert!( |
| 878 | message.contains("repair skipped: budget exhausted"), |
| 879 | "{message}" |
| 880 | ); |
| 881 | assert_eq!( |
| 882 | driver.spawn_count(), |
| 883 | 1, |
| 884 | "the repair must not spawn on an empty pool" |
| 885 | ); |
| 886 | assert!( |
| 887 | driver.events().iter().any(|event| matches!( |
| 888 | event, |
| 889 | ProgressEvent::TaskSchemaValidationFailed { attempt: 1, message, .. } |
| 890 | if message.contains("repair skipped: budget exhausted") |
| 891 | )), |
| 892 | "the refused repair must stay a schema failure with the reason named" |
| 893 | ); |
| 894 | } |
| 895 | |
| 896 | #[tokio::test] |
| 897 | async fn repair_is_refused_when_the_shared_wall_clock_is_spent() { |
| 898 | let driver = Arc::new(FakeDriver::new()); |
| 899 | driver.on_with_delay( |
| 900 | "slow prose", |
| 901 | FakeReply::Complete("eventually, still not json".to_string()), |
| 902 | Duration::from_millis(1_100), |
| 903 | ); |
| 904 | |
| 905 | let message = script_message( |
| 906 | run( |
| 907 | &driver, |
| 908 | r#" |
| 909 | return await task({ |
| 910 | description: "slow prose", |
| 911 | wallTimeSecs: 1, |
| 912 | responseSchema: { "type": "object" }, |
| 913 | }); |
| 914 | "#, |
| 915 | json!(null), |
| 916 | ) |
| 917 | .await, |
| 918 | ); |
| 919 | |
| 920 | assert!( |
| 921 | message.contains("repair skipped: no wall-time left from wallTimeSecs"), |
| 922 | "{message}" |
| 923 | ); |
| 924 | assert_eq!( |
| 925 | driver.spawn_count(), |
| 926 | 1, |
| 927 | "the repair inherits the spent clock, not a fresh one" |
| 928 | ); |
| 929 | } |
| 930 | |
| 931 | #[tokio::test] |
| 932 | async fn cancellation_during_repair_terminates_cleanly() { |
| 933 | let driver = Arc::new(FakeDriver::new()); |
| 934 | driver.on("[schema repair", FakeReply::Never); |
| 935 | driver.on( |
| 936 | "hang the repair", |
| 937 | FakeReply::Complete("prose, no json".to_string()), |
| 938 | ); |
| 939 | let cancel = WorkflowRunCancel::new(); |
| 940 | let run_cancel = cancel.clone(); |
| 941 | let run_driver = driver.clone(); |
| 942 | let handle = tokio::spawn(async move { |
| 943 | WorkflowVm::new() |
| 944 | .run_script_with_cancel( |
| 945 | r#" |
| 946 | return await task({ |
| 947 | description: "hang the repair", |
| 948 | responseSchema: { "type": "object" }, |
| 949 | }); |
| 950 | "#, |
| 951 | json!(null), |
| 952 | run_driver as Arc<dyn codewhale_workflow_js::WorkflowDriver>, |
| 953 | run_cancel, |
| 954 | ) |
| 955 | .await |
| 956 | }); |
| 957 | |
| 958 | tokio::time::timeout(Duration::from_secs(2), async { |
| 959 | while driver.spawn_count() < 2 { |
| 960 | tokio::task::yield_now().await; |
| 961 | } |
| 962 | }) |
| 963 | .await |
| 964 | .expect("repair should start"); |
| 965 | cancel.cancel(); |
| 966 | |
| 967 | let result = handle.await.expect("VM task should join"); |
| 968 | assert!( |
| 969 | matches!(result, Err(WorkflowJsError::Cancelled)), |
| 970 | "{result:?}" |
| 971 | ); |
| 972 | assert!( |
| 973 | !driver |
| 974 | .events() |
| 975 | .iter() |
| 976 | .any(|event| matches!(event, ProgressEvent::TaskSchemaValidationFailed { .. })), |
| 977 | "cancellation must not be rewritten into a schema failure" |
| 978 | ); |
| 979 | } |
| 980 | |
| 981 | #[tokio::test] |
| 982 | async fn parallel_fail_fast_rejects_with_the_typed_slot_error() { |
| 983 | let driver = Arc::new(FakeDriver::new()); |
| 984 | driver.on("beta", FakeReply::Fail("boom".to_string())); |
| 985 | let value = run( |
| 986 | &driver, |
| 987 | r#" |
| 988 | try { |
| 989 | await parallel([ |
| 990 | () => task({ description: "alpha" }), |
| 991 | () => task({ description: "beta" }), |
| 992 | ], { mode: "fail-fast" }); |
| 993 | return "no-error"; |
| 994 | } catch (err) { |
| 995 | return (err && err.kind) + ":" + (err && err.message); |
| 996 | } |
| 997 | "#, |
| 998 | json!(null), |
| 999 | ) |
| 1000 | .await |
| 1001 | .unwrap(); |
| 1002 | let text = value.as_str().unwrap(); |
| 1003 | assert!( |
| 1004 | // R9: a child that ran and failed is `agent`, distinct from the |
| 1005 | // `script` kind a plain `throw` in a thunk produces. |
| 1006 | text.starts_with("agent:") && text.contains("boom"), |
| 1007 | "fail-fast must reject with the typed slot error: {text}" |
| 1008 | ); |
| 1009 | assert!( |
| 1010 | driver.events().iter().any(|event| matches!( |
| 1011 | event, |
| 1012 | ProgressEvent::Log { message } if message.contains("fail-fast slot error") |
| 1013 | )), |
| 1014 | "fail-fast must leave a breadcrumb with the slot error" |
| 1015 | ); |
| 1016 | } |
| 1017 | |
| 1018 | #[tokio::test] |
| 1019 | async fn task_errors_carry_typed_kinds() { |
| 1020 | let driver = Arc::new(FakeDriver::new()); |
| 1021 | driver.on("budget", FakeReply::BudgetExhausted("limit 10".to_string())); |
| 1022 | driver.on("cancelled", FakeReply::Cancelled); |
| 1023 | driver.on("admission", FakeReply::Reject("admission cap".to_string())); |
| 1024 | let value = run( |
| 1025 | &driver, |
| 1026 | r#" |
| 1027 | const kinds = {}; |
| 1028 | for (const description of ["budget", "cancelled", "admission"]) { |
| 1029 | try { |
| 1030 | await task({ description }); |
| 1031 | kinds[description] = "none"; |
| 1032 | } catch (err) { |
| 1033 | kinds[description] = err && err.kind; |
| 1034 | } |
| 1035 | } |
| 1036 | return kinds; |
| 1037 | "#, |
| 1038 | json!(null), |
| 1039 | ) |
| 1040 | .await |
| 1041 | .unwrap(); |
| 1042 | assert_eq!( |
| 1043 | value, |
| 1044 | json!({"budget": "budget", "cancelled": "cancelled", "admission": "admission"}) |
| 1045 | ); |
| 1046 | } |
| 1047 | |
| 1048 | #[tokio::test] |
| 1049 | async fn pipeline_fail_fast_rejects_instead_of_nulling_the_item() { |
| 1050 | let value = run( |
| 1051 | &Arc::new(FakeDriver::new()), |
| 1052 | r#" |
| 1053 | const stage = async (value) => { |
| 1054 | if (value === 1) throw new Error("stage boom"); |
| 1055 | return value * 2; |
| 1056 | }; |
| 1057 | try { |
| 1058 | await pipeline([1, 2], { stages: [stage], mode: "fail-fast" }); |
| 1059 | return "no-error"; |
| 1060 | } catch (err) { |
| 1061 | return (err && err.kind) + ":" + (err && err.message); |
| 1062 | } |
| 1063 | "#, |
| 1064 | json!(null), |
| 1065 | ) |
| 1066 | .await |
| 1067 | .unwrap(); |
| 1068 | assert_eq!(value, json!("script:stage boom")); |
| 1069 | } |
| 1070 | |
| 1071 | #[tokio::test] |
| 1072 | async fn parallel_enforces_the_1000_item_cap_without_spawning() { |
| 1073 | let driver = Arc::new(FakeDriver::new()); |
| 1074 | let value = run( |
| 1075 | &driver, |
| 1076 | r#" |
| 1077 | const thunks = new Array(1001).fill(() => task({ description: "x" })); |
| 1078 | try { |
| 1079 | await parallel(thunks); |
| 1080 | return "no-throw"; |
| 1081 | } catch (err) { |
| 1082 | return String(err); |
| 1083 | } |
| 1084 | "#, |
| 1085 | json!(null), |
| 1086 | ) |
| 1087 | .await |
| 1088 | .unwrap(); |
| 1089 | let text = value.as_str().unwrap(); |
| 1090 | assert!(text.contains("max 1000"), "{text}"); |
| 1091 | assert_eq!(driver.spawn_count(), 0, "cap must reject before any spawn"); |
| 1092 | } |
| 1093 | |
| 1094 | #[tokio::test] |
| 1095 | async fn parallel_accepts_exactly_1000_items() { |
| 1096 | let driver = Arc::new(FakeDriver::new()); |
| 1097 | let value = run( |
| 1098 | &driver, |
| 1099 | r#" |
| 1100 | const thunks = new Array(1000).fill(() => Promise.resolve(1)); |
| 1101 | const results = await parallel(thunks); |
| 1102 | return results.length; |
| 1103 | "#, |
| 1104 | json!(null), |
| 1105 | ) |
| 1106 | .await |
| 1107 | .unwrap(); |
| 1108 | assert_eq!(value, json!(1000)); |
| 1109 | } |
| 1110 | |
| 1111 | #[tokio::test] |
| 1112 | async fn pipeline_has_no_barrier_between_stages() { |
| 1113 | let driver = Arc::new(FakeDriver::new()); |
| 1114 | // Item A crawls through stage 1; item B sprints through both stages. |
| 1115 | driver.on_with_delay( |
| 1116 | "s1:A", |
| 1117 | FakeReply::Complete("A1".to_string()), |
| 1118 | Duration::from_millis(300), |
| 1119 | ); |
| 1120 | driver.on_with_delay( |
| 1121 | "s1:B", |
| 1122 | FakeReply::Complete("B1".to_string()), |
| 1123 | Duration::from_millis(20), |
| 1124 | ); |
| 1125 | driver.on_with_delay( |
| 1126 | "s2:B1", |
| 1127 | FakeReply::Complete("B2".to_string()), |
| 1128 | Duration::from_millis(20), |
| 1129 | ); |
| 1130 | driver.on("s2:A1", FakeReply::Complete("A2".to_string())); |
| 1131 | |
| 1132 | let value = run( |
| 1133 | &driver, |
| 1134 | r#" |
| 1135 | return await pipeline( |
| 1136 | ["A", "B"], |
| 1137 | (v) => task({ description: "s1:" + v }), |
| 1138 | (v) => task({ description: "s2:" + v }), |
| 1139 | ); |
| 1140 | "#, |
| 1141 | json!(null), |
| 1142 | ) |
| 1143 | .await |
| 1144 | .unwrap(); |
| 1145 | assert_eq!(value, json!(["A2", "B2"])); |
| 1146 | |
| 1147 | // B's stage 2 must have been requested while A was still in stage 1 — |
| 1148 | // per-item chains, no stage barrier. |
| 1149 | let descriptions = driver.request_descriptions(); |
| 1150 | assert_eq!(descriptions[..2], ["s1:A".to_string(), "s1:B".to_string()]); |
| 1151 | assert_eq!( |
| 1152 | descriptions[2], "s2:B1", |
| 1153 | "expected B to reach stage 2 while A was still in stage 1: {descriptions:?}" |
| 1154 | ); |
| 1155 | assert_eq!(descriptions[3], "s2:A1"); |
| 1156 | } |
| 1157 | |
| 1158 | #[tokio::test] |
| 1159 | async fn pipeline_stage_error_drops_only_that_item() { |
| 1160 | let driver = Arc::new(FakeDriver::new()); |
| 1161 | driver.on("s1:B", FakeReply::Fail("boom".to_string())); |
| 1162 | let value = run( |
| 1163 | &driver, |
| 1164 | r#" |
| 1165 | return await pipeline( |
| 1166 | ["A", "B"], |
| 1167 | (v) => task({ description: "s1:" + v }), |
| 1168 | (v) => v + "+2", |
| 1169 | ); |
| 1170 | "#, |
| 1171 | json!(null), |
| 1172 | ) |
| 1173 | .await |
| 1174 | .unwrap(); |
| 1175 | assert_eq!(value, json!(["done:s1:A+2", null])); |
| 1176 | } |
| 1177 | |
| 1178 | #[tokio::test] |
| 1179 | async fn task_throws_once_budget_spent_reaches_total() { |
| 1180 | let driver = Arc::new(FakeDriver::new()); |
| 1181 | driver.set_budget(Some(100), 60); |
| 1182 | let value = run( |
| 1183 | &driver, |
| 1184 | r#" |
| 1185 | let completed = 0; |
| 1186 | try { |
| 1187 | while (true) { |
| 1188 | await task({ description: "chunk " + completed }); |
| 1189 | completed++; |
| 1190 | } |
| 1191 | } catch (err) { |
| 1192 | return { completed, message: String(err) }; |
| 1193 | } |
| 1194 | "#, |
| 1195 | json!(null), |
| 1196 | ) |
| 1197 | .await |
| 1198 | .unwrap(); |
| 1199 | assert_eq!(value["completed"], json!(2)); |
| 1200 | let message = value["message"].as_str().unwrap(); |
| 1201 | assert!(message.contains("budget exhausted"), "{message}"); |
| 1202 | assert_eq!(driver.spawn_count(), 2); |
| 1203 | } |
| 1204 | |
| 1205 | #[tokio::test] |
| 1206 | async fn budget_globals_reflect_live_driver_snapshots() { |
| 1207 | let driver = Arc::new(FakeDriver::new()); |
| 1208 | driver.set_budget(Some(1000), 100); |
| 1209 | let value = run( |
| 1210 | &driver, |
| 1211 | r#" |
| 1212 | const before = budget.remaining(); |
| 1213 | await task({ description: "one" }); |
| 1214 | return { |
| 1215 | total: budget.total, |
| 1216 | before, |
| 1217 | spent: budget.spent(), |
| 1218 | after: budget.remaining(), |
| 1219 | }; |
| 1220 | "#, |
| 1221 | json!(null), |
| 1222 | ) |
| 1223 | .await |
| 1224 | .unwrap(); |
| 1225 | assert_eq!( |
| 1226 | value, |
| 1227 | json!({"total": 1000, "before": 1000, "spent": 100, "after": 900}) |
| 1228 | ); |
| 1229 | } |
| 1230 | |
| 1231 | #[tokio::test] |
| 1232 | async fn unbounded_budget_reads_as_null_total_and_infinite_remaining() { |
| 1233 | let driver = Arc::new(FakeDriver::new()); |
| 1234 | let value = run( |
| 1235 | &driver, |
| 1236 | "return budget.total === null && budget.remaining() === Infinity;", |
| 1237 | json!(null), |
| 1238 | ) |
| 1239 | .await |
| 1240 | .unwrap(); |
| 1241 | assert_eq!(value, json!(true)); |
| 1242 | } |
| 1243 | |
| 1244 | #[tokio::test] |
| 1245 | async fn lifetime_cap_throws_on_spawn_attempt_1001() { |
| 1246 | let driver = Arc::new(FakeDriver::new()); |
| 1247 | let value = run( |
| 1248 | &driver, |
| 1249 | r#" |
| 1250 | let completed = 0; |
| 1251 | try { |
| 1252 | for (let i = 0; i < 1001; i++) { |
| 1253 | await task({ description: "t" + i }); |
| 1254 | completed++; |
| 1255 | } |
| 1256 | return "no-throw"; |
| 1257 | } catch (err) { |
| 1258 | return { completed, message: String(err) }; |
| 1259 | } |
| 1260 | "#, |
| 1261 | json!(null), |
| 1262 | ) |
| 1263 | .await |
| 1264 | .unwrap(); |
| 1265 | assert_eq!(value["completed"], json!(WORKFLOW_LIFETIME_CAP)); |
| 1266 | let message = value["message"].as_str().unwrap(); |
| 1267 | assert!(message.contains("lifetime agent cap (1000)"), "{message}"); |
| 1268 | assert_eq!(driver.spawn_count(), WORKFLOW_LIFETIME_CAP as usize); |
| 1269 | } |
| 1270 | |
| 1271 | #[tokio::test] |
| 1272 | async fn response_schema_returns_the_parsed_validated_object() { |
| 1273 | let driver = Arc::new(FakeDriver::new()); |
| 1274 | driver.on( |
| 1275 | "check", |
| 1276 | FakeReply::Complete(r#"{"refuted": true, "confidence": 0.9}"#.to_string()), |
| 1277 | ); |
| 1278 | let value = run( |
| 1279 | &driver, |
| 1280 | r#" |
| 1281 | const verdict = await task({ |
| 1282 | description: "check the claim", |
| 1283 | responseSchema: { |
| 1284 | type: "object", |
| 1285 | properties: { refuted: { type: "boolean" } }, |
| 1286 | required: ["refuted"], |
| 1287 | }, |
| 1288 | }); |
| 1289 | return verdict.refuted === true ? "refuted" : "upheld"; |
| 1290 | "#, |
| 1291 | json!(null), |
| 1292 | ) |
| 1293 | .await |
| 1294 | .unwrap(); |
| 1295 | assert_eq!(value, json!("refuted")); |
| 1296 | assert!(driver.requests()[0].response_schema.is_some()); |
| 1297 | } |
| 1298 | |
| 1299 | #[tokio::test] |
| 1300 | async fn response_schema_rejects_non_json_replies() { |
| 1301 | let driver = Arc::new(FakeDriver::new()); |
| 1302 | driver.on( |
| 1303 | "check", |
| 1304 | FakeReply::Complete("definitely not json".to_string()), |
| 1305 | ); |
| 1306 | let message = script_message( |
| 1307 | run( |
| 1308 | &driver, |
| 1309 | r#" |
| 1310 | return await task({ |
| 1311 | description: "check", |
| 1312 | responseSchema: { type: "object" }, |
| 1313 | }); |
| 1314 | "#, |
| 1315 | json!(null), |
| 1316 | ) |
| 1317 | .await, |
| 1318 | ); |
| 1319 | assert!(message.contains("not valid JSON"), "{message}"); |
| 1320 | } |
| 1321 | |
| 1322 | #[tokio::test] |
| 1323 | async fn response_schema_rejects_schema_violations() { |
| 1324 | let driver = Arc::new(FakeDriver::new()); |
| 1325 | driver.on( |
| 1326 | "check", |
| 1327 | FakeReply::Complete(r#"{"refuted": "yes"}"#.to_string()), |
| 1328 | ); |
| 1329 | let message = script_message( |
| 1330 | run( |
| 1331 | &driver, |
| 1332 | r#" |
| 1333 | return await task({ |
| 1334 | description: "check", |
| 1335 | responseSchema: { |
| 1336 | type: "object", |
| 1337 | properties: { refuted: { type: "boolean" } }, |
| 1338 | required: ["refuted"], |
| 1339 | }, |
| 1340 | }); |
| 1341 | "#, |
| 1342 | json!(null), |
| 1343 | ) |
| 1344 | .await, |
| 1345 | ); |
| 1346 | assert!(message.contains("responseSchema validation"), "{message}"); |
| 1347 | } |
| 1348 | |
| 1349 | #[tokio::test] |
| 1350 | async fn determinism_ban_date_now() { |
| 1351 | let driver = Arc::new(FakeDriver::new()); |
| 1352 | let message = script_message(run(&driver, "return Date.now();", json!(null)).await); |
| 1353 | assert!(message.contains("Date.now()"), "{message}"); |
| 1354 | } |
| 1355 | |
| 1356 | #[tokio::test] |
| 1357 | async fn determinism_ban_math_random() { |
| 1358 | let driver = Arc::new(FakeDriver::new()); |
| 1359 | let message = script_message(run(&driver, "return Math.random();", json!(null)).await); |
| 1360 | assert!(message.contains("Math.random()"), "{message}"); |
| 1361 | } |
| 1362 | |
| 1363 | #[tokio::test] |
| 1364 | async fn determinism_ban_new_date() { |
| 1365 | let driver = Arc::new(FakeDriver::new()); |
| 1366 | let message = script_message(run(&driver, "return new Date();", json!(null)).await); |
| 1367 | assert!(message.contains("unavailable"), "{message}"); |
| 1368 | } |
| 1369 | |
| 1370 | /// Explicit product surface for the sandboxed Workflow VM (#4129). |
| 1371 | /// |
| 1372 | /// Only these Workflow-owned calls may exist on `globalThis` beyond standard |
| 1373 | /// ECMAScript intrinsics. If a new host global is intentionally added, update |
| 1374 | /// this list in the same PR — the fail-closed inventory test below will break |
| 1375 | /// until the allowlist is extended deliberately. |
| 1376 | const WORKFLOW_ALLOWED_GLOBALS: &[&str] = &[ |
| 1377 | "task", "parallel", "pipeline", "phase", "log", "budget", "args", |
| 1378 | ]; |
| 1379 | |
| 1380 | /// Host / Node / Deno / browser surfaces that must never leak into the VM. |
| 1381 | /// |
| 1382 | /// Standard ECMAScript intrinsics (`Object`, `Function`, `eval`, `Promise`, …) |
| 1383 | /// remain available; this list is only host escape hatches. |
| 1384 | const SANDBOX_BANNED_GLOBALS: &[&str] = &[ |
| 1385 | "process", |
| 1386 | "require", |
| 1387 | "module", |
| 1388 | "exports", |
| 1389 | "__dirname", |
| 1390 | "__filename", |
| 1391 | "Buffer", |
| 1392 | "fs", |
| 1393 | "child_process", |
| 1394 | "os", |
| 1395 | "path", |
| 1396 | "net", |
| 1397 | "http", |
| 1398 | "https", |
| 1399 | "fetch", |
| 1400 | "XMLHttpRequest", |
| 1401 | "WebSocket", |
| 1402 | "Deno", |
| 1403 | "Bun", |
| 1404 | "Worker", |
| 1405 | ]; |
| 1406 | |
| 1407 | #[tokio::test] |
| 1408 | async fn sandbox_exposes_only_the_documented_workflow_calls() { |
| 1409 | let driver = Arc::new(FakeDriver::new()); |
| 1410 | let value = run( |
| 1411 | &driver, |
| 1412 | r#" |
| 1413 | return { |
| 1414 | task: typeof task, |
| 1415 | parallel: typeof parallel, |
| 1416 | pipeline: typeof pipeline, |
| 1417 | phase: typeof phase, |
| 1418 | log: typeof log, |
| 1419 | budget: typeof budget, |
| 1420 | args: typeof args, |
| 1421 | }; |
| 1422 | "#, |
| 1423 | json!({"ok": true}), |
| 1424 | ) |
| 1425 | .await |
| 1426 | .unwrap(); |
| 1427 | assert_eq!( |
| 1428 | value, |
| 1429 | json!({ |
| 1430 | "task": "function", |
| 1431 | "parallel": "function", |
| 1432 | "pipeline": "function", |
| 1433 | "phase": "function", |
| 1434 | "log": "function", |
| 1435 | "budget": "object", |
| 1436 | "args": "object", |
| 1437 | }) |
| 1438 | ); |
| 1439 | // Keep the constant and the live typeof probe in lockstep. |
| 1440 | assert_eq!( |
| 1441 | WORKFLOW_ALLOWED_GLOBALS, |
| 1442 | &[ |
| 1443 | "task", "parallel", "pipeline", "phase", "log", "budget", "args" |
| 1444 | ] |
| 1445 | ); |
| 1446 | } |
| 1447 | |
| 1448 | #[tokio::test] |
| 1449 | async fn sandbox_blocks_host_filesystem_shell_network_and_env_surfaces() { |
| 1450 | // Each probe must either throw / reject or resolve to a clearly absent |
| 1451 | // binding. We never allow a successful host escape. |
| 1452 | let probes: &[(&str, &str)] = &[ |
| 1453 | ( |
| 1454 | "process.env", |
| 1455 | r#" |
| 1456 | if (typeof process !== "undefined") { |
| 1457 | return process.env; |
| 1458 | } |
| 1459 | throw new Error("process is unavailable"); |
| 1460 | "#, |
| 1461 | ), |
| 1462 | ( |
| 1463 | "require('fs')", |
| 1464 | r#" |
| 1465 | if (typeof require === "function") { |
| 1466 | return require("fs"); |
| 1467 | } |
| 1468 | throw new Error("require is unavailable"); |
| 1469 | "#, |
| 1470 | ), |
| 1471 | ( |
| 1472 | "import", |
| 1473 | r#" |
| 1474 | // Dynamic import is a module-loader surface; the VM has no loader. |
| 1475 | return await import("fs"); |
| 1476 | "#, |
| 1477 | ), |
| 1478 | ( |
| 1479 | "fetch", |
| 1480 | r#" |
| 1481 | if (typeof fetch === "function") { |
| 1482 | return await fetch("https://example.invalid/"); |
| 1483 | } |
| 1484 | throw new Error("fetch is unavailable"); |
| 1485 | "#, |
| 1486 | ), |
| 1487 | ( |
| 1488 | "child_process", |
| 1489 | r#" |
| 1490 | if (typeof require === "function") { |
| 1491 | return require("child_process"); |
| 1492 | } |
| 1493 | if (typeof child_process !== "undefined") { |
| 1494 | return child_process; |
| 1495 | } |
| 1496 | throw new Error("child_process is unavailable"); |
| 1497 | "#, |
| 1498 | ), |
| 1499 | ( |
| 1500 | "Deno.env", |
| 1501 | r#" |
| 1502 | if (typeof Deno !== "undefined") { |
| 1503 | return Deno.env.toObject(); |
| 1504 | } |
| 1505 | throw new Error("Deno is unavailable"); |
| 1506 | "#, |
| 1507 | ), |
| 1508 | ]; |
| 1509 | |
| 1510 | for (label, source) in probes { |
| 1511 | let driver = Arc::new(FakeDriver::new()); |
| 1512 | let result = run(&driver, source, json!(null)).await; |
| 1513 | assert!( |
| 1514 | result.is_err(), |
| 1515 | "sandbox probe `{label}` must fail closed, got {result:?}" |
| 1516 | ); |
| 1517 | // No driver side-effect is expected from a sandbox probe. |
| 1518 | assert_eq!( |
| 1519 | driver.spawn_count(), |
| 1520 | 0, |
| 1521 | "probe `{label}` must not spawn tasks" |
| 1522 | ); |
| 1523 | } |
| 1524 | } |
| 1525 | |
| 1526 | #[tokio::test] |
| 1527 | async fn sandbox_global_inventory_fails_closed_on_new_host_leaks() { |
| 1528 | let driver = Arc::new(FakeDriver::new()); |
| 1529 | let value = run( |
| 1530 | &driver, |
| 1531 | r#" |
| 1532 | // Own enumerable + non-enumerable names on the global object. |
| 1533 | // Anything beyond standard ECMAScript + the Workflow allowlist is a |
| 1534 | // regression that must break this test so new leaks cannot land quietly. |
| 1535 | const names = Reflect.ownKeys(globalThis) |
| 1536 | .map((k) => String(k)) |
| 1537 | .sort(); |
| 1538 | return names; |
| 1539 | "#, |
| 1540 | json!(null), |
| 1541 | ) |
| 1542 | .await |
| 1543 | .unwrap(); |
| 1544 | let names: Vec<String> = serde_json::from_value(value).expect("name list is a JSON array"); |
| 1545 | |
| 1546 | // Fail closed: none of the banned host surfaces may appear. |
| 1547 | for banned in SANDBOX_BANNED_GLOBALS { |
| 1548 | assert!( |
| 1549 | !names.iter().any(|n| n == *banned), |
| 1550 | "banned global `{banned}` leaked into the Workflow VM: {names:?}" |
| 1551 | ); |
| 1552 | } |
| 1553 | |
| 1554 | // Every Workflow-owned call must still be present. |
| 1555 | for allowed in WORKFLOW_ALLOWED_GLOBALS { |
| 1556 | assert!( |
| 1557 | names.iter().any(|n| n == *allowed), |
| 1558 | "expected Workflow global `{allowed}` missing from inventory: {names:?}" |
| 1559 | ); |
| 1560 | } |
| 1561 | |
| 1562 | // Internal host helpers must not be script-visible. |
| 1563 | for internal in [ |
| 1564 | "__workflow_task", |
| 1565 | "__workflow_log", |
| 1566 | "__workflow_every_slot_failed", |
| 1567 | "__workflow_phase", |
| 1568 | "__workflow_budget_total", |
| 1569 | "__workflow_budget_spent", |
| 1570 | "__workflow_budget_remaining", |
| 1571 | ] { |
| 1572 | assert!( |
| 1573 | !names.iter().any(|n| n == internal), |
| 1574 | "internal host binding `{internal}` must stay hidden: {names:?}" |
| 1575 | ); |
| 1576 | } |
| 1577 | } |
| 1578 | |
| 1579 | #[tokio::test] |
| 1580 | async fn sandbox_rejects_commonjs_module_loader_and_eval_style_constructors() { |
| 1581 | let driver = Arc::new(FakeDriver::new()); |
| 1582 | // `eval` / `Function` are standard ES, but if they are present they must |
| 1583 | // still be unable to reach host modules. The banned-global inventory above |
| 1584 | // already fails closed if Node-style loaders appear; this probe documents |
| 1585 | // the intended product message for module load attempts. |
| 1586 | let message = script_message( |
| 1587 | run( |
| 1588 | &driver, |
| 1589 | r#" |
| 1590 | if (typeof require === "function") { |
| 1591 | return require("node:fs"); |
| 1592 | } |
| 1593 | throw new Error("require is unavailable"); |
| 1594 | "#, |
| 1595 | json!(null), |
| 1596 | ) |
| 1597 | .await, |
| 1598 | ); |
| 1599 | assert!( |
| 1600 | message.contains("unavailable") || message.contains("require"), |
| 1601 | "{message}" |
| 1602 | ); |
| 1603 | } |
| 1604 | |
| 1605 | #[tokio::test] |
| 1606 | async fn dropping_the_run_future_cancels_outstanding_tasks() { |
| 1607 | let driver = Arc::new(FakeDriver::new()); |
| 1608 | driver.on("hang", FakeReply::Never); |
| 1609 | let vm = WorkflowVm::new(); |
| 1610 | { |
| 1611 | let fut = vm.run_script( |
| 1612 | "await task({ description: 'hang forever' }); return 'unreachable';", |
| 1613 | json!(null), |
| 1614 | driver.clone() as Arc<dyn codewhale_workflow_js::WorkflowDriver>, |
| 1615 | ); |
| 1616 | let outcome = tokio::time::timeout(Duration::from_millis(400), fut).await; |
| 1617 | assert!(outcome.is_err(), "run should still be pending at timeout"); |
| 1618 | // The timed-out future is dropped here. |
| 1619 | } |
| 1620 | assert!( |
| 1621 | driver.cancel_all_calls() >= 1, |
| 1622 | "dropping the run future must cancel outstanding driver tasks" |
| 1623 | ); |
| 1624 | assert_eq!(driver.spawn_count(), 1); |
| 1625 | } |
| 1626 | |
| 1627 | #[tokio::test] |
| 1628 | async fn parallel_does_not_continue_after_external_run_cancellation() { |
| 1629 | let driver = Arc::new(FakeDriver::new()); |
| 1630 | driver.on("hang", FakeReply::Never); |
| 1631 | let cancel = WorkflowRunCancel::new(); |
| 1632 | let run_cancel = cancel.clone(); |
| 1633 | let run_driver = driver.clone(); |
| 1634 | let handle = tokio::spawn(async move { |
| 1635 | WorkflowVm::new() |
| 1636 | .run_script_with_cancel( |
| 1637 | r#" |
| 1638 | await parallel([() => task({ description: "hang" })]); |
| 1639 | phase("unreachable after cancellation"); |
| 1640 | return "wrong"; |
| 1641 | "#, |
| 1642 | json!(null), |
| 1643 | run_driver as Arc<dyn codewhale_workflow_js::WorkflowDriver>, |
| 1644 | run_cancel, |
| 1645 | ) |
| 1646 | .await |
| 1647 | }); |
| 1648 | |
| 1649 | tokio::time::timeout(Duration::from_secs(2), async { |
| 1650 | while driver.spawn_count() == 0 { |
| 1651 | tokio::task::yield_now().await; |
| 1652 | } |
| 1653 | }) |
| 1654 | .await |
| 1655 | .expect("task should start"); |
| 1656 | cancel.cancel(); |
| 1657 | |
| 1658 | let result = handle.await.expect("VM task should join"); |
| 1659 | assert!( |
| 1660 | matches!(result, Err(WorkflowJsError::Cancelled)), |
| 1661 | "{result:?}" |
| 1662 | ); |
| 1663 | assert!( |
| 1664 | !driver.events().iter().any(|event| matches!( |
| 1665 | event, |
| 1666 | ProgressEvent::Phase { title } if title == "unreachable after cancellation" |
| 1667 | )), |
| 1668 | "parallel() must not downgrade run cancellation into a null slot" |
| 1669 | ); |
| 1670 | } |
| 1671 | |
| 1672 | #[tokio::test] |
| 1673 | async fn script_error_rejects_cleanly_and_cancels_children() { |
| 1674 | let driver = Arc::new(FakeDriver::new()); |
| 1675 | let result = run( |
| 1676 | &driver, |
| 1677 | r#"await task({ description: "quick" }); throw new Error("boom");"#, |
| 1678 | json!(null), |
| 1679 | ) |
| 1680 | .await; |
| 1681 | let message = script_message(result); |
| 1682 | assert!(message.contains("boom"), "{message}"); |
| 1683 | assert!( |
| 1684 | driver.cancel_all_calls() >= 1, |
| 1685 | "a failed run must cancel its cascade" |
| 1686 | ); |
| 1687 | } |
| 1688 | |
| 1689 | #[tokio::test] |
| 1690 | async fn log_and_phase_events_reach_the_driver_in_order() { |
| 1691 | let driver = Arc::new(FakeDriver::new()); |
| 1692 | run( |
| 1693 | &driver, |
| 1694 | r#" |
| 1695 | phase("scan"); |
| 1696 | log("a"); |
| 1697 | log({ found: 2 }); |
| 1698 | phase("verify"); |
| 1699 | log("b"); |
| 1700 | return null; |
| 1701 | "#, |
| 1702 | json!(null), |
| 1703 | ) |
| 1704 | .await |
| 1705 | .unwrap(); |
| 1706 | assert_eq!( |
| 1707 | driver.events(), |
| 1708 | vec![ |
| 1709 | ProgressEvent::Phase { |
| 1710 | title: "scan".to_string() |
| 1711 | }, |
| 1712 | ProgressEvent::Log { |
| 1713 | message: "a".to_string() |
| 1714 | }, |
| 1715 | ProgressEvent::Log { |
| 1716 | message: r#"{"found":2}"#.to_string() |
| 1717 | }, |
| 1718 | ProgressEvent::Phase { |
| 1719 | title: "verify".to_string() |
| 1720 | }, |
| 1721 | ProgressEvent::Log { |
| 1722 | message: "b".to_string() |
| 1723 | }, |
| 1724 | ] |
| 1725 | ); |
| 1726 | } |
| 1727 | |
| 1728 | #[tokio::test] |
| 1729 | async fn promise_all_of_tasks_resolves_concurrently() { |
| 1730 | let driver = Arc::new(FakeDriver::new()); |
| 1731 | driver.on_with_delay( |
| 1732 | "left", |
| 1733 | FakeReply::Complete("L".to_string()), |
| 1734 | Duration::from_millis(50), |
| 1735 | ); |
| 1736 | driver.on_with_delay( |
| 1737 | "right", |
| 1738 | FakeReply::Complete("R".to_string()), |
| 1739 | Duration::from_millis(50), |
| 1740 | ); |
| 1741 | let started = std::time::Instant::now(); |
| 1742 | let value = run( |
| 1743 | &driver, |
| 1744 | r#" |
| 1745 | const [a, b] = await Promise.all([ |
| 1746 | task({ description: "left" }), |
| 1747 | task({ description: "right" }), |
| 1748 | ]); |
| 1749 | return a + "/" + b; |
| 1750 | "#, |
| 1751 | json!(null), |
| 1752 | ) |
| 1753 | .await |
| 1754 | .unwrap(); |
| 1755 | assert_eq!(value, json!("L/R")); |
| 1756 | // Two 50ms tasks awaited concurrently should not take ~100ms serially. |
| 1757 | // Generous bound to stay green on slow CI. |
| 1758 | assert!( |
| 1759 | started.elapsed() < Duration::from_millis(3000), |
| 1760 | "took {:?}", |
| 1761 | started.elapsed() |
| 1762 | ); |
| 1763 | assert_eq!(driver.spawn_count(), 2); |
| 1764 | } |
| 1765 | |
| 1766 | #[tokio::test] |
| 1767 | async fn export_default_async_function_runs_with_args() { |
| 1768 | let driver = Arc::new(FakeDriver::new()); |
| 1769 | let source = r#" |
| 1770 | export default async function (args) { |
| 1771 | return { doubled: args.n * 2 }; |
| 1772 | } |
| 1773 | "#; |
| 1774 | let value = run(&driver, source, json!({ "n": 21 })).await.unwrap(); |
| 1775 | assert_eq!(value, json!({ "doubled": 42 })); |
| 1776 | } |
| 1777 | |
| 1778 | #[tokio::test] |
| 1779 | async fn export_default_function_result_becomes_run_result() { |
| 1780 | let driver = Arc::new(FakeDriver::new()); |
| 1781 | let source = r#" |
| 1782 | function helper() { |
| 1783 | return "from-helper"; |
| 1784 | } |
| 1785 | export default function () { |
| 1786 | return helper(); |
| 1787 | } |
| 1788 | "#; |
| 1789 | let value = run(&driver, source, json!(null)).await.unwrap(); |
| 1790 | assert_eq!(value, json!("from-helper")); |
| 1791 | } |
| 1792 | |
| 1793 | #[tokio::test] |
| 1794 | async fn export_default_non_function_value_is_returned() { |
| 1795 | let driver = Arc::new(FakeDriver::new()); |
| 1796 | let value = run(&driver, "export default 7;", json!(null)) |
| 1797 | .await |
| 1798 | .unwrap(); |
| 1799 | assert_eq!(value, json!(7)); |
| 1800 | } |
| 1801 | |
| 1802 | #[tokio::test] |
| 1803 | async fn plain_scripts_are_untouched_by_export_desugaring() { |
| 1804 | let driver = Arc::new(FakeDriver::new()); |
| 1805 | // A string literal mentioning `export default` must not trigger the |
| 1806 | // module desugaring path. |
| 1807 | let value = run( |
| 1808 | &driver, |
| 1809 | "const note = \"export default docs\";\nreturn note.length;", |
| 1810 | json!(null), |
| 1811 | ) |
| 1812 | .await |
| 1813 | .unwrap(); |
| 1814 | assert_eq!(value, json!(19)); |
| 1815 | } |
| 1816 | |
| 1817 | #[tokio::test] |
| 1818 | async fn export_default_examples_inside_multiline_text_are_not_desugared() { |
| 1819 | let driver = Arc::new(FakeDriver::new()); |
| 1820 | let value = run( |
| 1821 | &driver, |
| 1822 | r#" |
| 1823 | const template = ` |
| 1824 | export default async function (args) { |
| 1825 | return args; |
| 1826 | } |
| 1827 | `; |
| 1828 | /* |
| 1829 | export default function () { |
| 1830 | return "comment example"; |
| 1831 | } |
| 1832 | */ |
| 1833 | return template.includes("export default async function"); |
| 1834 | "#, |
| 1835 | json!(null), |
| 1836 | ) |
| 1837 | .await |
| 1838 | .unwrap(); |
| 1839 | assert_eq!(value, json!(true)); |
| 1840 | } |
| 1841 | |
| 1842 | #[tokio::test] |
| 1843 | async fn task_accepts_agent_tool_spellings() { |
| 1844 | // The `agent` tool and `task()` are written by the same authors; a schema |
| 1845 | // that runs on one surface must not be an unknown-field error on the |
| 1846 | // other. snake_case spellings and `workspace_policy` are aliases. |
| 1847 | let driver = Arc::new(FakeDriver::new()); |
| 1848 | let value = run( |
| 1849 | &driver, |
| 1850 | r#" |
| 1851 | return await task({ |
| 1852 | prompt: "cross-surface schema", |
| 1853 | subagent_type: "implementer", |
| 1854 | workspace_policy: "worktree", |
| 1855 | write_authority: "worktree_write", |
| 1856 | write_roots: ["crates/tui/src"], |
| 1857 | token_budget: 5000, |
| 1858 | max_steps: 4, |
| 1859 | }); |
| 1860 | "#, |
| 1861 | json!(null), |
| 1862 | ) |
| 1863 | .await |
| 1864 | .unwrap(); |
| 1865 | assert_eq!(value, json!("done:cross-surface schema")); |
| 1866 | let requests = driver.requests(); |
| 1867 | assert_eq!(requests.len(), 1); |
| 1868 | assert!( |
| 1869 | requests[0].worktree, |
| 1870 | "workspace_policy worktree maps to worktree isolation" |
| 1871 | ); |
| 1872 | assert_eq!( |
| 1873 | requests[0].write_authority.as_deref(), |
| 1874 | Some("worktree_write") |
| 1875 | ); |
| 1876 | assert_eq!(requests[0].token_budget, Some(5000)); |
| 1877 | |
| 1878 | // "shared" is accepted and stays non-worktree; contradictions and unknown |
| 1879 | // values still fail loudly. |
| 1880 | let error = run( |
| 1881 | &driver, |
| 1882 | r#"return await task({ prompt: "x", workspacePolicy: "shared", worktree: true });"#, |
| 1883 | json!(null), |
| 1884 | ) |
| 1885 | .await |
| 1886 | .unwrap_err(); |
| 1887 | assert!(script_message(Err(error)).contains("conflicts with worktree")); |
| 1888 | let error = run( |
| 1889 | &driver, |
| 1890 | r#"return await task({ prompt: "x", workspacePolicy: "solo" });"#, |
| 1891 | json!(null), |
| 1892 | ) |
| 1893 | .await |
| 1894 | .unwrap_err(); |
| 1895 | assert!(script_message(Err(error)).contains("must be shared or worktree")); |
| 1896 | } |
| 1897 | |
| 1898 | #[tokio::test] |
| 1899 | async fn vm_rejected_task_options_notify_the_driver() { |
| 1900 | // A task() whose options fail VM validation throws before spawn_task, and |
| 1901 | // inside parallel() that throw collapses to a null slot. The driver must |
| 1902 | // still receive a TaskRejected event so the run record can refuse to call |
| 1903 | // the run a plain success (morning-report issue #2). |
| 1904 | let driver = Arc::new(FakeDriver::new()); |
| 1905 | let value = run( |
| 1906 | &driver, |
| 1907 | r#" |
| 1908 | return await parallel([ |
| 1909 | () => task({ prompt: "bad slot", label: "L-bad", phase: "P1", cwd: "/absolute/path" }), |
| 1910 | ]); |
| 1911 | "#, |
| 1912 | json!(null), |
| 1913 | ) |
| 1914 | .await |
| 1915 | .unwrap(); |
| 1916 | assert_eq!(value, json!([null])); |
| 1917 | assert!( |
| 1918 | driver.requests().is_empty(), |
| 1919 | "no dispatch reached the driver" |
| 1920 | ); |
| 1921 | let rejected: Vec<_> = driver |
| 1922 | .events() |
| 1923 | .into_iter() |
| 1924 | .filter_map(|event| match event { |
| 1925 | ProgressEvent::TaskRejected { |
| 1926 | label, |
| 1927 | phase, |
| 1928 | message, |
| 1929 | } => Some((label, phase, message)), |
| 1930 | _ => None, |
| 1931 | }) |
| 1932 | .collect(); |
| 1933 | assert_eq!(rejected.len(), 1, "one rejection event per refused slot"); |
| 1934 | let (label, phase, message) = &rejected[0]; |
| 1935 | assert_eq!(label.as_deref(), Some("L-bad")); |
| 1936 | assert_eq!(phase.as_deref(), Some("P1")); |
| 1937 | assert!(message.contains("bounded repo-relative paths"), "{message}"); |
| 1938 | } |
| 1939 | |
| 1940 | // --------------------------------------------------------------------------- |
| 1941 | // R9: typed slot errors, inspectable settled failures, explicit modes. |
| 1942 | // --------------------------------------------------------------------------- |
| 1943 | |
| 1944 | /// Every way a `task()` can die gets its own kind, assigned by the host where |
| 1945 | /// the failure happened. Before R9 all six collapsed into two buckets |
| 1946 | /// ("budget"/"cancelled" if the message happened to say so, "task" otherwise), |
| 1947 | /// so a dead subagent and a typo'd script throw were the same thing. |
| 1948 | #[tokio::test] |
| 1949 | async fn every_task_failure_mode_carries_its_own_kind() { |
| 1950 | let driver = Arc::new(FakeDriver::new()); |
| 1951 | driver.on("agent case", FakeReply::Fail("boom".to_string())); |
| 1952 | driver.on( |
| 1953 | "budget case", |
| 1954 | FakeReply::BudgetExhausted("limit 10".to_string()), |
| 1955 | ); |
| 1956 | driver.on("cancelled case", FakeReply::Cancelled); |
| 1957 | driver.on( |
| 1958 | "admission case", |
| 1959 | FakeReply::Reject("admission cap".to_string()), |
| 1960 | ); |
| 1961 | driver.on( |
| 1962 | "driver case", |
| 1963 | FakeReply::Unavailable("driver gone".to_string()), |
| 1964 | ); |
| 1965 | driver.on("dropped case", FakeReply::DropCompletion); |
| 1966 | driver.on( |
| 1967 | "schema case", |
| 1968 | FakeReply::Complete("not json at all".to_string()), |
| 1969 | ); |
| 1970 | |
| 1971 | let value = run( |
| 1972 | &driver, |
| 1973 | r#" |
| 1974 | const kinds = {}; |
| 1975 | const probe = async (name, opts) => { |
| 1976 | try { |
| 1977 | await task(opts); |
| 1978 | kinds[name] = "none"; |
| 1979 | } catch (err) { |
| 1980 | kinds[name] = err && err.kind; |
| 1981 | } |
| 1982 | }; |
| 1983 | await probe("agent", { description: "agent case" }); |
| 1984 | await probe("budget", { description: "budget case" }); |
| 1985 | await probe("cancelled", { description: "cancelled case" }); |
| 1986 | await probe("admission", { description: "admission case" }); |
| 1987 | await probe("driver", { description: "driver case" }); |
| 1988 | await probe("dropped", { description: "dropped case" }); |
| 1989 | await probe("schema", { |
| 1990 | description: "schema case", |
| 1991 | schemaRepairAttempts: 0, |
| 1992 | responseSchema: { type: "object" }, |
| 1993 | }); |
| 1994 | // A malformed options object never reaches a child either. |
| 1995 | await probe("bad-options", { description: "x", nosuchoption: 1 }); |
| 1996 | try { |
| 1997 | await task("not an object"); |
| 1998 | kinds["not-an-object"] = "none"; |
| 1999 | } catch (err) { |
| 2000 | kinds["not-an-object"] = String(err && err.kind); |
| 2001 | } |
| 2002 | return kinds; |
| 2003 | "#, |
| 2004 | json!(null), |
| 2005 | ) |
| 2006 | .await |
| 2007 | .unwrap(); |
| 2008 | |
| 2009 | assert_eq!( |
| 2010 | value, |
| 2011 | json!({ |
| 2012 | "agent": "agent", |
| 2013 | "budget": "budget", |
| 2014 | "cancelled": "cancelled", |
| 2015 | "admission": "admission", |
| 2016 | "driver": "driver", |
| 2017 | "dropped": "driver", |
| 2018 | "schema": "schema", |
| 2019 | "bad-options": "admission", |
| 2020 | // A TypeError raised by the prelude's own argument check never |
| 2021 | // came from the host, so it is a script error, not a task kind. |
| 2022 | "not-an-object": "undefined", |
| 2023 | }) |
| 2024 | ); |
| 2025 | } |
| 2026 | |
| 2027 | /// The classifier reads `Error.kind`, never the message text. A child is free |
| 2028 | /// to say "budget exhausted" or "responseSchema" in its own failure prose; |
| 2029 | /// under the old substring classifier that forged a fatal kind and aborted an |
| 2030 | /// otherwise healthy fan-out. |
| 2031 | #[tokio::test] |
| 2032 | async fn slot_kinds_cannot_be_forged_from_child_failure_text() { |
| 2033 | let driver = Arc::new(FakeDriver::new()); |
| 2034 | driver.on( |
| 2035 | "liar", |
| 2036 | FakeReply::Fail( |
| 2037 | "the reviewer said the run cancelled because budget exhausted and responseSchema \ |
| 2038 | validation failed" |
| 2039 | .to_string(), |
| 2040 | ), |
| 2041 | ); |
| 2042 | |
| 2043 | let value = run( |
| 2044 | &driver, |
| 2045 | r#" |
| 2046 | const results = await parallel([ |
| 2047 | () => task({ description: "liar" }), |
| 2048 | () => task({ description: "honest" }), |
| 2049 | ]); |
| 2050 | return { |
| 2051 | slots: results, |
| 2052 | kinds: results.errors.map((entry) => entry.kind), |
| 2053 | }; |
| 2054 | "#, |
| 2055 | json!(null), |
| 2056 | ) |
| 2057 | .await |
| 2058 | .expect("a child's prose must not cancel the run"); |
| 2059 | |
| 2060 | assert_eq!( |
| 2061 | value, |
| 2062 | json!({ |
| 2063 | "slots": [null, "done:honest"], |
| 2064 | "kinds": ["agent"], |
| 2065 | }) |
| 2066 | ); |
| 2067 | } |
| 2068 | |
| 2069 | /// The settled default is unchanged on the wire — same slots, same length, |
| 2070 | /// same JSON — but the run can now ask why a slot is null instead of guessing. |
| 2071 | #[tokio::test] |
| 2072 | async fn settled_parallel_keeps_null_slots_and_attaches_an_inspectable_ledger() { |
| 2073 | let driver = Arc::new(FakeDriver::new()); |
| 2074 | driver.on("beta", FakeReply::Fail("boom".to_string())); |
| 2075 | driver.on( |
| 2076 | "delta", |
| 2077 | FakeReply::BudgetExhausted("pool drained".to_string()), |
| 2078 | ); |
| 2079 | |
| 2080 | let value = run( |
| 2081 | &driver, |
| 2082 | r#" |
| 2083 | const results = await parallel([ |
| 2084 | () => task({ description: "alpha" }), |
| 2085 | () => task({ description: "beta" }), |
| 2086 | () => task({ description: "gamma" }), |
| 2087 | () => task({ description: "delta" }), |
| 2088 | ]); |
| 2089 | return { |
| 2090 | slots: results, |
| 2091 | length: results.length, |
| 2092 | // Non-enumerable: the array still serializes as a plain array. |
| 2093 | encoded: JSON.stringify(results), |
| 2094 | errors: results.errors.map((entry) => ({ |
| 2095 | index: entry.index, |
| 2096 | kind: entry.kind, |
| 2097 | says: entry.message.indexOf("boom") !== -1 |
| 2098 | || entry.message.indexOf("pool drained") !== -1, |
| 2099 | })), |
| 2100 | }; |
| 2101 | "#, |
| 2102 | json!(null), |
| 2103 | ) |
| 2104 | .await |
| 2105 | .unwrap(); |
| 2106 | |
| 2107 | assert_eq!( |
| 2108 | value, |
| 2109 | json!({ |
| 2110 | "slots": ["done:alpha", null, "done:gamma", null], |
| 2111 | "length": 4, |
| 2112 | "encoded": "[\"done:alpha\",null,\"done:gamma\",null]", |
| 2113 | "errors": [ |
| 2114 | {"index": 1, "kind": "agent", "says": true}, |
| 2115 | {"index": 3, "kind": "budget", "says": true}, |
| 2116 | ], |
| 2117 | }) |
| 2118 | ); |
| 2119 | } |
| 2120 | |
| 2121 | /// A clean fan-out still gets the ledger, empty and frozen — a script can read |
| 2122 | /// `results.errors.length` unconditionally. |
| 2123 | #[tokio::test] |
| 2124 | async fn a_clean_fan_out_still_reports_an_empty_frozen_error_ledger() { |
| 2125 | let value = run( |
| 2126 | &Arc::new(FakeDriver::new()), |
| 2127 | r#" |
| 2128 | const results = await parallel([() => task({ description: "alpha" })]); |
| 2129 | let mutated = false; |
| 2130 | try { |
| 2131 | results.errors = ["forged"]; |
| 2132 | mutated = true; |
| 2133 | } catch (_) { |
| 2134 | mutated = false; |
| 2135 | } |
| 2136 | return { |
| 2137 | count: results.errors.length, |
| 2138 | frozen: Object.isFrozen(results.errors), |
| 2139 | mutated: mutated, |
| 2140 | stillEmpty: results.errors.length, |
| 2141 | }; |
| 2142 | "#, |
| 2143 | json!(null), |
| 2144 | ) |
| 2145 | .await |
| 2146 | .unwrap(); |
| 2147 | assert_eq!( |
| 2148 | value, |
| 2149 | json!({"count": 0, "frozen": true, "mutated": false, "stillEmpty": 0}) |
| 2150 | ); |
| 2151 | } |
| 2152 | |
| 2153 | /// `settled` is the spelling of today's default; naming it explicitly changes |
| 2154 | /// nothing. |
| 2155 | #[tokio::test] |
| 2156 | async fn explicit_settled_mode_matches_the_default() { |
| 2157 | let driver = Arc::new(FakeDriver::new()); |
| 2158 | driver.on("beta", FakeReply::Fail("boom".to_string())); |
| 2159 | let value = run( |
| 2160 | &driver, |
| 2161 | r#" |
| 2162 | const thunks = () => [ |
| 2163 | () => task({ description: "alpha" }), |
| 2164 | () => task({ description: "beta" }), |
| 2165 | ]; |
| 2166 | const implicit = await parallel(thunks()); |
| 2167 | const explicit = await parallel(thunks(), { mode: "settled" }); |
| 2168 | return { |
| 2169 | implicit: implicit, |
| 2170 | explicit: explicit, |
| 2171 | sameKinds: JSON.stringify(implicit.errors.map((e) => e.kind)) |
| 2172 | === JSON.stringify(explicit.errors.map((e) => e.kind)), |
| 2173 | }; |
| 2174 | "#, |
| 2175 | json!(null), |
| 2176 | ) |
| 2177 | .await |
| 2178 | .unwrap(); |
| 2179 | assert_eq!( |
| 2180 | value, |
| 2181 | json!({ |
| 2182 | "implicit": ["done:alpha", null], |
| 2183 | "explicit": ["done:alpha", null], |
| 2184 | "sameKinds": true, |
| 2185 | }) |
| 2186 | ); |
| 2187 | } |
| 2188 | |
| 2189 | /// A typo'd mode used to read as `settled`: the author believed slots were |
| 2190 | /// now fatal while they kept silently dropping. It throws instead. |
| 2191 | #[tokio::test] |
| 2192 | async fn an_unknown_mode_is_refused_rather_than_silently_settled() { |
| 2193 | let driver = Arc::new(FakeDriver::new()); |
| 2194 | let value = run( |
| 2195 | &driver, |
| 2196 | r#" |
| 2197 | const errs = []; |
| 2198 | for (const mode of ["failfast", "all-settled", 7]) { |
| 2199 | try { |
| 2200 | await parallel([() => task({ description: "alpha" })], { mode }); |
| 2201 | errs.push("no-error"); |
| 2202 | } catch (err) { |
| 2203 | errs.push(err.message); |
| 2204 | } |
| 2205 | } |
| 2206 | try { |
| 2207 | await pipeline([1], { stages: [(v) => v], mode: "failfast" }); |
| 2208 | errs.push("no-error"); |
| 2209 | } catch (err) { |
| 2210 | errs.push(err.message); |
| 2211 | } |
| 2212 | return errs; |
| 2213 | "#, |
| 2214 | json!(null), |
| 2215 | ) |
| 2216 | .await |
| 2217 | .unwrap(); |
| 2218 | let messages = value.as_array().unwrap(); |
| 2219 | assert_eq!(messages.len(), 4, "{value}"); |
| 2220 | for message in messages { |
| 2221 | let text = message.as_str().unwrap(); |
| 2222 | assert!( |
| 2223 | text.contains("unknown mode") && text.contains("settled, fail-fast, partial"), |
| 2224 | "{text}" |
| 2225 | ); |
| 2226 | } |
| 2227 | assert_eq!( |
| 2228 | driver.spawn_count(), |
| 2229 | 0, |
| 2230 | "a refused mode must not spawn anything" |
| 2231 | ); |
| 2232 | } |
| 2233 | |
| 2234 | /// Partial mode is the "inspect every outcome" contract: no failure is erased, |
| 2235 | /// and none of them can be mistaken for a value. |
| 2236 | #[tokio::test] |
| 2237 | async fn partial_mode_types_every_non_cancellation_failure() { |
| 2238 | let driver = Arc::new(FakeDriver::new()); |
| 2239 | driver.on("dead", FakeReply::Fail("boom".to_string())); |
| 2240 | driver.on("broke", FakeReply::BudgetExhausted("drained".to_string())); |
| 2241 | driver.on("refused", FakeReply::Reject("admission cap".to_string())); |
| 2242 | |
| 2243 | let value = run( |
| 2244 | &driver, |
| 2245 | r#" |
| 2246 | const results = await parallel([ |
| 2247 | () => task({ description: "alive" }), |
| 2248 | () => task({ description: "dead" }), |
| 2249 | () => task({ description: "broke" }), |
| 2250 | () => task({ description: "refused" }), |
| 2251 | ], { mode: "partial" }); |
| 2252 | return { |
| 2253 | shapes: results.map((slot) => |
| 2254 | slot && typeof slot === "object" && slot.__taskError |
| 2255 | ? slot.__taskError.kind + "@" + slot.__taskError.index |
| 2256 | : String(slot) |
| 2257 | ), |
| 2258 | ledger: results.errors.map((entry) => entry.kind), |
| 2259 | noNulls: results.every((slot) => slot !== null), |
| 2260 | }; |
| 2261 | "#, |
| 2262 | json!(null), |
| 2263 | ) |
| 2264 | .await |
| 2265 | .expect("partial mode completes the fan-out"); |
| 2266 | |
| 2267 | assert_eq!( |
| 2268 | value, |
| 2269 | json!({ |
| 2270 | "shapes": ["done:alive", "agent@1", "budget@2", "admission@3"], |
| 2271 | "ledger": ["agent", "budget", "admission"], |
| 2272 | "noNulls": true, |
| 2273 | }) |
| 2274 | ); |
| 2275 | } |
| 2276 | |
| 2277 | /// `pipeline` speaks the same three modes and keeps the same ledger. |
| 2278 | #[tokio::test] |
| 2279 | async fn pipeline_supports_settled_fail_fast_and_partial_with_a_ledger() { |
| 2280 | let driver = Arc::new(FakeDriver::new()); |
| 2281 | driver.on("bad-1", FakeReply::Fail("boom".to_string())); |
| 2282 | |
| 2283 | let value = run( |
| 2284 | &driver, |
| 2285 | r#" |
| 2286 | const stage = (item) => task({ description: item }); |
| 2287 | const settled = await pipeline(["ok-0", "bad-1", "ok-2"], stage); |
| 2288 | const partial = await pipeline(["ok-0", "bad-1"], { |
| 2289 | stages: [stage], |
| 2290 | mode: "partial", |
| 2291 | }); |
| 2292 | let failFast = "no-error"; |
| 2293 | try { |
| 2294 | await pipeline(["ok-0", "bad-1"], { stages: [stage], mode: "fail-fast" }); |
| 2295 | } catch (err) { |
| 2296 | failFast = err.kind + ":" + (err.message.indexOf("boom") !== -1); |
| 2297 | } |
| 2298 | return { |
| 2299 | settled: settled, |
| 2300 | settledLedger: settled.errors.map((e) => e.index + ":" + e.kind), |
| 2301 | partial: partial.map((slot) => |
| 2302 | slot && typeof slot === "object" && slot.__taskError |
| 2303 | ? slot.__taskError.kind |
| 2304 | : slot |
| 2305 | ), |
| 2306 | failFast: failFast, |
| 2307 | }; |
| 2308 | "#, |
| 2309 | json!(null), |
| 2310 | ) |
| 2311 | .await |
| 2312 | .unwrap(); |
| 2313 | |
| 2314 | assert_eq!( |
| 2315 | value, |
| 2316 | json!({ |
| 2317 | "settled": ["done:ok-0", null, "done:ok-2"], |
| 2318 | "settledLedger": ["1:agent"], |
| 2319 | "partial": ["done:ok-0", "agent"], |
| 2320 | "failFast": "agent:true", |
| 2321 | }) |
| 2322 | ); |
| 2323 | } |
| 2324 | |
| 2325 | /// A fan-out where nothing survived is a dead fan-out. The default still |
| 2326 | /// resolves (existing scripts keep working) but the run log says so in a line |
| 2327 | /// the host status classifier and an operator can both find. |
| 2328 | #[tokio::test] |
| 2329 | async fn a_fan_out_where_every_slot_failed_says_so_in_the_run_log() { |
| 2330 | let driver = Arc::new(FakeDriver::new()); |
| 2331 | driver.on("doomed", FakeReply::Fail("boom".to_string())); |
| 2332 | let value = run( |
| 2333 | &driver, |
| 2334 | r#" |
| 2335 | const results = await parallel([ |
| 2336 | () => task({ description: "doomed a" }), |
| 2337 | () => task({ description: "doomed b" }), |
| 2338 | ]); |
| 2339 | return { slots: results, failed: results.errors.length }; |
| 2340 | "#, |
| 2341 | json!(null), |
| 2342 | ) |
| 2343 | .await |
| 2344 | .unwrap(); |
| 2345 | assert_eq!(value, json!({"slots": [null, null], "failed": 2})); |
| 2346 | assert!( |
| 2347 | driver.events().iter().any(|event| matches!( |
| 2348 | event, |
| 2349 | ProgressEvent::Log { message } |
| 2350 | if message.contains("every slot failed (2 of 2)") |
| 2351 | && message.contains("no work survived") |
| 2352 | )), |
| 2353 | "a fully-failed fan-out must be named in the run log: {:?}", |
| 2354 | driver.events() |
| 2355 | ); |
| 2356 | } |
| 2357 | |
| 2358 | /// Cancellation stays fatal in every mode — it is the run's deadline, not a |
| 2359 | /// per-slot outcome — and partial mode does not get to keep it as a value. |
| 2360 | #[tokio::test] |
| 2361 | async fn cancellation_is_fatal_in_partial_pipeline_mode() { |
| 2362 | let driver = Arc::new(FakeDriver::new()); |
| 2363 | driver.on("hang", FakeReply::Never); |
| 2364 | let cancel = WorkflowRunCancel::new(); |
| 2365 | let run_cancel = cancel.clone(); |
| 2366 | let run_driver = driver.clone(); |
| 2367 | let handle = tokio::spawn(async move { |
| 2368 | WorkflowVm::new() |
| 2369 | .run_script_with_cancel( |
| 2370 | r#" |
| 2371 | await pipeline(["hang"], { |
| 2372 | stages: [(item) => task({ description: item })], |
| 2373 | mode: "partial", |
| 2374 | }); |
| 2375 | "#, |
| 2376 | json!(null), |
| 2377 | run_driver as Arc<dyn codewhale_workflow_js::WorkflowDriver>, |
| 2378 | run_cancel, |
| 2379 | ) |
| 2380 | .await |
| 2381 | }); |
| 2382 | |
| 2383 | tokio::time::timeout(Duration::from_secs(2), async { |
| 2384 | while driver.spawn_count() == 0 { |
| 2385 | tokio::task::yield_now().await; |
| 2386 | } |
| 2387 | }) |
| 2388 | .await |
| 2389 | .expect("task should start"); |
| 2390 | cancel.cancel(); |
| 2391 | |
| 2392 | let result = handle.await.expect("VM task should join"); |
| 2393 | assert!( |
| 2394 | matches!(result, Err(WorkflowJsError::Cancelled)), |
| 2395 | "pipeline partial mode must not downgrade cancellation into a slot value: {result:?}" |
| 2396 | ); |
| 2397 | } |
| 2398 | |
| 2399 | /// The dropped-slot breadcrumb now names the kind, so the run log alone |
| 2400 | /// distinguishes "the agent failed" from "we never got to spawn it". |
| 2401 | #[tokio::test] |
| 2402 | async fn the_dropped_slot_breadcrumb_names_the_kind_and_the_slot() { |
| 2403 | let driver = Arc::new(FakeDriver::new()); |
| 2404 | driver.on("beta", FakeReply::Fail("boom".to_string())); |
| 2405 | run( |
| 2406 | &driver, |
| 2407 | r#" |
| 2408 | return await parallel([ |
| 2409 | () => task({ description: "alpha" }), |
| 2410 | () => task({ description: "beta" }), |
| 2411 | ]); |
| 2412 | "#, |
| 2413 | json!(null), |
| 2414 | ) |
| 2415 | .await |
| 2416 | .unwrap(); |
| 2417 | assert!( |
| 2418 | driver.events().iter().any(|event| matches!( |
| 2419 | event, |
| 2420 | ProgressEvent::Log { message } |
| 2421 | if message.contains("dropped a failed slot as null") |
| 2422 | && message.contains("kind=agent") |
| 2423 | && message.contains("slot 1") |
| 2424 | )), |
| 2425 | "the breadcrumb must name the kind and the slot: {:?}", |
| 2426 | driver.events() |
| 2427 | ); |
| 2428 | } |
| 2429 | |
| 2430 | struct EchoInvoker; |
| 2431 | |
| 2432 | #[async_trait::async_trait] |
| 2433 | impl codewhale_workflow_js::ToolInvoker for EchoInvoker { |
| 2434 | async fn invoke( |
| 2435 | &self, |
| 2436 | request: codewhale_workflow_js::ToolCallRequest, |
| 2437 | ) -> Result<codewhale_workflow_js::ToolCallResponse, codewhale_workflow_js::DriverError> { |
| 2438 | use codewhale_workflow_js::{DriverError, ToolCallResponse}; |
| 2439 | if request.tool == "boom" { |
| 2440 | return Ok(ToolCallResponse { |
| 2441 | ok: false, |
| 2442 | result: json!("kaput"), |
| 2443 | }); |
| 2444 | } |
| 2445 | if request.tool == "deny" { |
| 2446 | return Err(DriverError::Rejected("nope".to_string())); |
| 2447 | } |
| 2448 | Ok(ToolCallResponse { |
| 2449 | ok: true, |
| 2450 | result: json!({ "echo": request.input }), |
| 2451 | }) |
| 2452 | } |
| 2453 | } |
| 2454 | |
| 2455 | async fn run_tools(source: &str) -> Result<serde_json::Value, WorkflowJsError> { |
| 2456 | let driver = Arc::new(FakeDriver::new()); |
| 2457 | WorkflowVm::new() |
| 2458 | .run_tools_script( |
| 2459 | source, |
| 2460 | json!(null), |
| 2461 | driver.clone() as Arc<dyn codewhale_workflow_js::WorkflowDriver>, |
| 2462 | Arc::new(EchoInvoker) as Arc<dyn codewhale_workflow_js::ToolInvoker>, |
| 2463 | WorkflowRunCancel::new(), |
| 2464 | ) |
| 2465 | .await |
| 2466 | } |
| 2467 | |
| 2468 | #[tokio::test] |
| 2469 | async fn tools_surface_is_absent_without_invoker() { |
| 2470 | let driver = Arc::new(FakeDriver::new()); |
| 2471 | let value = run(&driver, "return typeof tools;", json!(null)) |
| 2472 | .await |
| 2473 | .unwrap(); |
| 2474 | assert_eq!(value, json!("undefined")); |
| 2475 | } |
| 2476 | |
| 2477 | #[tokio::test] |
| 2478 | async fn tools_call_round_trips() { |
| 2479 | let value = |
| 2480 | run_tools(r#"const r = await tools.call("read", { path: "x" }); return r.echo.path;"#) |
| 2481 | .await |
| 2482 | .unwrap(); |
| 2483 | assert_eq!(value, json!("x")); |
| 2484 | } |
| 2485 | |
| 2486 | #[tokio::test] |
| 2487 | async fn tools_call_refusal_throws_admission_kind() { |
| 2488 | let value = run_tools( |
| 2489 | r#"try { await tools.call("deny", {}); return "no-throw"; } catch (e) { return e.kind; }"#, |
| 2490 | ) |
| 2491 | .await |
| 2492 | .unwrap(); |
| 2493 | assert_eq!(value, json!("admission")); |
| 2494 | } |
| 2495 | |
| 2496 | #[tokio::test] |
| 2497 | async fn tools_call_failure_throws_agent_kind() { |
| 2498 | let value = run_tools( |
| 2499 | r#"try { await tools.call("boom", {}); return "no-throw"; } catch (e) { return e.kind; }"#, |
| 2500 | ) |
| 2501 | .await |
| 2502 | .unwrap(); |
| 2503 | assert_eq!(value, json!("agent")); |
| 2504 | } |
| 2505 | |
| 2506 | #[tokio::test] |
| 2507 | async fn tools_call_cap_rejects_runaway_loops() { |
| 2508 | let err = run_tools( |
| 2509 | r#"for (let i = 0; i < 55; i++) { await tools.call("read", {}); } return "never";"#, |
| 2510 | ) |
| 2511 | .await; |
| 2512 | let message = script_message(err); |
| 2513 | assert!( |
| 2514 | message.contains("per-run tool-call cap"), |
| 2515 | "unexpected: {message}" |
| 2516 | ); |
| 2517 | } |
| 2518 |