| 1 | //! Provider-neutral experimental search authoring for Workflow. |
| 2 | //! |
| 3 | //! This module is an authoring and freeze boundary, not a new runtime or |
| 4 | //! scheduler. A validated search still has to be lowered by the Workflow host |
| 5 | //! into Fleet workers plus a runtime-owned evaluator. In particular, worker |
| 6 | //! self-reports are never promoted to hard-gate evidence here. |
| 7 | |
| 8 | use serde::{Deserialize, Serialize}; |
| 9 | use sha2::{Digest, Sha256}; |
| 10 | use std::path::{Component, Path}; |
| 11 | use thiserror::Error; |
| 12 | |
| 13 | use crate::{DEFAULT_FLEET_WORKFLOW_MAX_AGENTS, experimental_search::SearchSpecError::*}; |
| 14 | |
| 15 | pub const WORKFLOW_SEARCH_SCHEMA_VERSION: u32 = 1; |
| 16 | /// Fallback live-worker ceiling for one search admission batch. |
| 17 | /// |
| 18 | /// This is the answer when the host resolves no Fleet concurrency limit at |
| 19 | /// all — not a second knob. 16 matches the Workflow host's live-child ceiling |
| 20 | /// today (`codewhale_workflow_js::WORKFLOW_MAX_CONCURRENT`, from which the tui |
| 21 | /// driver sizes its per-run admission semaphore). This crate cannot import |
| 22 | /// that constant directly because `codewhale-workflow-js` depends on |
| 23 | /// `codewhale-workflow`. |
| 24 | /// |
| 25 | /// Prefer passing the live limit: every validation entry point has a |
| 26 | /// `*_with_limit` twin that takes the resolved Fleet ceiling, and the frozen |
| 27 | /// receipt records which of the two actually bounded the search. |
| 28 | pub const WORKFLOW_SEARCH_DEFAULT_MAX_CONCURRENT: u16 = 16; |
| 29 | |
| 30 | /// Where a search's live-worker ceiling came from. |
| 31 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] |
| 32 | #[serde(rename_all = "snake_case")] |
| 33 | pub enum SearchConcurrencySource { |
| 34 | /// Resolved from the host's Fleet concurrency configuration. |
| 35 | FleetLimit, |
| 36 | /// No Fleet limit resolved, so [`WORKFLOW_SEARCH_DEFAULT_MAX_CONCURRENT`] |
| 37 | /// applied. |
| 38 | #[default] |
| 39 | Default, |
| 40 | } |
| 41 | |
| 42 | /// The live-worker ceiling that bounded a search, plus where it came from. |
| 43 | /// |
| 44 | /// Carried on the frozen receipt so an operator reading a run can tell a |
| 45 | /// deliberately small Fleet pool from this crate's fallback. |
| 46 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] |
| 47 | pub struct ResolvedSearchConcurrency { |
| 48 | pub limit: u16, |
| 49 | pub source: SearchConcurrencySource, |
| 50 | } |
| 51 | |
| 52 | impl Default for ResolvedSearchConcurrency { |
| 53 | fn default() -> Self { |
| 54 | Self::resolve(None) |
| 55 | } |
| 56 | } |
| 57 | |
| 58 | impl ResolvedSearchConcurrency { |
| 59 | /// Resolve the ceiling from an already-resolved Fleet limit. |
| 60 | /// |
| 61 | /// `None` — and a nonsensical `Some(0)`, which would admit nothing — fall |
| 62 | /// back to [`WORKFLOW_SEARCH_DEFAULT_MAX_CONCURRENT`]. |
| 63 | #[must_use] |
| 64 | pub fn resolve(fleet_limit: Option<u16>) -> Self { |
| 65 | match fleet_limit.filter(|limit| *limit > 0) { |
| 66 | Some(limit) => Self { |
| 67 | limit, |
| 68 | source: SearchConcurrencySource::FleetLimit, |
| 69 | }, |
| 70 | None => Self { |
| 71 | limit: WORKFLOW_SEARCH_DEFAULT_MAX_CONCURRENT, |
| 72 | source: SearchConcurrencySource::Default, |
| 73 | }, |
| 74 | } |
| 75 | } |
| 76 | |
| 77 | /// Resolve straight from the two config seams the host already owns: |
| 78 | /// `[workflow] max_concurrent` (the per-run live-agent ceiling, see |
| 79 | /// `codewhale_config::WorkflowConfigToml::max_concurrent`) and a Fleet |
| 80 | /// profile's `delegation.max_concurrency` hint |
| 81 | /// (`codewhale_config::FleetDelegationHints::max_concurrency`). |
| 82 | /// |
| 83 | /// The lower of the present values wins: a profile that asks for fewer |
| 84 | /// workers than the run allows is a real bound, and a run ceiling below a |
| 85 | /// profile hint is the admission the host will actually grant. |
| 86 | #[must_use] |
| 87 | pub fn from_fleet_config( |
| 88 | workflow_max_concurrent: Option<u32>, |
| 89 | profile_max_concurrency: Option<usize>, |
| 90 | ) -> Self { |
| 91 | let workflow = |
| 92 | workflow_max_concurrent.map(|value| u16::try_from(value).unwrap_or(u16::MAX)); |
| 93 | let profile = profile_max_concurrency.map(|value| u16::try_from(value).unwrap_or(u16::MAX)); |
| 94 | let resolved = match (workflow.filter(|v| *v > 0), profile.filter(|v| *v > 0)) { |
| 95 | (Some(left), Some(right)) => Some(left.min(right)), |
| 96 | (Some(value), None) | (None, Some(value)) => Some(value), |
| 97 | (None, None) => None, |
| 98 | }; |
| 99 | Self::resolve(resolved) |
| 100 | } |
| 101 | |
| 102 | /// One-line receipt copy naming the ceiling and its origin. |
| 103 | #[must_use] |
| 104 | pub fn receipt_line(&self) -> String { |
| 105 | let origin = match self.source { |
| 106 | SearchConcurrencySource::FleetLimit => "resolved Fleet limit", |
| 107 | SearchConcurrencySource::Default => "default, no Fleet limit resolved", |
| 108 | }; |
| 109 | format!("live-worker ceiling {limit} ({origin})", limit = self.limit) |
| 110 | } |
| 111 | } |
| 112 | |
| 113 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] |
| 114 | pub struct WorkflowSearchSpec { |
| 115 | #[serde(default = "default_schema_version")] |
| 116 | pub schema_version: u32, |
| 117 | pub name: String, |
| 118 | pub objective: String, |
| 119 | pub population: u16, |
| 120 | pub rounds: Vec<u16>, |
| 121 | pub concurrency: u16, |
| 122 | pub worker: SearchWorkerSpec, |
| 123 | #[serde(default)] |
| 124 | pub budget: SearchBudgetSpec, |
| 125 | pub hard_gates: SearchHardGateSpec, |
| 126 | pub score: SearchScoreSpec, |
| 127 | #[serde(default)] |
| 128 | pub selection: SearchSelectionSpec, |
| 129 | #[serde(default)] |
| 130 | pub integration_policy: SearchIntegrationPolicy, |
| 131 | } |
| 132 | |
| 133 | impl WorkflowSearchSpec { |
| 134 | /// Parse and validate against the fallback ceiling. Hosts that know their |
| 135 | /// Fleet limit should call [`Self::from_toml_with_limit`]. |
| 136 | pub fn from_toml(source: &str) -> Result<Self, SearchSpecError> { |
| 137 | Self::from_toml_with_limit(source, None) |
| 138 | } |
| 139 | |
| 140 | /// Parse and validate against the resolved Fleet concurrency limit. |
| 141 | pub fn from_toml_with_limit( |
| 142 | source: &str, |
| 143 | fleet_limit: Option<u16>, |
| 144 | ) -> Result<Self, SearchSpecError> { |
| 145 | let spec: Self = toml::from_str(source).map_err(|error| Parse(error.to_string()))?; |
| 146 | spec.validate_with_limit(fleet_limit)?; |
| 147 | Ok(spec) |
| 148 | } |
| 149 | |
| 150 | pub fn validate(&self) -> Result<(), SearchSpecError> { |
| 151 | self.validate_with_limit(None).map(|_| ()) |
| 152 | } |
| 153 | |
| 154 | /// Validate against the resolved Fleet concurrency limit, returning the |
| 155 | /// ceiling that applied so the caller can echo it in a receipt. |
| 156 | pub fn validate_with_limit( |
| 157 | &self, |
| 158 | fleet_limit: Option<u16>, |
| 159 | ) -> Result<ResolvedSearchConcurrency, SearchSpecError> { |
| 160 | let concurrency_ceiling = ResolvedSearchConcurrency::resolve(fleet_limit); |
| 161 | if self.schema_version != WORKFLOW_SEARCH_SCHEMA_VERSION { |
| 162 | return Err(UnsupportedSchemaVersion(self.schema_version)); |
| 163 | } |
| 164 | validate_name(&self.name)?; |
| 165 | validate_text("objective", &self.objective, 32_768)?; |
| 166 | if !(2..=DEFAULT_FLEET_WORKFLOW_MAX_AGENTS as u16).contains(&self.population) { |
| 167 | return Err(InvalidPopulation(self.population)); |
| 168 | } |
| 169 | if self.concurrency == 0 |
| 170 | || self.concurrency > concurrency_ceiling.limit |
| 171 | || self.concurrency > self.population |
| 172 | { |
| 173 | return Err(InvalidConcurrency { |
| 174 | concurrency: self.concurrency, |
| 175 | population: self.population, |
| 176 | limit: concurrency_ceiling.limit, |
| 177 | }); |
| 178 | } |
| 179 | validate_rounds(self.population, &self.rounds)?; |
| 180 | validate_text("worker.model", &self.worker.model, 256)?; |
| 181 | if self.worker.write_roots.is_empty() && self.worker.exact_files.is_empty() { |
| 182 | return Err(UnboundedWriteScope); |
| 183 | } |
| 184 | validate_repo_relative_paths("worker.write_roots", &self.worker.write_roots, 128)?; |
| 185 | validate_repo_relative_paths("worker.exact_files", &self.worker.exact_files, 256)?; |
| 186 | if self.budget.max_cost_microusd == Some(0) { |
| 187 | return Err(ZeroBudget("max_cost_microusd")); |
| 188 | } |
| 189 | if self.budget.max_tokens == Some(0) { |
| 190 | return Err(ZeroBudget("max_tokens")); |
| 191 | } |
| 192 | if !self.hard_gates.forbid_test_changes { |
| 193 | return Err(TestWeakeningAllowed); |
| 194 | } |
| 195 | if self.hard_gates.commands.is_empty() { |
| 196 | return Err(MissingHardGates); |
| 197 | } |
| 198 | validate_string_list("hard_gates.commands", &self.hard_gates.commands, 32, 4_096)?; |
| 199 | validate_string_list( |
| 200 | "hard_gates.protected_paths", |
| 201 | &self.hard_gates.protected_paths, |
| 202 | 256, |
| 203 | 1_024, |
| 204 | )?; |
| 205 | validate_text("score.command", &self.score.command, 4_096)?; |
| 206 | validate_text("score.metric", &self.score.metric, 256)?; |
| 207 | if !(1..=25).contains(&self.score.trials) { |
| 208 | return Err(InvalidTrials(self.score.trials)); |
| 209 | } |
| 210 | if self.score.tie_breakers.is_empty() { |
| 211 | return Err(MissingTieBreakers); |
| 212 | } |
| 213 | Ok(concurrency_ceiling) |
| 214 | } |
| 215 | |
| 216 | /// Freeze the exact public inputs and evaluator identity before admission. |
| 217 | /// The evaluator bytes are hashed, not exposed to generation workers. |
| 218 | pub fn freeze( |
| 219 | &self, |
| 220 | baseline_commit: &str, |
| 221 | resolved_model: &str, |
| 222 | public_evidence: &[u8], |
| 223 | evaluator: &[u8], |
| 224 | ) -> Result<FrozenWorkflowSearch, SearchSpecError> { |
| 225 | self.freeze_with_limit( |
| 226 | baseline_commit, |
| 227 | resolved_model, |
| 228 | public_evidence, |
| 229 | evaluator, |
| 230 | None, |
| 231 | ) |
| 232 | } |
| 233 | |
| 234 | /// Freeze against the resolved Fleet concurrency limit. |
| 235 | /// |
| 236 | /// The resolved ceiling is recorded on the receipt but deliberately kept |
| 237 | /// out of the preregistration hash: the hash freezes the scientific inputs |
| 238 | /// (spec, model, public evidence, evaluator identity), and how many |
| 239 | /// workers the operator's pool happened to allow is an operational fact |
| 240 | /// about the run, not part of what was preregistered. |
| 241 | pub fn freeze_with_limit( |
| 242 | &self, |
| 243 | baseline_commit: &str, |
| 244 | resolved_model: &str, |
| 245 | public_evidence: &[u8], |
| 246 | evaluator: &[u8], |
| 247 | fleet_limit: Option<u16>, |
| 248 | ) -> Result<FrozenWorkflowSearch, SearchSpecError> { |
| 249 | let resolved_concurrency = self.validate_with_limit(fleet_limit)?; |
| 250 | validate_commit(baseline_commit)?; |
| 251 | validate_text("resolved_model", resolved_model, 256)?; |
| 252 | if evaluator.is_empty() { |
| 253 | return Err(EmptyEvaluator); |
| 254 | } |
| 255 | |
| 256 | let public_evidence_hash = sha256_label(public_evidence); |
| 257 | let evaluator_hash = sha256_label(evaluator); |
| 258 | let freeze_input = FreezeInput { |
| 259 | spec: self, |
| 260 | baseline_commit, |
| 261 | requested_model: &self.worker.model, |
| 262 | resolved_model, |
| 263 | public_evidence_hash: &public_evidence_hash, |
| 264 | evaluator_hash: &evaluator_hash, |
| 265 | }; |
| 266 | let encoded = |
| 267 | serde_json::to_vec(&freeze_input).map_err(|error| FreezeEncoding(error.to_string()))?; |
| 268 | let preregistration_hash = sha256_label(&encoded); |
| 269 | let search_id = format!("search-{}", &preregistration_hash[7..23]); |
| 270 | |
| 271 | Ok(FrozenWorkflowSearch { |
| 272 | schema_version: self.schema_version, |
| 273 | search_id, |
| 274 | baseline_commit: baseline_commit.to_string(), |
| 275 | preregistration_hash, |
| 276 | public_evidence_hash, |
| 277 | evaluator_hash, |
| 278 | requested_model: self.worker.model.clone(), |
| 279 | resolved_model: resolved_model.to_string(), |
| 280 | candidate_ids: self.candidate_ids(), |
| 281 | resolved_concurrency, |
| 282 | }) |
| 283 | } |
| 284 | |
| 285 | #[must_use] |
| 286 | pub fn candidate_ids(&self) -> Vec<String> { |
| 287 | let width = self.population.to_string().len().max(3); |
| 288 | (1..=self.population) |
| 289 | .map(|index| format!("cand_{index:0width$}")) |
| 290 | .collect() |
| 291 | } |
| 292 | |
| 293 | /// Deterministic admission batches. Fleet owns actual scheduling and may |
| 294 | /// run fewer workers when its configured pool or provider quota is lower. |
| 295 | pub fn admission_batches(&self) -> Result<Vec<Vec<String>>, SearchSpecError> { |
| 296 | self.admission_batches_with_limit(None) |
| 297 | } |
| 298 | |
| 299 | /// Deterministic admission batches, validated against the resolved Fleet |
| 300 | /// concurrency limit. |
| 301 | pub fn admission_batches_with_limit( |
| 302 | &self, |
| 303 | fleet_limit: Option<u16>, |
| 304 | ) -> Result<Vec<Vec<String>>, SearchSpecError> { |
| 305 | self.validate_with_limit(fleet_limit)?; |
| 306 | Ok(self |
| 307 | .candidate_ids() |
| 308 | .chunks(usize::from(self.concurrency)) |
| 309 | .map(<[String]>::to_vec) |
| 310 | .collect()) |
| 311 | } |
| 312 | } |
| 313 | |
| 314 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] |
| 315 | pub struct SearchWorkerSpec { |
| 316 | #[serde(default)] |
| 317 | pub provider: Option<String>, |
| 318 | pub model: String, |
| 319 | #[serde(default)] |
| 320 | pub reasoning_effort: SearchReasoningEffort, |
| 321 | #[serde(default)] |
| 322 | pub write_authority: SearchWriteAuthority, |
| 323 | #[serde(default)] |
| 324 | pub write_roots: Vec<String>, |
| 325 | #[serde(default)] |
| 326 | pub exact_files: Vec<String>, |
| 327 | } |
| 328 | |
| 329 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] |
| 330 | #[serde(rename_all = "snake_case")] |
| 331 | pub enum SearchReasoningEffort { |
| 332 | Off, |
| 333 | Low, |
| 334 | Medium, |
| 335 | #[default] |
| 336 | High, |
| 337 | Max, |
| 338 | } |
| 339 | |
| 340 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] |
| 341 | #[serde(rename_all = "snake_case")] |
| 342 | pub enum SearchWriteAuthority { |
| 343 | #[default] |
| 344 | WorktreeWrite, |
| 345 | } |
| 346 | |
| 347 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] |
| 348 | pub struct SearchBudgetSpec { |
| 349 | #[serde(default)] |
| 350 | pub max_cost_microusd: Option<u64>, |
| 351 | #[serde(default)] |
| 352 | pub max_tokens: Option<u64>, |
| 353 | } |
| 354 | |
| 355 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] |
| 356 | pub struct SearchHardGateSpec { |
| 357 | pub commands: Vec<String>, |
| 358 | #[serde(default = "default_true")] |
| 359 | pub forbid_test_changes: bool, |
| 360 | #[serde(default)] |
| 361 | pub protected_paths: Vec<String>, |
| 362 | } |
| 363 | |
| 364 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] |
| 365 | pub struct SearchScoreSpec { |
| 366 | pub command: String, |
| 367 | pub metric: String, |
| 368 | #[serde(default)] |
| 369 | pub direction: SearchDirection, |
| 370 | #[serde(default = "default_trials")] |
| 371 | pub trials: u16, |
| 372 | #[serde(default)] |
| 373 | pub tie_breakers: Vec<SearchTieBreaker>, |
| 374 | } |
| 375 | |
| 376 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] |
| 377 | #[serde(rename_all = "snake_case")] |
| 378 | pub enum SearchDirection { |
| 379 | #[default] |
| 380 | Minimize, |
| 381 | Maximize, |
| 382 | } |
| 383 | |
| 384 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] |
| 385 | #[serde(rename_all = "snake_case")] |
| 386 | pub enum SearchTieBreaker { |
| 387 | DiffLines, |
| 388 | CostMicrousd, |
| 389 | Score, |
| 390 | } |
| 391 | |
| 392 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] |
| 393 | pub struct SearchSelectionSpec { |
| 394 | #[serde(default)] |
| 395 | pub policy: SearchSelectionPolicy, |
| 396 | #[serde(default = "default_true")] |
| 397 | pub retain_diversity: bool, |
| 398 | #[serde(default)] |
| 399 | pub ordering: Vec<SearchSelectionMetric>, |
| 400 | } |
| 401 | |
| 402 | impl Default for SearchSelectionSpec { |
| 403 | fn default() -> Self { |
| 404 | Self { |
| 405 | policy: SearchSelectionPolicy::Pareto, |
| 406 | retain_diversity: true, |
| 407 | ordering: Vec::new(), |
| 408 | } |
| 409 | } |
| 410 | } |
| 411 | |
| 412 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] |
| 413 | #[serde(rename_all = "snake_case")] |
| 414 | pub enum SearchSelectionPolicy { |
| 415 | #[default] |
| 416 | Pareto, |
| 417 | Ordered, |
| 418 | } |
| 419 | |
| 420 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] |
| 421 | #[serde(rename_all = "snake_case")] |
| 422 | pub enum SearchSelectionMetric { |
| 423 | Score, |
| 424 | Runtime, |
| 425 | DiffLines, |
| 426 | CostMicrousd, |
| 427 | } |
| 428 | |
| 429 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] |
| 430 | #[serde(rename_all = "snake_case")] |
| 431 | pub enum SearchIntegrationPolicy { |
| 432 | /// Produce a verified, reviewable winner or NONE. Never apply or merge it. |
| 433 | #[default] |
| 434 | ReviewOnly, |
| 435 | } |
| 436 | |
| 437 | #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] |
| 438 | pub struct FrozenWorkflowSearch { |
| 439 | pub schema_version: u32, |
| 440 | pub search_id: String, |
| 441 | pub baseline_commit: String, |
| 442 | pub preregistration_hash: String, |
| 443 | pub public_evidence_hash: String, |
| 444 | pub evaluator_hash: String, |
| 445 | pub requested_model: String, |
| 446 | pub resolved_model: String, |
| 447 | pub candidate_ids: Vec<String>, |
| 448 | /// The live-worker ceiling that bounded this search, and whether it came |
| 449 | /// from Fleet config or the crate fallback. `#[serde(default)]` so |
| 450 | /// receipts written before the bound was recorded still load. |
| 451 | #[serde(default)] |
| 452 | pub resolved_concurrency: ResolvedSearchConcurrency, |
| 453 | } |
| 454 | |
| 455 | impl FrozenWorkflowSearch { |
| 456 | /// Receipt line naming the ceiling that actually bounded this run. |
| 457 | #[must_use] |
| 458 | pub fn concurrency_receipt_line(&self) -> String { |
| 459 | self.resolved_concurrency.receipt_line() |
| 460 | } |
| 461 | } |
| 462 | |
| 463 | #[derive(Serialize)] |
| 464 | struct FreezeInput<'a> { |
| 465 | spec: &'a WorkflowSearchSpec, |
| 466 | baseline_commit: &'a str, |
| 467 | requested_model: &'a str, |
| 468 | resolved_model: &'a str, |
| 469 | public_evidence_hash: &'a str, |
| 470 | evaluator_hash: &'a str, |
| 471 | } |
| 472 | |
| 473 | #[derive(Debug, Clone, PartialEq, Eq, Error)] |
| 474 | pub enum SearchSpecError { |
| 475 | #[error("failed to parse Workflow search TOML: {0}")] |
| 476 | Parse(String), |
| 477 | #[error("unsupported Workflow search schema version {0}")] |
| 478 | UnsupportedSchemaVersion(u32), |
| 479 | #[error("search name must be a 1-96 character lowercase token")] |
| 480 | InvalidName, |
| 481 | #[error("{field} must be non-empty and no longer than {max} characters")] |
| 482 | InvalidText { field: &'static str, max: usize }, |
| 483 | #[error("population {0} must be between 2 and 1000")] |
| 484 | InvalidPopulation(u16), |
| 485 | #[error( |
| 486 | "concurrency {concurrency} must be between 1 and {limit} (resolved live-worker ceiling) and no greater than population {population}" |
| 487 | )] |
| 488 | InvalidConcurrency { |
| 489 | concurrency: u16, |
| 490 | population: u16, |
| 491 | limit: u16, |
| 492 | }, |
| 493 | #[error("rounds must start at population, decrease strictly, and end at 1")] |
| 494 | InvalidRounds, |
| 495 | #[error("write-capable search workers require write_roots or exact_files")] |
| 496 | UnboundedWriteScope, |
| 497 | #[error("{field} contains an empty, oversized, or duplicate entry")] |
| 498 | InvalidStringList { field: &'static str }, |
| 499 | #[error("{field} entries must be bounded repo-relative paths without parent traversal")] |
| 500 | InvalidWriteScope { field: &'static str }, |
| 501 | #[error("{0} must be greater than zero when set")] |
| 502 | ZeroBudget(&'static str), |
| 503 | #[error("experimental search must forbid test changes")] |
| 504 | TestWeakeningAllowed, |
| 505 | #[error("experimental search requires at least one runtime-owned hard-gate command")] |
| 506 | MissingHardGates, |
| 507 | #[error("score.trials must be between 1 and 25, got {0}")] |
| 508 | InvalidTrials(u16), |
| 509 | #[error("experimental search requires at least one deterministic tie-breaker")] |
| 510 | MissingTieBreakers, |
| 511 | #[error("baseline_commit must be a 7-64 character hexadecimal commit id")] |
| 512 | InvalidBaselineCommit, |
| 513 | #[error("evaluator bytes must be non-empty")] |
| 514 | EmptyEvaluator, |
| 515 | #[error("failed to encode frozen Workflow search: {0}")] |
| 516 | FreezeEncoding(String), |
| 517 | } |
| 518 | |
| 519 | fn default_schema_version() -> u32 { |
| 520 | WORKFLOW_SEARCH_SCHEMA_VERSION |
| 521 | } |
| 522 | |
| 523 | fn default_trials() -> u16 { |
| 524 | 5 |
| 525 | } |
| 526 | |
| 527 | fn default_true() -> bool { |
| 528 | true |
| 529 | } |
| 530 | |
| 531 | fn validate_name(value: &str) -> Result<(), SearchSpecError> { |
| 532 | if value.is_empty() |
| 533 | || value.len() > 96 |
| 534 | || !value |
| 535 | .bytes() |
| 536 | .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"-_".contains(&byte)) |
| 537 | { |
| 538 | return Err(InvalidName); |
| 539 | } |
| 540 | Ok(()) |
| 541 | } |
| 542 | |
| 543 | fn validate_text(field: &'static str, value: &str, max: usize) -> Result<(), SearchSpecError> { |
| 544 | if value.trim().is_empty() || value.chars().count() > max { |
| 545 | return Err(InvalidText { field, max }); |
| 546 | } |
| 547 | Ok(()) |
| 548 | } |
| 549 | |
| 550 | fn validate_rounds(population: u16, rounds: &[u16]) -> Result<(), SearchSpecError> { |
| 551 | if rounds.len() < 2 |
| 552 | || rounds.first() != Some(&population) |
| 553 | || rounds.last() != Some(&1) |
| 554 | || rounds.windows(2).any(|pair| pair[0] <= pair[1]) |
| 555 | { |
| 556 | return Err(InvalidRounds); |
| 557 | } |
| 558 | Ok(()) |
| 559 | } |
| 560 | |
| 561 | fn validate_string_list( |
| 562 | field: &'static str, |
| 563 | values: &[String], |
| 564 | max_items: usize, |
| 565 | max_chars: usize, |
| 566 | ) -> Result<(), SearchSpecError> { |
| 567 | if values.len() > max_items |
| 568 | || values |
| 569 | .iter() |
| 570 | .any(|value| value.trim().is_empty() || value.chars().count() > max_chars) |
| 571 | || values |
| 572 | .iter() |
| 573 | .enumerate() |
| 574 | .any(|(index, value)| values[..index].contains(value)) |
| 575 | { |
| 576 | return Err(InvalidStringList { field }); |
| 577 | } |
| 578 | Ok(()) |
| 579 | } |
| 580 | |
| 581 | fn validate_repo_relative_paths( |
| 582 | field: &'static str, |
| 583 | values: &[String], |
| 584 | max_items: usize, |
| 585 | ) -> Result<(), SearchSpecError> { |
| 586 | validate_string_list(field, values, max_items, 1_024)?; |
| 587 | if values.iter().any(|value| { |
| 588 | let trimmed = value.trim(); |
| 589 | let normalized = trimmed.replace('\\', "/"); |
| 590 | let windows_drive = normalized.as_bytes().get(1) == Some(&b':') |
| 591 | && normalized |
| 592 | .as_bytes() |
| 593 | .first() |
| 594 | .is_some_and(u8::is_ascii_alphabetic); |
| 595 | trimmed != value |
| 596 | || trimmed.chars().any(|ch| matches!(ch, '\0' | '\r' | '\n')) |
| 597 | || windows_drive |
| 598 | || Path::new(&normalized).is_absolute() |
| 599 | || Path::new(&normalized).components().any(|component| { |
| 600 | matches!( |
| 601 | component, |
| 602 | Component::ParentDir | Component::RootDir | Component::Prefix(_) |
| 603 | ) |
| 604 | }) |
| 605 | }) { |
| 606 | return Err(InvalidWriteScope { field }); |
| 607 | } |
| 608 | Ok(()) |
| 609 | } |
| 610 | |
| 611 | fn validate_commit(value: &str) -> Result<(), SearchSpecError> { |
| 612 | if !(7..=64).contains(&value.len()) || !value.bytes().all(|byte| byte.is_ascii_hexdigit()) { |
| 613 | return Err(InvalidBaselineCommit); |
| 614 | } |
| 615 | Ok(()) |
| 616 | } |
| 617 | |
| 618 | fn sha256_label(bytes: &[u8]) -> String { |
| 619 | let digest = Sha256::digest(bytes); |
| 620 | let mut output = String::with_capacity(71); |
| 621 | output.push_str("sha256:"); |
| 622 | for byte in digest { |
| 623 | use std::fmt::Write as _; |
| 624 | let _ = write!(output, "{byte:02x}"); |
| 625 | } |
| 626 | output |
| 627 | } |
| 628 | |
| 629 | #[cfg(test)] |
| 630 | mod tests { |
| 631 | use super::*; |
| 632 | |
| 633 | const SPEC: &str = r#" |
| 634 | name = "speed-up-certificate" |
| 635 | objective = "Reduce runtime without changing exact results" |
| 636 | population = 32 |
| 637 | rounds = [32, 8, 3, 1] |
| 638 | concurrency = 16 |
| 639 | integration_policy = "review_only" |
| 640 | |
| 641 | [worker] |
| 642 | provider = "deepseek" |
| 643 | model = "deepseek-v4-flash" |
| 644 | reasoning_effort = "max" |
| 645 | write_authority = "worktree_write" |
| 646 | write_roots = ["code"] |
| 647 | |
| 648 | [budget] |
| 649 | max_cost_microusd = 5000000 |
| 650 | max_tokens = 10000000 |
| 651 | |
| 652 | [hard_gates] |
| 653 | commands = ["cargo test --locked", "git diff --exit-code -- expected.json"] |
| 654 | forbid_test_changes = true |
| 655 | protected_paths = ["tests", "expected.json"] |
| 656 | |
| 657 | [score] |
| 658 | command = "./scripts/benchmark_candidate.sh" |
| 659 | direction = "minimize" |
| 660 | metric = "median_runtime_ms" |
| 661 | trials = 5 |
| 662 | tie_breakers = ["diff_lines", "cost_microusd"] |
| 663 | |
| 664 | [selection] |
| 665 | policy = "pareto" |
| 666 | retain_diversity = true |
| 667 | "#; |
| 668 | |
| 669 | #[test] |
| 670 | fn parses_valid_search_and_queues_through_live_cap() { |
| 671 | let spec = WorkflowSearchSpec::from_toml(SPEC).expect("valid search spec"); |
| 672 | |
| 673 | assert_eq!(spec.population, 32); |
| 674 | let batches = spec.admission_batches().expect("validated admission"); |
| 675 | assert_eq!(batches.len(), 2); |
| 676 | assert_eq!(batches[0].len(), 16); |
| 677 | assert_eq!(spec.candidate_ids()[0], "cand_001"); |
| 678 | assert_eq!(spec.candidate_ids()[31], "cand_032"); |
| 679 | } |
| 680 | |
| 681 | #[test] |
| 682 | fn freeze_is_deterministic_and_model_version_sensitive() { |
| 683 | let spec = WorkflowSearchSpec::from_toml(SPEC).expect("valid search spec"); |
| 684 | let first = spec |
| 685 | .freeze( |
| 686 | "33bc6a98", |
| 687 | "DeepSeek-V4-Flash-0731", |
| 688 | b"public evidence", |
| 689 | b"private evaluator", |
| 690 | ) |
| 691 | .expect("freeze succeeds"); |
| 692 | let replay = spec |
| 693 | .freeze( |
| 694 | "33bc6a98", |
| 695 | "DeepSeek-V4-Flash-0731", |
| 696 | b"public evidence", |
| 697 | b"private evaluator", |
| 698 | ) |
| 699 | .expect("freeze succeeds"); |
| 700 | let drifted = spec |
| 701 | .freeze( |
| 702 | "33bc6a98", |
| 703 | "DeepSeek-V4-Flash-next", |
| 704 | b"public evidence", |
| 705 | b"private evaluator", |
| 706 | ) |
| 707 | .expect("freeze succeeds"); |
| 708 | |
| 709 | assert_eq!(first, replay); |
| 710 | assert_ne!(first.search_id, drifted.search_id); |
| 711 | assert_ne!(first.preregistration_hash, drifted.preregistration_hash); |
| 712 | assert_eq!(first.requested_model, "deepseek-v4-flash"); |
| 713 | assert_eq!(first.resolved_model, "DeepSeek-V4-Flash-0731"); |
| 714 | } |
| 715 | |
| 716 | #[test] |
| 717 | fn rejects_unsafe_or_unbounded_searches() { |
| 718 | let mut spec = WorkflowSearchSpec::from_toml(SPEC).expect("valid search spec"); |
| 719 | spec.hard_gates.forbid_test_changes = false; |
| 720 | assert_eq!(spec.validate(), Err(SearchSpecError::TestWeakeningAllowed)); |
| 721 | |
| 722 | spec.hard_gates.forbid_test_changes = true; |
| 723 | spec.worker.write_roots.clear(); |
| 724 | assert_eq!(spec.validate(), Err(SearchSpecError::UnboundedWriteScope)); |
| 725 | } |
| 726 | |
| 727 | #[test] |
| 728 | fn rejects_invalid_rounds_and_excess_live_concurrency() { |
| 729 | let mut spec = WorkflowSearchSpec::from_toml(SPEC).expect("valid search spec"); |
| 730 | spec.rounds = vec![32, 8, 8, 1]; |
| 731 | assert_eq!(spec.validate(), Err(SearchSpecError::InvalidRounds)); |
| 732 | |
| 733 | spec.rounds = vec![32, 1]; |
| 734 | spec.concurrency = 17; |
| 735 | assert_eq!( |
| 736 | spec.validate(), |
| 737 | Err(SearchSpecError::InvalidConcurrency { |
| 738 | concurrency: 17, |
| 739 | population: 32, |
| 740 | limit: 16, |
| 741 | }) |
| 742 | ); |
| 743 | } |
| 744 | |
| 745 | #[test] |
| 746 | fn ceiling_follows_the_resolved_fleet_limit() { |
| 747 | let mut spec = WorkflowSearchSpec::from_toml(SPEC).expect("valid search spec"); |
| 748 | |
| 749 | // A smaller Fleet pool rejects the spec's 16-wide batches... |
| 750 | assert_eq!( |
| 751 | spec.validate_with_limit(Some(8)), |
| 752 | Err(SearchSpecError::InvalidConcurrency { |
| 753 | concurrency: 16, |
| 754 | population: 32, |
| 755 | limit: 8, |
| 756 | }) |
| 757 | ); |
| 758 | |
| 759 | // ...and a larger one admits concurrency the fallback would refuse. |
| 760 | spec.concurrency = 24; |
| 761 | let resolved = spec |
| 762 | .validate_with_limit(Some(32)) |
| 763 | .expect("24 workers fit a 32-wide Fleet limit"); |
| 764 | assert_eq!( |
| 765 | resolved, |
| 766 | ResolvedSearchConcurrency { |
| 767 | limit: 32, |
| 768 | source: SearchConcurrencySource::FleetLimit, |
| 769 | } |
| 770 | ); |
| 771 | let batches = spec |
| 772 | .admission_batches_with_limit(Some(32)) |
| 773 | .expect("validated admission"); |
| 774 | assert_eq!(batches.len(), 2); |
| 775 | assert_eq!(batches[0].len(), 24); |
| 776 | } |
| 777 | |
| 778 | #[test] |
| 779 | fn fallback_ceiling_stays_sixteen_when_no_limit_resolves() { |
| 780 | let mut spec = WorkflowSearchSpec::from_toml(SPEC).expect("valid search spec"); |
| 781 | assert_eq!( |
| 782 | spec.validate_with_limit(None), |
| 783 | Ok(ResolvedSearchConcurrency { |
| 784 | limit: WORKFLOW_SEARCH_DEFAULT_MAX_CONCURRENT, |
| 785 | source: SearchConcurrencySource::Default, |
| 786 | }) |
| 787 | ); |
| 788 | assert_eq!(WORKFLOW_SEARCH_DEFAULT_MAX_CONCURRENT, 16); |
| 789 | |
| 790 | // A zero limit is not a ceiling of zero — it is an unresolved one. |
| 791 | assert_eq!( |
| 792 | spec.validate_with_limit(Some(0)), |
| 793 | spec.validate_with_limit(None) |
| 794 | ); |
| 795 | |
| 796 | spec.concurrency = 17; |
| 797 | assert_eq!( |
| 798 | spec.validate_with_limit(None), |
| 799 | Err(SearchSpecError::InvalidConcurrency { |
| 800 | concurrency: 17, |
| 801 | population: 32, |
| 802 | limit: 16, |
| 803 | }) |
| 804 | ); |
| 805 | } |
| 806 | |
| 807 | #[test] |
| 808 | fn fleet_config_seam_takes_the_lower_present_bound() { |
| 809 | assert_eq!( |
| 810 | ResolvedSearchConcurrency::from_fleet_config(Some(16), Some(4)), |
| 811 | ResolvedSearchConcurrency { |
| 812 | limit: 4, |
| 813 | source: SearchConcurrencySource::FleetLimit, |
| 814 | } |
| 815 | ); |
| 816 | assert_eq!( |
| 817 | ResolvedSearchConcurrency::from_fleet_config(Some(6), None), |
| 818 | ResolvedSearchConcurrency { |
| 819 | limit: 6, |
| 820 | source: SearchConcurrencySource::FleetLimit, |
| 821 | } |
| 822 | ); |
| 823 | assert_eq!( |
| 824 | ResolvedSearchConcurrency::from_fleet_config(None, None), |
| 825 | ResolvedSearchConcurrency::default() |
| 826 | ); |
| 827 | assert_eq!( |
| 828 | ResolvedSearchConcurrency::default().source, |
| 829 | SearchConcurrencySource::Default |
| 830 | ); |
| 831 | } |
| 832 | |
| 833 | #[test] |
| 834 | fn freeze_receipt_echoes_the_bound_that_applied() { |
| 835 | let spec = WorkflowSearchSpec::from_toml(SPEC).expect("valid search spec"); |
| 836 | |
| 837 | let fallback = spec |
| 838 | .freeze("33bc6a98", "DeepSeek-V4-Flash-0731", b"evidence", b"eval") |
| 839 | .expect("freeze succeeds"); |
| 840 | assert_eq!( |
| 841 | fallback.resolved_concurrency, |
| 842 | ResolvedSearchConcurrency { |
| 843 | limit: 16, |
| 844 | source: SearchConcurrencySource::Default, |
| 845 | } |
| 846 | ); |
| 847 | assert_eq!( |
| 848 | fallback.concurrency_receipt_line(), |
| 849 | "live-worker ceiling 16 (default, no Fleet limit resolved)" |
| 850 | ); |
| 851 | |
| 852 | let bounded = spec |
| 853 | .freeze_with_limit( |
| 854 | "33bc6a98", |
| 855 | "DeepSeek-V4-Flash-0731", |
| 856 | b"evidence", |
| 857 | b"eval", |
| 858 | Some(16), |
| 859 | ) |
| 860 | .expect("freeze succeeds"); |
| 861 | assert_eq!( |
| 862 | bounded.resolved_concurrency, |
| 863 | ResolvedSearchConcurrency { |
| 864 | limit: 16, |
| 865 | source: SearchConcurrencySource::FleetLimit, |
| 866 | } |
| 867 | ); |
| 868 | assert_eq!( |
| 869 | bounded.concurrency_receipt_line(), |
| 870 | "live-worker ceiling 16 (resolved Fleet limit)" |
| 871 | ); |
| 872 | |
| 873 | // The preregistration identity is unchanged by the operational bound. |
| 874 | assert_eq!( |
| 875 | fallback.preregistration_hash, bounded.preregistration_hash, |
| 876 | "the resolved ceiling must not perturb the preregistration hash" |
| 877 | ); |
| 878 | |
| 879 | let receipt = serde_json::to_value(&bounded).expect("receipt serializes"); |
| 880 | assert_eq!(receipt["resolved_concurrency"]["limit"], 16); |
| 881 | assert_eq!(receipt["resolved_concurrency"]["source"], "fleet_limit"); |
| 882 | } |
| 883 | |
| 884 | #[test] |
| 885 | fn admission_refuses_unvalidated_zero_concurrency_without_panicking() { |
| 886 | let mut spec = WorkflowSearchSpec::from_toml(SPEC).expect("valid search spec"); |
| 887 | spec.concurrency = 0; |
| 888 | |
| 889 | assert_eq!( |
| 890 | spec.admission_batches(), |
| 891 | Err(SearchSpecError::InvalidConcurrency { |
| 892 | concurrency: 0, |
| 893 | population: 32, |
| 894 | limit: 16, |
| 895 | }) |
| 896 | ); |
| 897 | } |
| 898 | |
| 899 | #[test] |
| 900 | fn rejects_write_scopes_that_escape_or_obscure_the_repo_boundary() { |
| 901 | let mut spec = WorkflowSearchSpec::from_toml(SPEC).expect("valid search spec"); |
| 902 | for unsafe_path in ["../outside", "/tmp/outside", r"C:\outside"] { |
| 903 | spec.worker.write_roots = vec![unsafe_path.to_string()]; |
| 904 | assert_eq!( |
| 905 | spec.validate(), |
| 906 | Err(SearchSpecError::InvalidWriteScope { |
| 907 | field: "worker.write_roots", |
| 908 | }), |
| 909 | "path should be rejected: {unsafe_path}" |
| 910 | ); |
| 911 | } |
| 912 | } |
| 913 | |
| 914 | #[test] |
| 915 | fn deserialization_refuses_auto_merge_policy() { |
| 916 | let source = SPEC.replace("review_only", "auto_merge"); |
| 917 | let error = WorkflowSearchSpec::from_toml(&source).expect_err("must reject auto merge"); |
| 918 | |
| 919 | assert!(matches!(error, SearchSpecError::Parse(_))); |
| 920 | } |
| 921 | } |
| 922 |