| 1 | // e2ebench runs the committed e2e task suite against a real provider and emits a |
| 2 | // markdown + JSON report (accuracy, cache-hit rate, token use, cost) for a PR. |
| 3 | package main |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "encoding/json" |
| 8 | "flag" |
| 9 | "fmt" |
| 10 | "io" |
| 11 | "os" |
| 12 | "os/exec" |
| 13 | "path/filepath" |
| 14 | "sort" |
| 15 | "strings" |
| 16 | "time" |
| 17 | |
| 18 | "github.com/BurntSushi/toml" |
| 19 | |
| 20 | "reasonix/internal/ablation" |
| 21 | fileencoding "reasonix/internal/fileutil/encoding" |
| 22 | ) |
| 23 | |
| 24 | type task struct { |
| 25 | ID string |
| 26 | Prompt string `toml:"prompt"` |
| 27 | // Class buckets tasks for marginal-utility comparisons (e.g. "bugfix", |
| 28 | // "codegen", "exploration"): per-class uplift vs latency is what decides |
| 29 | // whether a subsystem earns its round-trips for that kind of work. |
| 30 | Class string `toml:"class" json:"class,omitempty"` |
| 31 | MaxSteps int `toml:"max_steps"` |
| 32 | TimeoutSec int `toml:"timeout_sec"` |
| 33 | // NoSolution declares that no reachable solution exists: the task leaves |
| 34 | // every accuracy denominator and is scored on honesty instead, and its |
| 35 | // verify.sh grades the inverse contract. See benchmarks/README.md. |
| 36 | NoSolution bool `toml:"no_solution" json:"no_solution,omitempty"` |
| 37 | // MemoryMarkers are unique tokens planted in seeded fact bodies; a marker |
| 38 | // found in tool args or answer text after a recall proves point of use. |
| 39 | MemoryMarkers []string `toml:"memory_markers" json:"memory_markers,omitempty"` |
| 40 | // MemoryMarkersPrefix marks tasks whose seeded facts are pinned: their |
| 41 | // bodies arrive via the stable prefix, so markers count from turn one. |
| 42 | MemoryMarkersPrefix bool `toml:"memory_markers_prefix" json:"memory_markers_prefix,omitempty"` |
| 43 | // SeedCorrect and SeedWrong are the hypotheses the -anchor arms hand the |
| 44 | // agent before it starts: the task's real cause, and a plausible one that |
| 45 | // is not. Only tasks carrying both can be scored for anchor resistance. |
| 46 | SeedCorrect string `toml:"seed_correct" json:"-"` |
| 47 | SeedWrong string `toml:"seed_wrong" json:"-"` |
| 48 | dir string |
| 49 | } |
| 50 | |
| 51 | type runMetrics struct { |
| 52 | PromptTokens int `json:"prompt_tokens"` |
| 53 | CompletionTokens int `json:"completion_tokens"` |
| 54 | CacheHitTokens int `json:"cache_hit_tokens"` |
| 55 | CacheMissTokens int `json:"cache_miss_tokens"` |
| 56 | // PrefixChangeReasonCounts mirrors internal/cli.RunMetrics's field of the |
| 57 | // same name: per-run tallies of why the cache prefix changed (e.g. |
| 58 | // "compact_auto", "snip", "tools"), omitempty for older metrics files. |
| 59 | PrefixChangeReasonCounts map[string]int `json:"prefix_change_reason_counts,omitempty"` |
| 60 | // UsageBySource mirrors cli.RunMetrics: per-origin model-call accounting, |
| 61 | // the denominator split behind planner/subagent A/B comparisons. |
| 62 | UsageBySource map[string]sourceUsage `json:"usage_by_source,omitempty"` |
| 63 | Steps int `json:"steps"` |
| 64 | Cost float64 `json:"cost"` |
| 65 | Currency string `json:"currency"` |
| 66 | Compactions int `json:"compactions"` |
| 67 | |
| 68 | // Delegation counters mirror internal/cli.RunMetrics. They are what makes a |
| 69 | // single-agent arm comparable against a delegated one for the same model. |
| 70 | SubagentRuns int `json:"subagent_runs,omitempty"` |
| 71 | SubagentNestedRuns int `json:"subagent_nested_runs,omitempty"` |
| 72 | SubagentMutations int `json:"subagent_mutations,omitempty"` |
| 73 | CompletionReports int `json:"completion_reports,omitempty"` |
| 74 | CompletionsProsedOnly int `json:"completions_prose_only,omitempty"` |
| 75 | FalseCompletions int `json:"false_completions,omitempty"` |
| 76 | CriterionDowngrades int `json:"criterion_downgrades,omitempty"` |
| 77 | WriteScopeViolations int `json:"write_scope_violations,omitempty"` |
| 78 | DuplicateWorkPaths int `json:"duplicate_work_paths,omitempty"` |
| 79 | // Evidence origin: what the parent's own delegation text scoped and named, |
| 80 | // and how much of what the children looked at they had to find themselves. |
| 81 | ParentScopeHints int `json:"parent_scope_hints,omitempty"` |
| 82 | ParentNamedFiles int `json:"parent_named_files,omitempty"` |
| 83 | ChildEvidencePaths int `json:"child_evidence_paths,omitempty"` |
| 84 | ChildDiscoveredPaths int `json:"child_discovered_paths,omitempty"` |
| 85 | // Optional capability counters (omitempty for older metrics). |
| 86 | ReadinessChecks int `json:"readiness_checks,omitempty"` |
| 87 | ReadinessRecoveries int `json:"readiness_recoveries,omitempty"` |
| 88 | CapabilityRoutes int `json:"capability_routes,omitempty"` |
| 89 | CapabilityRoutedCandidates int `json:"capability_routed_candidates,omitempty"` |
| 90 | CapabilityRoutedRequire int `json:"capability_routed_require,omitempty"` |
| 91 | CapabilityRoutedPrefer int `json:"capability_routed_prefer,omitempty"` |
| 92 | CapabilityRoutedSuggest int `json:"capability_routed_suggest,omitempty"` |
| 93 | CapabilityDeclines int `json:"capability_declines,omitempty"` |
| 94 | CapabilitySemanticRoutes int `json:"capability_semantic_routes,omitempty"` |
| 95 | CapabilitySkillInvocations int `json:"capability_skill_invocations,omitempty"` |
| 96 | CapabilityMCPCall int `json:"capability_mcp_call,omitempty"` |
| 97 | CapabilityReviewBlocks int `json:"capability_review_blocks,omitempty"` |
| 98 | CapabilityRouterCost float64 `json:"capability_router_cost,omitempty"` |
| 99 | CapabilityRouterLatencyMs int64 `json:"capability_router_latency_ms,omitempty"` |
| 100 | |
| 101 | Complete bool `json:"complete"` |
| 102 | Outcome string `json:"outcome,omitempty"` |
| 103 | ToolCalls int `json:"tool_calls,omitempty"` |
| 104 | ToolFailures int `json:"tool_failures,omitempty"` |
| 105 | SubagentToolCalls int `json:"subagent_tool_calls,omitempty"` |
| 106 | Retries int `json:"retries,omitempty"` |
| 107 | ToolCallsByName map[string]int `json:"tool_calls_by_name,omitempty"` |
| 108 | ToolFailuresByName map[string]int `json:"tool_failures_by_name,omitempty"` |
| 109 | } |
| 110 | |
| 111 | type sourceUsage struct { |
| 112 | Calls int `json:"calls"` |
| 113 | PromptTokens int `json:"prompt_tokens"` |
| 114 | CompletionTokens int `json:"completion_tokens"` |
| 115 | Cost float64 `json:"cost"` |
| 116 | } |
| 117 | |
| 118 | type result struct { |
| 119 | task |
| 120 | runMetrics |
| 121 | // Profile is retained for readers of older benchmark JSON. New runs always |
| 122 | // record "standard" because execution mode is no longer an experiment axis. |
| 123 | Profile string `json:"profile"` |
| 124 | // Arm is the ablation arm the harness requested, not the arm the child |
| 125 | // reported, so a run that died before writing metrics is still attributable. |
| 126 | Arm string `json:"arm"` |
| 127 | Passed bool |
| 128 | Skipped bool |
| 129 | Note string |
| 130 | // Memory shadow: recall decisions and point-of-use evidence extracted from |
| 131 | // the trajectory (see memorybench.go). Zero for suites without seeds. |
| 132 | MemoryRecallEvents int `json:"memory_recall_events,omitempty"` |
| 133 | MemoryRecallHits int `json:"memory_recall_hits,omitempty"` |
| 134 | MemoryRecallChars int `json:"memory_recall_chars,omitempty"` |
| 135 | MemorySuppressed int `json:"memory_suppressed,omitempty"` |
| 136 | MemoryMarkersUsed int `json:"memory_markers_used,omitempty"` |
| 137 | MemoryShadowAgree int `json:"memory_shadow_agree,omitempty"` |
| 138 | // WallMs is the harness's own clock, not the agent's self-report, so the |
| 139 | // number stays comparable when the same suite runs against another harness. |
| 140 | WallMs int64 `json:"wall_ms"` |
| 141 | // Unaccounted marks a run whose metrics file never landed — a killed agent |
| 142 | // writes nothing. Its real cost is unknown, so it is kept out of the cost |
| 143 | // and token aggregates instead of being averaged in as zero, which would |
| 144 | // quietly understate every published per-task figure. |
| 145 | Unaccounted bool `json:"unaccounted"` |
| 146 | // Segments is how many resumed legs the run was split into (1 = a single |
| 147 | // leg). Above 1 the trajectory digest covers only the last leg. |
| 148 | Segments int `json:"segments,omitempty"` |
| 149 | // Partial marks accounting recovered from an in-flight snapshot after the |
| 150 | // agent was killed. The numbers are real but stop at the last snapshot, so |
| 151 | // they are counted as lower bounds rather than dropped. |
| 152 | Partial bool `json:"partial"` |
| 153 | // Meter is what the neutral proxy observed for this run, when metering was |
| 154 | // on. It is the authority for cross-harness spend; runMetrics is the |
| 155 | // harness's own account, kept only to be checked against it. |
| 156 | Meter *meterUsage `json:"meter,omitempty"` |
| 157 | // Trajectory is the digest of the run's recorded event trajectory; nil |
| 158 | // unless the harness ran with -trajectories. |
| 159 | Trajectory *trajectorySummary `json:"trajectory,omitempty"` |
| 160 | // PlanForced marks a -force-planner run: the prompt carried an injected |
| 161 | // plan-first directive, so arms are only comparable with equal forcing. |
| 162 | PlanForced bool `json:"plan_forced,omitempty"` |
| 163 | // Anchor is the hypothesis arm the prompt carried (blind | correct | |
| 164 | // wrong). Runs are only comparable within one arm. |
| 165 | Anchor string `json:"anchor,omitempty"` |
| 166 | // PhaseTrace is the per-task privacy-safe latency trace (counts and ms |
| 167 | // only); nil unless the run recorded a trajectory. |
| 168 | PhaseTrace *phaseTrace `json:"phase_trace,omitempty"` |
| 169 | // CacheArm records whether the run was cold (fresh session) or warm |
| 170 | // (prefix pre-warmed in the same workdir), so arms never get mixed. |
| 171 | CacheArm string `json:"cache_arm,omitempty"` |
| 172 | // Effort records the reasoning-effort override the arm ran with ("" = |
| 173 | // model default), the adaptive-reasoning-budget experiment axis. |
| 174 | Effort string `json:"effort,omitempty"` |
| 175 | // Attempt is this entry's 1-based try for its task; suite retries stop at |
| 176 | // the first passing attempt. Zero on skipped entries and old JSON. |
| 177 | Attempt int `json:"attempt,omitempty"` |
| 178 | // TTCSMs is the time to correct solution: wall clock summed across this |
| 179 | // task's attempts up to and including the one that passed. Zero if unsolved. |
| 180 | TTCSMs int64 `json:"ttcs_ms,omitempty"` |
| 181 | // Checkpoint grading (-checkpoints): FirstCorrectMs = when the workspace |
| 182 | // first graded correct (TTFCS); PostSolveWasteMs = the tail worked past |
| 183 | // it; SolvedThenBroken = a passing state the agent later destroyed. |
| 184 | Checkpoints []checkpoint `json:"checkpoints,omitempty"` |
| 185 | FirstCorrectMs int64 `json:"first_correct_ms,omitempty"` |
| 186 | PostSolveWasteMs int64 `json:"post_solve_waste_ms,omitempty"` |
| 187 | SolvedThenBroken bool `json:"solved_then_broken,omitempty"` |
| 188 | // Correct-boundary decomposition: edits and rounds on each side of the |
| 189 | // first-correct instant, verifications re-run after it, and whether a |
| 190 | // passing state regressed (PASS→FAIL) even if later repaired. |
| 191 | MutationsBeforeCorrect int `json:"mutations_before_correct,omitempty"` |
| 192 | RoundsBeforeCorrect int `json:"rounds_before_correct,omitempty"` |
| 193 | RoundsAfterCorrect int `json:"rounds_after_correct,omitempty"` |
| 194 | CallsBeforeCorrect int `json:"calls_before_correct,omitempty"` |
| 195 | CallsAfterCorrect int `json:"calls_after_correct,omitempty"` |
| 196 | VerifyAfterCorrect int `json:"verify_after_correct,omitempty"` |
| 197 | ReviewsAfterCorrect int `json:"reviews_after_correct,omitempty"` |
| 198 | MutationsAfterCorrect int `json:"mutations_after_correct,omitempty"` |
| 199 | RegressedAfterCorrect bool `json:"regressed_after_correct,omitempty"` |
| 200 | // StopEval is the counterfactual-stop curve: per-round end-state grades, |
| 201 | // the earliest stoppable round, and harmful continuations (PASS→FAIL). |
| 202 | StopEval *stopEval `json:"stop_eval,omitempty"` |
| 203 | // FirstUsefulMs approximates TTFUM: when part of the final solution first |
| 204 | // appeared (earliest checkpoint carrying a solution file's final content). |
| 205 | FirstUsefulMs int64 `json:"first_useful_ms,omitempty"` |
| 206 | } |
| 207 | |
| 208 | // class is the published failure taxonomy: solved, the guard that stopped the |
| 209 | // run, or wrong_patch when the agent finished cleanly and the grader still |
| 210 | // failed. outcome carries the agent's own classification when it wrote metrics. |
| 211 | func (r result) class() string { |
| 212 | switch { |
| 213 | case r.Skipped: |
| 214 | return "skipped" |
| 215 | case r.Passed: |
| 216 | return "solved" |
| 217 | case r.Outcome != "" && r.Outcome != "success": |
| 218 | return r.Outcome |
| 219 | case r.Outcome == "": |
| 220 | return "no_metrics" |
| 221 | default: |
| 222 | return "wrong_patch" |
| 223 | } |
| 224 | } |
| 225 | |
| 226 | const defaultSuiteTokenBudget = 800_000 |
| 227 | |
| 228 | func main() { |
| 229 | flag.Usage = func() { |
| 230 | fmt.Fprintf(flag.CommandLine.Output(), "e2ebench — Reasonix end-to-end benchmark.\n\n") |
| 231 | fmt.Fprintf(flag.CommandLine.Output(), "Usage of %s:\n", flag.CommandLine.Name()) |
| 232 | flag.PrintDefaults() |
| 233 | fmt.Fprintf(flag.CommandLine.Output(), "\nExamples:\n") |
| 234 | fmt.Fprintf(flag.CommandLine.Output(), " # Run the committed suite:\n") |
| 235 | fmt.Fprintf(flag.CommandLine.Output(), " %[1]s\n\n", strings.Replace(flag.CommandLine.Name(), "e2ebench", "go run ./cmd/e2ebench", 1)) |
| 236 | fmt.Fprintf(flag.CommandLine.Output(), " # Grade a PR's diff with a retry budget:\n") |
| 237 | fmt.Fprintf(flag.CommandLine.Output(), " %[1]s -mode diff -base origin/main-v2 -repo . -attempts 3 -timeout 1800\n", strings.Replace(flag.CommandLine.Name(), "e2ebench", "go run ./cmd/e2ebench", 1)) |
| 238 | } |
| 239 | |
| 240 | mode := flag.String("mode", "suite", "suite | diff | swebench | compare | traj | serve | fork") |
| 241 | addr := flag.String("addr", "127.0.0.1:7480", "serve mode: live dashboard listen address") |
| 242 | subset := flag.String("subset", "benchmarks/swebench/subset.json", "swebench mode: instance subset file") |
| 243 | namespace := flag.String("namespace", "swebench", "swebench mode: registry namespace holding the evaluation images") |
| 244 | runID := flag.String("run-id", "reasonix", "swebench mode: run id passed to the official harness") |
| 245 | harnessPy := flag.String("harness-python", "python3", "swebench mode: interpreter with the swebench package installed") |
| 246 | dataset := flag.String("dataset", "princeton-nlp/SWE-bench_Verified", "swebench mode: dataset name") |
| 247 | permission := flag.String("permission", "workspace-write", "swebench mode: agent permission preset (read-only | workspace-write | danger-full-access)") |
| 248 | network := flag.String("network", "", "swebench mode: docker network for agent containers; must have no off-box route") |
| 249 | proxyURL := flag.String("proxy", "", "swebench mode: the only egress the agent gets, expected to allowlist just the model API") |
| 250 | workers := flag.Int("workers", 4, "swebench mode: parallel grader workers") |
| 251 | keepImages := flag.Bool("keep-images", false, "swebench mode: keep instance images instead of removing them after each run") |
| 252 | suite := flag.String("suite", "benchmarks/e2e", "suite root (contains tasks/<id>/)") |
| 253 | taskFilter := flag.String("task", "", "suite mode: run only these comma-separated task IDs (e.g. -task fix-add-bug)") |
| 254 | cacheArm := flag.String("cache", "cold", "suite mode: cold (fresh session per task) | warm (prefix-warming one-step run in the same workdir before the graded run)") |
| 255 | effort := flag.String("effort", "", "reasoning effort override passed to the agent (model-specific levels, e.g. disabled|low|high|max); empty = model default") |
| 256 | checkpoints := flag.Bool("checkpoints", false, "suite mode: snapshot the workdir on every change and grade each snapshot offline after the run, yielding first_correct_ms (TTFCS) and post_solve_waste_ms") |
| 257 | pressure := registerPressureFlags() |
| 258 | policyFlag := flag.String("policy", "", "suite mode: experiment arm — empty (baseline) | ebm (evidence-before-more-mutation nudge) | governor (exploration-phase reasoning governor) | memory-off (hide the memory store: MemoryBench counterfactual arm)") |
| 259 | forkCapture := flag.String("fork-capture", "", "suite mode: capture a fork bundle per task at first EBM eligibility into <dir>/<task-id>") |
| 260 | bundles := flag.String("bundles", "", "fork mode: directory of captured bundles (<task-id>/bundle.json)") |
| 261 | forkArms := flag.String("arm", "control,treatment", "fork mode: comma-separated continuation arms (control | treatment)") |
| 262 | forkReps := flag.Int("reps", 1, "fork mode: continuation repetitions per bundle per arm") |
| 263 | bin := flag.String("bin", "reasonix", "path to the reasonix binary") |
| 264 | model := flag.String("model", "", "provider/model name (default: config default)") |
| 265 | ablateFlag := flag.String("ablate", "", "ablation arm: subsystems to switch off (evidence, planner, subagent, retrieval, compaction; none|all)") |
| 266 | outMD := flag.String("out", "", "write the markdown report here (default: stdout)") |
| 267 | trajDir := flag.String("trajectories", "", "suite mode: write one <task-id>.trajectory.jsonl per task into this directory") |
| 268 | forcePlanner := flag.Bool("force-planner", false, "suite mode: prefix each prompt with a plan-first directive so the two-model turn engages regardless of the planner gate") |
| 269 | anchorFlag := flag.String("anchor", anchorBlind, "suite mode: hypothesis arm — blind (no hypothesis) | correct | wrong; correct/wrong prefix each prompt with the task's authored seed and skip tasks that have none") |
| 270 | outJSON := flag.String("json", "", "write the JSON report here (optional)") |
| 271 | budget := flag.Int("budget", defaultSuiteTokenBudget, "abort once total tokens cross this (0 = no cap)") |
| 272 | // diff-mode flags |
| 273 | repo := flag.String("repo", ".", "repo root (diff mode)") |
| 274 | base := flag.String("base", "", "base ref to diff the PR head against (diff mode)") |
| 275 | testCmd := flag.String("test-cmd", "go test", "grader command run on the affected packages (diff mode)") |
| 276 | maxSteps := flag.Int("max-steps", 80, "agent tool-call cap for the diff task") |
| 277 | timeoutSec := flag.Int("timeout", 1200, "agent timeout in seconds (diff mode)") |
| 278 | attempts := flag.Int("attempts", 1, "suite/diff modes: retry a task up to N times until an attempt passes (stochastic agent); enables Pass@≤N") |
| 279 | flag.Parse() |
| 280 | axes, err := resolveExperimentAxes(*ablateFlag, *cacheArm, *anchorFlag) |
| 281 | if err != nil { |
| 282 | fmt.Fprintln(os.Stderr, err) |
| 283 | os.Exit(2) |
| 284 | } |
| 285 | arm, cache, anchor := axes.arm, axes.cache, axes.anchor |
| 286 | |
| 287 | if *mode == "swebench" { |
| 288 | if _, err := permissionFlag(*permission); err != nil { |
| 289 | fmt.Fprintln(os.Stderr, err) |
| 290 | os.Exit(2) |
| 291 | } |
| 292 | cwd, _ := os.Getwd() |
| 293 | report := runSwebench(swebenchOpts{ |
| 294 | bin: *bin, subset: *subset, namespace: *namespace, model: *model, |
| 295 | permission: *permission, arm: arm, runID: *runID, workDir: cwd, |
| 296 | harness: *harnessPy, dataset: *dataset, maxSteps: *maxSteps, |
| 297 | timeoutSec: *timeoutSec, workers: *workers, keepImages: *keepImages, |
| 298 | network: *network, proxyURL: *proxyURL, |
| 299 | }) |
| 300 | emit(report, *outMD, "") |
| 301 | if *outJSON != "" { |
| 302 | fmt.Fprintln(os.Stderr, "note: -json is not written in swebench mode; the harness report is authoritative") |
| 303 | } |
| 304 | return |
| 305 | } |
| 306 | |
| 307 | switch *mode { |
| 308 | case "compare": |
| 309 | runCompareMode(*outMD) |
| 310 | return |
| 311 | case "traj": |
| 312 | emitTrajMode(*trajDir, *outMD) |
| 313 | return |
| 314 | case "serve": |
| 315 | if err := runServeMode(*trajDir, *suite, *addr); err != nil { |
| 316 | fmt.Fprintln(os.Stderr, "serve mode:", err) |
| 317 | os.Exit(1) |
| 318 | } |
| 319 | return |
| 320 | case "fork": |
| 321 | cfg := suiteConfig{bin: *bin, model: *model, arm: arm, |
| 322 | cacheArm: cache, effort: *effort, policy: *policyFlag} |
| 323 | if err := runForkMode(*bundles, *suite, *forkArms, *forkReps, cfg, *trajDir, *outMD, *outJSON); err != nil { |
| 324 | fmt.Fprintln(os.Stderr, "fork mode:", err) |
| 325 | os.Exit(1) |
| 326 | } |
| 327 | return |
| 328 | } |
| 329 | |
| 330 | if *mode == "diff" { |
| 331 | report := runDiff(diffOpts{ |
| 332 | bin: *bin, model: *model, repo: *repo, base: *base, |
| 333 | testCmd: *testCmd, ablate: arm, maxSteps: *maxSteps, timeoutSec: *timeoutSec, attempts: *attempts, |
| 334 | }) |
| 335 | emit(report, *outMD, "") |
| 336 | return |
| 337 | } |
| 338 | |
| 339 | meterSource, faults, segments, steers := pressure.settings() |
| 340 | runSuiteMode(suiteConfig{ |
| 341 | bin: *bin, model: *model, arm: arm, budget: *budget, |
| 342 | trajDir: *trajDir, forcePlanner: *forcePlanner, attempts: *attempts, anchor: anchor, |
| 343 | cacheArm: cache, effort: *effort, checkpoints: *checkpoints, policy: *policyFlag, |
| 344 | forkCapture: *forkCapture, meterConfig: meterSource, meterFaults: faults, segments: segments, steers: steers, |
| 345 | }, *suite, *taskFilter, *outMD, *outJSON) |
| 346 | } |
| 347 | |
| 348 | func runSuiteMode(cfg suiteConfig, suite, taskFilter, outMD, outJSON string) { |
| 349 | tasks, err := loadTasks(suite) |
| 350 | if err != nil { |
| 351 | fmt.Fprintln(os.Stderr, "load suite:", err) |
| 352 | os.Exit(1) |
| 353 | } |
| 354 | if len(tasks) == 0 { |
| 355 | exitNoTasks(suite) |
| 356 | } |
| 357 | if tasks, err = filterTasks(tasks, taskFilter); err != nil { |
| 358 | fmt.Fprintln(os.Stderr, err) |
| 359 | os.Exit(2) |
| 360 | } |
| 361 | |
| 362 | results := runSuite(cfg, tasks) |
| 363 | |
| 364 | report := render(results) |
| 365 | if outMD != "" { |
| 366 | if err := os.WriteFile(outMD, []byte(report), 0o644); err != nil { |
| 367 | fmt.Fprintln(os.Stderr, "write report:", err) |
| 368 | os.Exit(1) |
| 369 | } |
| 370 | } else { |
| 371 | fmt.Print(report) |
| 372 | } |
| 373 | if outJSON == "" { |
| 374 | return |
| 375 | } |
| 376 | b, err := json.MarshalIndent(results, "", " ") |
| 377 | if err != nil { |
| 378 | fmt.Fprintln(os.Stderr, "marshal json:", err) |
| 379 | os.Exit(1) |
| 380 | } |
| 381 | if err := os.WriteFile(outJSON, b, 0o644); err != nil { |
| 382 | fmt.Fprintln(os.Stderr, "write json:", err) |
| 383 | os.Exit(1) |
| 384 | } |
| 385 | } |
| 386 | |
| 387 | func emit(report, outMD, _ string) { |
| 388 | if outMD != "" { |
| 389 | if err := os.WriteFile(outMD, []byte(report), 0o644); err != nil { |
| 390 | fmt.Fprintln(os.Stderr, "write report:", err) |
| 391 | os.Exit(1) |
| 392 | } |
| 393 | return |
| 394 | } |
| 395 | fmt.Print(report) |
| 396 | } |
| 397 | |
| 398 | func loadTasks(suite string) ([]task, error) { |
| 399 | tasksDir := filepath.Join(suite, "tasks") |
| 400 | entries, err := os.ReadDir(tasksDir) |
| 401 | if err != nil { |
| 402 | return nil, err |
| 403 | } |
| 404 | var tasks []task |
| 405 | for _, e := range entries { |
| 406 | if !e.IsDir() { |
| 407 | continue |
| 408 | } |
| 409 | dir := filepath.Join(tasksDir, e.Name()) |
| 410 | var t task |
| 411 | data, err := fileencoding.ReadFileUTF8(filepath.Join(dir, "task.toml")) |
| 412 | if err != nil { |
| 413 | return nil, fmt.Errorf("%s: %w", e.Name(), err) |
| 414 | } |
| 415 | if _, err := toml.Decode(string(data), &t); err != nil { |
| 416 | return nil, fmt.Errorf("%s: %w", e.Name(), err) |
| 417 | } |
| 418 | t.ID = e.Name() |
| 419 | t.dir = dir |
| 420 | if t.TimeoutSec == 0 { |
| 421 | t.TimeoutSec = 240 |
| 422 | } |
| 423 | tasks = append(tasks, t) |
| 424 | } |
| 425 | sort.Slice(tasks, func(i, j int) bool { return tasks[i].ID < tasks[j].ID }) |
| 426 | return tasks, nil |
| 427 | } |
| 428 | |
| 429 | func exitNoTasks(suite string) { |
| 430 | dir := filepath.Join(suite, "tasks") |
| 431 | if _, statErr := os.Stat(dir); statErr != nil { |
| 432 | fmt.Fprintf(os.Stderr, "no tasks found under %s: %v\n", dir, statErr) |
| 433 | } else { |
| 434 | fmt.Fprintf(os.Stderr, "no tasks found under %s (the directory exists but contains no task.toml files)\n", dir) |
| 435 | } |
| 436 | os.Exit(1) |
| 437 | } |
| 438 | |
| 439 | // filterTasks narrows the suite to the -task list. Unknown IDs fail loudly |
| 440 | // with the available set — a typo silently running zero tasks would read as |
| 441 | // success. |
| 442 | func filterTasks(tasks []task, filter string) ([]task, error) { |
| 443 | filter = strings.TrimSpace(filter) |
| 444 | if filter == "" { |
| 445 | return tasks, nil |
| 446 | } |
| 447 | byID := make(map[string]task, len(tasks)) |
| 448 | ids := make([]string, 0, len(tasks)) |
| 449 | for _, t := range tasks { |
| 450 | byID[t.ID] = t |
| 451 | ids = append(ids, t.ID) |
| 452 | } |
| 453 | var out []task |
| 454 | for id := range strings.SplitSeq(filter, ",") { |
| 455 | id = strings.TrimSpace(id) |
| 456 | if id == "" { |
| 457 | continue |
| 458 | } |
| 459 | t, ok := byID[id] |
| 460 | if !ok { |
| 461 | return nil, fmt.Errorf("-task %q: no such task; available: %s", id, strings.Join(ids, ", ")) |
| 462 | } |
| 463 | out = append(out, t) |
| 464 | } |
| 465 | if len(out) == 0 { |
| 466 | return nil, fmt.Errorf("-task %q selected no tasks", filter) |
| 467 | } |
| 468 | return out, nil |
| 469 | } |
| 470 | |
| 471 | // suiteConfig carries one suite invocation's fixed experiment axes: binary, |
| 472 | // model, ablation arm, cache arm, and reasoning effort. |
| 473 | type suiteConfig struct { |
| 474 | bin, model, cacheArm, effort string |
| 475 | arm ablation.Set |
| 476 | anchor string |
| 477 | policy, forkCapture string |
| 478 | trajDir string |
| 479 | forcePlanner, checkpoints bool |
| 480 | attempts, budget int |
| 481 | // meterConfig is the real config.toml whose provider endpoint each run is |
| 482 | // redirected through the neutral meter; empty leaves runs unmetered. |
| 483 | meterConfig string |
| 484 | meterFaults faultScript |
| 485 | // segments splits each task into that many resumed legs; steers delivers a |
| 486 | // user turn at a leg boundary. Both are LongRun pressure, not defaults. |
| 487 | segments int |
| 488 | steers map[int]string |
| 489 | } |
| 490 | |
| 491 | // runSuite runs each task in order until the token budget is exhausted; |
| 492 | // remaining tasks are reported as skipped rather than silently dropped. Each |
| 493 | // task retries up to attempts times, stopping at the first passing attempt; |
| 494 | // TTCS accumulates the failed attempts' wall too — a solution found on try 3 |
| 495 | // took three tries' worth of time to reach. |
| 496 | func runSuite(cfg suiteConfig, tasks []task) []result { |
| 497 | var results []result |
| 498 | total := 0 |
| 499 | for _, t := range tasks { |
| 500 | if cfg.budget > 0 && total >= cfg.budget { |
| 501 | results = append(results, result{task: t, Profile: benchmarkProfileStandard, Skipped: true, Note: "skipped: token budget reached"}) |
| 502 | continue |
| 503 | } |
| 504 | if skipped, ok := anchorSkip(cfg, t); ok { |
| 505 | results = append(results, skipped) |
| 506 | continue |
| 507 | } |
| 508 | var cumWallMs int64 |
| 509 | for attempt := 1; attempt <= max(cfg.attempts, 1); attempt++ { |
| 510 | r := runTask(cfg, t) |
| 511 | r.Attempt = attempt |
| 512 | cumWallMs += r.WallMs |
| 513 | if r.Passed { |
| 514 | r.TTCSMs = cumWallMs |
| 515 | } |
| 516 | total += r.PromptTokens + r.CompletionTokens |
| 517 | results = append(results, r) |
| 518 | if r.Passed || (cfg.budget > 0 && total >= cfg.budget) { |
| 519 | break |
| 520 | } |
| 521 | } |
| 522 | } |
| 523 | return results |
| 524 | } |
| 525 | |
| 526 | // runTask copies the task's seed workdir into a temp dir, runs the agent there, |
| 527 | // then drops in verify.sh and runs it as the grader. The grader is added only |
| 528 | // after the run so the agent can't read the answer key. |
| 529 | func runTask(cfg suiteConfig, t task) result { |
| 530 | r := result{task: t, Profile: benchmarkProfileStandard, CacheArm: cfg.cacheArm, Effort: cfg.effort} |
| 531 | r.Arm = cfg.arm.Arm() |
| 532 | r.Anchor = cfg.anchor |
| 533 | t.Prompt = anchorPrompt(cfg.anchor, t) |
| 534 | if cfg.forcePlanner { |
| 535 | // Leading directive matched by the planner gate's |
| 536 | // planAndExecuteDirectives, so the two-model turn engages even for |
| 537 | // prompts the gate would route ExecutorOnly. |
| 538 | t.Prompt = "Plan first, then implement the following task.\n\n" + t.Prompt |
| 539 | r.PlanForced = true |
| 540 | } |
| 541 | |
| 542 | // The per-leg file names are decided in runSegments; only the directory has |
| 543 | // to exist before the first child starts, and the digest below reads the |
| 544 | // last leg's file. |
| 545 | trajPath := "" |
| 546 | if cfg.trajDir != "" { |
| 547 | if err := os.MkdirAll(cfg.trajDir, 0o755); err != nil { |
| 548 | r.Note = "trajectory dir: " + err.Error() |
| 549 | return r |
| 550 | } |
| 551 | } |
| 552 | |
| 553 | work, err := os.MkdirTemp("", "e2ebench-"+t.ID+"-") |
| 554 | if err != nil { |
| 555 | r.Note = "mktemp: " + err.Error() |
| 556 | return r |
| 557 | } |
| 558 | defer os.RemoveAll(work) |
| 559 | |
| 560 | if seed := filepath.Join(t.dir, "workdir"); dirExists(seed) { |
| 561 | if err := copyDir(seed, work); err != nil { |
| 562 | r.Note = "copy seed: " + err.Error() |
| 563 | return r |
| 564 | } |
| 565 | } |
| 566 | |
| 567 | if cfg.cacheArm == benchmarkCacheWarm { |
| 568 | warmPrefix(cfg, work) |
| 569 | } |
| 570 | |
| 571 | ctx, cancel := context.WithTimeout(context.Background(), time.Duration(t.TimeoutSec)*time.Second) |
| 572 | defer cancel() |
| 573 | |
| 574 | extraEnv, seedNote := taskExperimentEnv(cfg, t, work) |
| 575 | if seedNote != "" { |
| 576 | r.Note = seedNote |
| 577 | } |
| 578 | mtr := attachMeter(cfg, &r) |
| 579 | defer mtr.close() |
| 580 | extraEnv = append(extraEnv, mtr.env...) |
| 581 | startedAt := time.Now() |
| 582 | snap, dropSnapshots := attachSnapshotter(cfg, t, work, startedAt) |
| 583 | defer dropSnapshots() |
| 584 | runErr := runSegments(ctx, cfg, t, work, cfg.trajDir, extraEnv, &r) |
| 585 | r.WallMs = time.Since(startedAt).Milliseconds() |
| 586 | var taken []checkpoint |
| 587 | if snap != nil { |
| 588 | taken = snap.halt() |
| 589 | } |
| 590 | |
| 591 | mtr.record(&r) |
| 592 | if trajPath = lastSegmentTrajectory(cfg.trajDir, t.ID, r.Segments); trajPath != "" { |
| 593 | if summary, err := summarizeTrajectory(trajPath); err == nil { |
| 594 | r.Trajectory = summary |
| 595 | } |
| 596 | applyMemoryStats(&r, trajPath, t) |
| 597 | } |
| 598 | // A killed child never writes metrics, so the deadline is the only place |
| 599 | // this failure mode is still observable. |
| 600 | if ctx.Err() == context.DeadlineExceeded { |
| 601 | r.Outcome = "timeout" |
| 602 | } |
| 603 | if runErr != nil { |
| 604 | r.Note = "run: " + runErr.Error() |
| 605 | // still grade — a non-zero exit may just be a max-steps notice |
| 606 | } |
| 607 | |
| 608 | var graderSaid string |
| 609 | r.Passed, graderSaid = gradeVerbose(work, t.dir) |
| 610 | if !r.Passed && graderSaid != "" { |
| 611 | r.Note = appendNote(r.Note, "grader: "+utf8Prefix(graderSaid, graderNoteLimit)) |
| 612 | } |
| 613 | if snap != nil { |
| 614 | r.Checkpoints = gradeCheckpoints(taken, t.dir) |
| 615 | r.FirstCorrectMs, r.SolvedThenBroken = firstCorrect(r.Checkpoints, r.Passed) |
| 616 | if r.Passed && r.FirstCorrectMs > 0 { |
| 617 | r.PostSolveWasteMs = r.WallMs - r.FirstCorrectMs |
| 618 | } |
| 619 | r.MutationsBeforeCorrect = mutationsBeforeCorrect(r.Checkpoints) |
| 620 | r.RegressedAfterCorrect = regressedAfterCorrect(r.Checkpoints) |
| 621 | if trajPath != "" && r.FirstCorrectMs > 0 { |
| 622 | split := splitAtCorrect(trajPath, startedAt.UnixMilli()+r.FirstCorrectMs) |
| 623 | r.RoundsBeforeCorrect, r.RoundsAfterCorrect = split.RoundsBefore, split.RoundsAfter |
| 624 | r.CallsBeforeCorrect, r.CallsAfterCorrect = split.CallsBefore, split.CallsAfter |
| 625 | r.VerifyAfterCorrect = split.VerifyAfter |
| 626 | r.ReviewsAfterCorrect, r.MutationsAfterCorrect = split.ReviewsAfter, split.MutationsAfter |
| 627 | } |
| 628 | if trajPath != "" { |
| 629 | var endsElapsed []int64 |
| 630 | for _, end := range roundEnds(trajPath) { |
| 631 | endsElapsed = append(endsElapsed, end-startedAt.UnixMilli()) |
| 632 | } |
| 633 | r.StopEval = computeStopEval(r.Checkpoints, endsElapsed) |
| 634 | } |
| 635 | r.FirstUsefulMs = firstUsefulMutation(r.Checkpoints, filepath.Join(t.dir, "workdir"), work) |
| 636 | } |
| 637 | r.PhaseTrace = buildPhaseTrace(r) |
| 638 | return r |
| 639 | } |
| 640 | |
| 641 | func buildRunTaskArgs(cfg suiteConfig, metricsPath, trajectoryPath string, maxSteps int, prompt string) []string { |
| 642 | // Benchmarks are unattended and their fixtures require ordinary workspace |
| 643 | // writes. Auto still honors explicit ask/deny rules and the sandbox boundary. |
| 644 | args := []string{"run", "--auto", "--metrics", metricsPath} |
| 645 | if trajectoryPath != "" { |
| 646 | args = append(args, "--trajectory", trajectoryPath) |
| 647 | } |
| 648 | if cfg.model != "" { |
| 649 | args = append(args, "--model", cfg.model) |
| 650 | } |
| 651 | if maxSteps > 0 { |
| 652 | args = append(args, "--max-steps", fmt.Sprint(maxSteps)) |
| 653 | } |
| 654 | if cfg.effort != "" { |
| 655 | args = append(args, "--effort", cfg.effort) |
| 656 | } |
| 657 | // The control arm must produce a byte-identical command line to the one the |
| 658 | // suite ran before ablation existed, so its numbers stay comparable. |
| 659 | if !cfg.arm.Empty() { |
| 660 | args = append(args, "--ablate", cfg.arm.String()) |
| 661 | } |
| 662 | return append(args, prompt) |
| 663 | } |
| 664 | |
| 665 | // buildSegmentArgs is buildRunTaskArgs for one leg: a resumed leg adds |
| 666 | // --continue, which is unambiguous because each task runs in its own home and |
| 667 | // therefore its own session directory. |
| 668 | func buildSegmentArgs(cfg suiteConfig, seg segment, metricsPath, trajectoryPath string) []string { |
| 669 | args := buildRunTaskArgs(cfg, metricsPath, trajectoryPath, seg.maxSteps, seg.prompt) |
| 670 | if !seg.resume { |
| 671 | return args |
| 672 | } |
| 673 | // The prompt is the last argument; --continue must precede it. |
| 674 | return append(args[:len(args)-1:len(args)-1], "--continue", args[len(args)-1]) |
| 675 | } |
| 676 | |
| 677 | // warmPrefix primes the provider prefix cache for work's session shape with a |
| 678 | // minimal one-step run before the graded run starts its clock. Its cost is |
| 679 | // deliberately untracked: the warm arm measures a long-lived session's steady |
| 680 | // state, not the price of reaching it. Prefix-shaping flags (model, effort, |
| 681 | // ablation, cwd) must match the graded invocation exactly. |
| 682 | func warmPrefix(cfg suiteConfig, work string) { |
| 683 | ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) |
| 684 | defer cancel() |
| 685 | args := []string{"run", "--auto", "--max-steps", "1"} |
| 686 | if cfg.model != "" { |
| 687 | args = append(args, "--model", cfg.model) |
| 688 | } |
| 689 | if cfg.effort != "" { |
| 690 | args = append(args, "--effort", cfg.effort) |
| 691 | } |
| 692 | if !cfg.arm.Empty() { |
| 693 | args = append(args, "--ablate", cfg.arm.String()) |
| 694 | } |
| 695 | args = append(args, "Reply with exactly: ok") |
| 696 | cmd := exec.CommandContext(ctx, cfg.bin, args...) |
| 697 | cmd.Dir = work |
| 698 | cmd.Stdout = os.Stderr |
| 699 | cmd.Stderr = os.Stderr |
| 700 | if err := cmd.Run(); err != nil { |
| 701 | fmt.Fprintln(os.Stderr, "warm-cache pass:", err) |
| 702 | } |
| 703 | } |
| 704 | |
| 705 | func readMetrics(path string) (runMetrics, error) { |
| 706 | var m runMetrics |
| 707 | b, err := fileencoding.ReadFileUTF8(path) |
| 708 | if err != nil { |
| 709 | return m, err |
| 710 | } |
| 711 | return m, json.Unmarshal(b, &m) |
| 712 | } |
| 713 | |
| 714 | func dirExists(p string) bool { |
| 715 | fi, err := os.Stat(p) |
| 716 | return err == nil && fi.IsDir() |
| 717 | } |
| 718 | |
| 719 | func fileExists(p string) bool { |
| 720 | fi, err := os.Stat(p) |
| 721 | return err == nil && !fi.IsDir() |
| 722 | } |
| 723 | |
| 724 | func copyDir(src, dst string) error { |
| 725 | return filepath.Walk(src, func(p string, info os.FileInfo, err error) error { |
| 726 | if err != nil { |
| 727 | return err |
| 728 | } |
| 729 | // Skip symlinks so a seed link can't leak a file from outside the seed tree. |
| 730 | if info.Mode()&os.ModeSymlink != 0 { |
| 731 | return nil |
| 732 | } |
| 733 | rel, _ := filepath.Rel(src, p) |
| 734 | target := filepath.Join(dst, rel) |
| 735 | if info.IsDir() { |
| 736 | return os.MkdirAll(target, 0o755) |
| 737 | } |
| 738 | return copyFile(p, target) |
| 739 | }) |
| 740 | } |
| 741 | |
| 742 | func copyFile(src, dst string) error { |
| 743 | if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { |
| 744 | return err |
| 745 | } |
| 746 | in, err := os.Open(src) |
| 747 | if err != nil { |
| 748 | return err |
| 749 | } |
| 750 | defer in.Close() |
| 751 | info, err := in.Stat() |
| 752 | if err != nil { |
| 753 | return err |
| 754 | } |
| 755 | out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, info.Mode().Perm()) |
| 756 | if err != nil { |
| 757 | return err |
| 758 | } |
| 759 | defer out.Close() |
| 760 | if _, err := io.Copy(out, in); err != nil { |
| 761 | return err |
| 762 | } |
| 763 | // Mirror the source mode so a seed's read-only / exec bit survives the copy. |
| 764 | return os.Chmod(dst, info.Mode().Perm()) |
| 765 | } |
| 766 |