| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "fmt" |
| 5 | "hash/fnv" |
| 6 | "io/fs" |
| 7 | "os" |
| 8 | "path/filepath" |
| 9 | "strings" |
| 10 | "time" |
| 11 | ) |
| 12 | |
| 13 | // checkpoint is one workspace snapshot taken during a run, graded offline by |
| 14 | // the hidden grader after the run ends. Pass answers the question no final |
| 15 | // grade can: from which moment was the workspace already a correct answer. |
| 16 | type checkpoint struct { |
| 17 | Seq int `json:"seq"` |
| 18 | ElapsedMs int64 `json:"elapsed_ms"` |
| 19 | Pass bool `json:"pass"` |
| 20 | |
| 21 | dir string |
| 22 | } |
| 23 | |
| 24 | // snapshotter polls a running task's workdir and copies it on every observed |
| 25 | // content change. Torn copies of files mid-write are acceptable: they grade |
| 26 | // as failures, which is the truthful state of that instant. |
| 27 | type snapshotter struct { |
| 28 | src, dst string |
| 29 | start time.Time |
| 30 | stop chan struct{} |
| 31 | done chan struct{} |
| 32 | poll <-chan time.Time |
| 33 | pollAck chan<- struct{} |
| 34 | taken []checkpoint |
| 35 | lastSig uint64 |
| 36 | } |
| 37 | |
| 38 | const snapshotPollInterval = 300 * time.Millisecond |
| 39 | |
| 40 | func startSnapshotter(src, dst string, start time.Time) *snapshotter { |
| 41 | return startSnapshotterWithPoll(src, dst, start, nil, nil) |
| 42 | } |
| 43 | |
| 44 | // startSnapshotterWithPoll injects a deterministic poll stream for tests. The |
| 45 | // production path uses snapshotPollInterval; tests can acknowledge each poll |
| 46 | // after the snapshotter has finished processing it without sleeping. |
| 47 | func startSnapshotterWithPoll(src, dst string, start time.Time, poll <-chan time.Time, pollAck chan<- struct{}) *snapshotter { |
| 48 | s := &snapshotter{src: src, dst: dst, start: start, stop: make(chan struct{}), done: make(chan struct{}), poll: poll, pollAck: pollAck} |
| 49 | s.lastSig = dirSignature(src) |
| 50 | go s.run() |
| 51 | return s |
| 52 | } |
| 53 | |
| 54 | func (s *snapshotter) run() { |
| 55 | defer close(s.done) |
| 56 | poll := s.poll |
| 57 | var ticker *time.Ticker |
| 58 | if poll == nil { |
| 59 | ticker = time.NewTicker(snapshotPollInterval) |
| 60 | poll = ticker.C |
| 61 | defer ticker.Stop() |
| 62 | } |
| 63 | for { |
| 64 | select { |
| 65 | case <-s.stop: |
| 66 | return |
| 67 | case <-poll: |
| 68 | s.snapshotIfChanged() |
| 69 | if s.pollAck != nil { |
| 70 | s.pollAck <- struct{}{} |
| 71 | } |
| 72 | } |
| 73 | } |
| 74 | } |
| 75 | |
| 76 | func (s *snapshotter) snapshotIfChanged() { |
| 77 | sig := dirSignature(s.src) |
| 78 | if sig == s.lastSig { |
| 79 | return |
| 80 | } |
| 81 | s.lastSig = sig |
| 82 | elapsed := time.Since(s.start).Milliseconds() |
| 83 | dir := filepath.Join(s.dst, fmt.Sprintf("%03d-%dms", len(s.taken)+1, elapsed)) |
| 84 | if err := copyDir(s.src, dir); err != nil { |
| 85 | return |
| 86 | } |
| 87 | dropHarnessArtifacts(dir) |
| 88 | s.taken = append(s.taken, checkpoint{Seq: len(s.taken) + 1, ElapsedMs: elapsed, dir: dir}) |
| 89 | } |
| 90 | |
| 91 | // halt stops polling, takes one final snapshot if the tail changed after the |
| 92 | // last tick, and returns everything captured. |
| 93 | func (s *snapshotter) halt() []checkpoint { |
| 94 | close(s.stop) |
| 95 | <-s.done |
| 96 | s.snapshotIfChanged() |
| 97 | return s.taken |
| 98 | } |
| 99 | |
| 100 | // dirSignature hashes the workdir's shape (path, size, mtime). The agent's |
| 101 | // metrics sidecar updates every turn without touching the workspace, so it |
| 102 | // is excluded — as are bytecode caches, which no grader reads. |
| 103 | func dirSignature(root string) uint64 { |
| 104 | h := fnv.New64a() |
| 105 | _ = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { |
| 106 | if err != nil { |
| 107 | return nil |
| 108 | } |
| 109 | name := d.Name() |
| 110 | if d.IsDir() { |
| 111 | if name == "__pycache__" { |
| 112 | return filepath.SkipDir |
| 113 | } |
| 114 | return nil |
| 115 | } |
| 116 | if isHarnessArtifact(name) { |
| 117 | return nil |
| 118 | } |
| 119 | info, err := d.Info() |
| 120 | if err != nil { |
| 121 | return nil |
| 122 | } |
| 123 | rel, _ := filepath.Rel(root, path) |
| 124 | fmt.Fprintf(h, "%s|%d|%d\n", rel, info.Size(), info.ModTime().UnixNano()) |
| 125 | return nil |
| 126 | }) |
| 127 | return h.Sum64() |
| 128 | } |
| 129 | |
| 130 | // gradeCheckpoints runs the hidden grader over every snapshot, oldest first, |
| 131 | // after the agent is gone — the agent never sees a verdict. |
| 132 | func gradeCheckpoints(checkpoints []checkpoint, taskDir string) []checkpoint { |
| 133 | for i := range checkpoints { |
| 134 | checkpoints[i].Pass = grade(checkpoints[i].dir, taskDir) |
| 135 | } |
| 136 | return checkpoints |
| 137 | } |
| 138 | |
| 139 | // firstCorrect returns the elapsed ms of the earliest passing snapshot (0 if |
| 140 | // none) and whether a passing state was later broken (a snapshot passed but |
| 141 | // the final workspace failed). |
| 142 | func firstCorrect(checkpoints []checkpoint, finalPassed bool) (firstMs int64, solvedThenBroken bool) { |
| 143 | anyPassed := false |
| 144 | for _, cp := range checkpoints { |
| 145 | if cp.Pass { |
| 146 | anyPassed = true |
| 147 | if firstMs == 0 { |
| 148 | firstMs = cp.ElapsedMs |
| 149 | } |
| 150 | } |
| 151 | } |
| 152 | return firstMs, anyPassed && !finalPassed |
| 153 | } |
| 154 | |
| 155 | // solveProfile buckets a checkpointed run into the "why slow" cases that |
| 156 | // demand opposite optimizations: early_correct = termination/verification |
| 157 | // overhead; late_correct = exploration/decision efficiency; never_correct = |
| 158 | // capability (chasing latency is pointless); solved_then_broke = a passing |
| 159 | // state destroyed afterwards. Empty when the run took no checkpoints. |
| 160 | func solveProfile(r result) string { |
| 161 | if len(r.Checkpoints) == 0 { |
| 162 | return "" |
| 163 | } |
| 164 | switch { |
| 165 | case r.SolvedThenBroken: |
| 166 | return "solved_then_broke" |
| 167 | case !r.Passed: |
| 168 | return "never_correct" |
| 169 | case r.FirstCorrectMs == 0: |
| 170 | return "late_correct" // only the final state passed |
| 171 | case r.PostSolveWasteMs >= 5000 && r.PostSolveWasteMs*3 >= r.WallMs: |
| 172 | return "early_correct" |
| 173 | default: |
| 174 | return "late_correct" |
| 175 | } |
| 176 | } |
| 177 | |
| 178 | // renderSolveProfiles is the triage line: how many runs fall into each case, |
| 179 | // with the early-correct bucket's median waste — the directly recoverable tail. |
| 180 | func renderSolveProfiles(results []result) string { |
| 181 | counts := map[string]int{} |
| 182 | var earlyWaste []int64 |
| 183 | for _, r := range results { |
| 184 | profile := solveProfile(r) |
| 185 | if profile == "" { |
| 186 | continue |
| 187 | } |
| 188 | counts[profile]++ |
| 189 | if profile == "early_correct" { |
| 190 | earlyWaste = append(earlyWaste, r.PostSolveWasteMs) |
| 191 | } |
| 192 | } |
| 193 | if len(counts) == 0 { |
| 194 | return "" |
| 195 | } |
| 196 | line := "**Solve profile**: " |
| 197 | parts := []string{} |
| 198 | regressed, withCorrect := 0, 0 |
| 199 | var roundsBefore, roundsAfter, verifyAfter, mutationsBefore []int64 |
| 200 | for _, r := range results { |
| 201 | if len(r.Checkpoints) == 0 { |
| 202 | continue |
| 203 | } |
| 204 | if r.FirstCorrectMs > 0 { |
| 205 | withCorrect++ |
| 206 | roundsBefore = append(roundsBefore, int64(r.RoundsBeforeCorrect)) |
| 207 | roundsAfter = append(roundsAfter, int64(r.RoundsAfterCorrect)) |
| 208 | verifyAfter = append(verifyAfter, int64(r.VerifyAfterCorrect)) |
| 209 | mutationsBefore = append(mutationsBefore, int64(r.MutationsBeforeCorrect)) |
| 210 | } |
| 211 | if r.RegressedAfterCorrect { |
| 212 | regressed++ |
| 213 | } |
| 214 | } |
| 215 | for _, profile := range []string{"early_correct", "late_correct", "never_correct", "solved_then_broke"} { |
| 216 | if counts[profile] == 0 { |
| 217 | continue |
| 218 | } |
| 219 | part := fmt.Sprintf("**%s** %d", profile, counts[profile]) |
| 220 | if profile == "early_correct" { |
| 221 | part += fmt.Sprintf(" (median waste %s)", dur(median(earlyWaste))) |
| 222 | } |
| 223 | parts = append(parts, part) |
| 224 | } |
| 225 | line += joinParts(parts) |
| 226 | stoppable, harmful, past := 0, 0, []int64{} |
| 227 | for _, r := range results { |
| 228 | if r.StopEval == nil { |
| 229 | continue |
| 230 | } |
| 231 | if r.StopEval.FirstStoppableRound > 0 { |
| 232 | stoppable++ |
| 233 | past = append(past, int64(r.StopEval.ContinuationsPast)) |
| 234 | } |
| 235 | harmful += r.StopEval.HarmfulContinuation |
| 236 | } |
| 237 | if stoppable > 0 { |
| 238 | line += fmt.Sprintf("\n\n**Stop policy**: stoppable before final in %d runs · **continuations past stoppable** median %d rounds · **harmful continuations** %d", stoppable, median(past), harmful) |
| 239 | } |
| 240 | if diagnosis := renderDiagnosis(results); diagnosis != "" { |
| 241 | line += "\n\n" + strings.TrimSuffix(diagnosis, "\n\n") |
| 242 | } |
| 243 | if withCorrect > 0 { |
| 244 | var ttfum []int64 |
| 245 | for _, r := range results { |
| 246 | if r.FirstUsefulMs > 0 { |
| 247 | ttfum = append(ttfum, r.FirstUsefulMs) |
| 248 | } |
| 249 | } |
| 250 | line += fmt.Sprintf("\n\n**Correct boundary** (medians): **mutations before** %d · **rounds before** %d · **rounds after** %d · **verifications after** %d · **regression-after-correct** %s", |
| 251 | median(mutationsBefore), median(roundsBefore), median(roundsAfter), median(verifyAfter), |
| 252 | pct(regressed, withCorrect)) |
| 253 | if len(ttfum) > 0 { |
| 254 | line += fmt.Sprintf(" · **TTFUM median** %s", dur(median(ttfum))) |
| 255 | } |
| 256 | } |
| 257 | return line + "\n\n" |
| 258 | } |
| 259 | |
| 260 | func joinParts(parts []string) string { |
| 261 | return strings.Join(parts, " · ") |
| 262 | } |
| 263 | |
| 264 | // mutationsBeforeCorrect counts the workspace states tried before the first |
| 265 | // passing one — how many edits it took to find the correct patch. |
| 266 | func mutationsBeforeCorrect(checkpoints []checkpoint) int { |
| 267 | for i, cp := range checkpoints { |
| 268 | if cp.Pass { |
| 269 | return i |
| 270 | } |
| 271 | } |
| 272 | return len(checkpoints) |
| 273 | } |
| 274 | |
| 275 | // regressedAfterCorrect reports a PASS followed by a later FAIL — the agent |
| 276 | // kept "improving" a correct answer and broke it, even if it repaired the |
| 277 | // damage before finishing. |
| 278 | func regressedAfterCorrect(checkpoints []checkpoint) bool { |
| 279 | seenPass := false |
| 280 | for _, cp := range checkpoints { |
| 281 | if cp.Pass { |
| 282 | seenPass = true |
| 283 | } else if seenPass { |
| 284 | return true |
| 285 | } |
| 286 | } |
| 287 | return false |
| 288 | } |
| 289 | |
| 290 | // stopEval is the counterfactual-stop readout: had the run stopped at the |
| 291 | // end of round N, would the grader have passed? Derived by aligning round |
| 292 | // boundaries with the checkpoint grid — no agent participation. |
| 293 | type stopEval struct { |
| 294 | Curve []bool `json:"curve"` // round-end verdicts, 1-based rounds |
| 295 | FirstStoppableRound int `json:"first_stoppable_round,omitempty"` // 0 = never |
| 296 | ContinuationsPast int `json:"continuations_past,omitempty"` // rounds run after first stoppable |
| 297 | HarmfulContinuation int `json:"harmful_continuations,omitempty"` // PASS→FAIL round transitions |
| 298 | } |
| 299 | |
| 300 | // computeStopEval grades each round's end state as the last checkpoint at or |
| 301 | // before that boundary; a boundary before any snapshot is the seed (fail). |
| 302 | func computeStopEval(checkpoints []checkpoint, roundEndElapsedMs []int64) *stopEval { |
| 303 | if len(checkpoints) == 0 || len(roundEndElapsedMs) == 0 { |
| 304 | return nil |
| 305 | } |
| 306 | eval := &stopEval{} |
| 307 | prev := false |
| 308 | for i, end := range roundEndElapsedMs { |
| 309 | pass := false |
| 310 | for _, cp := range checkpoints { |
| 311 | if cp.ElapsedMs <= end { |
| 312 | pass = cp.Pass |
| 313 | } else { |
| 314 | break |
| 315 | } |
| 316 | } |
| 317 | eval.Curve = append(eval.Curve, pass) |
| 318 | if pass && eval.FirstStoppableRound == 0 { |
| 319 | eval.FirstStoppableRound = i + 1 |
| 320 | } |
| 321 | if prev && !pass { |
| 322 | eval.HarmfulContinuation++ |
| 323 | } |
| 324 | prev = pass |
| 325 | } |
| 326 | if eval.FirstStoppableRound > 0 { |
| 327 | eval.ContinuationsPast = len(eval.Curve) - eval.FirstStoppableRound |
| 328 | } |
| 329 | return eval |
| 330 | } |
| 331 | |
| 332 | // firstUsefulMutation approximates TTFUM — when part of the final solution |
| 333 | // first appeared: the earliest checkpoint in which any file that differs |
| 334 | // between seed and final already carries its exact final content. Cosmetic |
| 335 | // late edits make this an overestimate; that bias is stated, not hidden. |
| 336 | func firstUsefulMutation(checkpoints []checkpoint, seedDir, finalDir string) int64 { |
| 337 | solution := solutionFiles(seedDir, finalDir) |
| 338 | if len(solution) == 0 { |
| 339 | return 0 |
| 340 | } |
| 341 | for _, cp := range checkpoints { |
| 342 | for rel, want := range solution { |
| 343 | got, err := os.ReadFile(filepath.Join(cp.dir, rel)) |
| 344 | if err == nil && string(got) == want { |
| 345 | return cp.ElapsedMs |
| 346 | } |
| 347 | } |
| 348 | } |
| 349 | return 0 |
| 350 | } |
| 351 | |
| 352 | // solutionFiles maps relative path → final content for every file the run |
| 353 | // created or changed; harness artifacts are not part of anyone's solution. |
| 354 | func solutionFiles(seedDir, finalDir string) map[string]string { |
| 355 | skip := map[string]bool{"verify.sh": true} |
| 356 | out := map[string]string{} |
| 357 | _ = filepath.WalkDir(finalDir, func(path string, d fs.DirEntry, err error) error { |
| 358 | if err != nil { |
| 359 | return nil |
| 360 | } |
| 361 | if d.IsDir() { |
| 362 | if d.Name() == "__pycache__" { |
| 363 | return filepath.SkipDir |
| 364 | } |
| 365 | return nil |
| 366 | } |
| 367 | rel, _ := filepath.Rel(finalDir, path) |
| 368 | if base := filepath.Base(rel); skip[base] || isHarnessArtifact(base) { |
| 369 | return nil |
| 370 | } |
| 371 | final, err := os.ReadFile(path) |
| 372 | if err != nil { |
| 373 | return nil |
| 374 | } |
| 375 | seed, err := os.ReadFile(filepath.Join(seedDir, rel)) |
| 376 | if err == nil && string(seed) == string(final) { |
| 377 | return nil // unchanged from seed: not part of the solution |
| 378 | } |
| 379 | out[rel] = string(final) |
| 380 | return nil |
| 381 | }) |
| 382 | return out |
| 383 | } |
| 384 | |
| 385 | // attachSnapshotter starts per-change workdir snapshots when checkpoint grading |
| 386 | // is on. It always returns a usable cleanup so the caller needs no branch; a |
| 387 | // temp-dir failure degrades to no snapshots rather than failing the run. |
| 388 | func attachSnapshotter(cfg suiteConfig, t task, work string, startedAt time.Time) (*snapshotter, func()) { |
| 389 | if !cfg.checkpoints { |
| 390 | return nil, func() {} |
| 391 | } |
| 392 | dir, err := os.MkdirTemp("", "e2ebench-cp-"+t.ID+"-") |
| 393 | if err != nil { |
| 394 | return nil, func() {} |
| 395 | } |
| 396 | return startSnapshotter(work, dir, startedAt), func() { _ = os.RemoveAll(dir) } |
| 397 | } |
| 398 | |
| 399 | // isHarnessArtifact reports whether a work-dir entry belongs to the benchmark |
| 400 | // rather than to anyone's solution. Segmented runs write one metrics file per |
| 401 | // leg, so the name is a prefix match — a hard-coded ".run-metrics.json" would |
| 402 | // let every later leg's file read as a change the agent made. |
| 403 | func isHarnessArtifact(name string) bool { |
| 404 | return strings.HasPrefix(name, ".run-metrics") && strings.HasSuffix(name, ".json") |
| 405 | } |
| 406 | |
| 407 | // dropHarnessArtifacts removes them from a snapshot copy. |
| 408 | func dropHarnessArtifacts(dir string) { |
| 409 | entries, err := os.ReadDir(dir) |
| 410 | if err != nil { |
| 411 | return |
| 412 | } |
| 413 | for _, e := range entries { |
| 414 | if !e.IsDir() && isHarnessArtifact(e.Name()) { |
| 415 | _ = os.Remove(filepath.Join(dir, e.Name())) |
| 416 | } |
| 417 | } |
| 418 | } |
| 419 |