| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "bufio" |
| 5 | "encoding/json" |
| 6 | "os" |
| 7 | ) |
| 8 | |
| 9 | // boundarySplit is everything countable on each side of the first-correct |
| 10 | // instant — the numbers that end the exploration-vs-termination argument. |
| 11 | type boundarySplit struct { |
| 12 | RoundsBefore, RoundsAfter int |
| 13 | CallsBefore, CallsAfter int |
| 14 | VerifyAfter int |
| 15 | ReviewsAfter, MutationsAfter int |
| 16 | } |
| 17 | |
| 18 | // splitAtCorrect scans a trajectory once and tallies rounds, tool calls, |
| 19 | // verifications, reviews and mutations relative to the cutoff instant. |
| 20 | func splitAtCorrect(path string, cutoffUnixMs int64) boundarySplit { |
| 21 | var out boundarySplit |
| 22 | f, err := os.Open(path) |
| 23 | if err != nil { |
| 24 | return out |
| 25 | } |
| 26 | defer f.Close() |
| 27 | inModel := true |
| 28 | sc := bufio.NewScanner(f) |
| 29 | sc.Buffer(make([]byte, 0, 1<<20), 16<<20) |
| 30 | for sc.Scan() { |
| 31 | var rec trajectoryRecord |
| 32 | if err := json.Unmarshal(sc.Bytes(), &rec); err != nil { |
| 33 | continue |
| 34 | } |
| 35 | if rec.Event == nil || rec.Event.Tool == nil || rec.Event.Tool.ParentID != "" { |
| 36 | continue |
| 37 | } |
| 38 | after := rec.TS > cutoffUnixMs |
| 39 | switch rec.Event.Kind { |
| 40 | case "tool_dispatch": |
| 41 | if inModel { |
| 42 | inModel = false |
| 43 | if after { |
| 44 | out.RoundsAfter++ |
| 45 | } else { |
| 46 | out.RoundsBefore++ |
| 47 | } |
| 48 | } |
| 49 | case "tool_result": |
| 50 | inModel = true |
| 51 | tl := rec.Event.Tool |
| 52 | if after { |
| 53 | out.CallsAfter++ |
| 54 | } else { |
| 55 | out.CallsBefore++ |
| 56 | } |
| 57 | if v := tl.Execution; v != nil && after && (v.Verification == "passed" || v.Verification == "failed") { |
| 58 | out.VerifyAfter++ |
| 59 | } |
| 60 | if after && tl.Name == "review_report" { |
| 61 | out.ReviewsAfter++ |
| 62 | } |
| 63 | if after && !tl.ReadOnly && !bookkeepingTools[tl.Name] && tl.Name != "review_report" { |
| 64 | out.MutationsAfter++ |
| 65 | } |
| 66 | } |
| 67 | } |
| 68 | return out |
| 69 | } |
| 70 | |
| 71 | // roundEnds returns the unix-ms end of each top-level tool round: the last |
| 72 | // tool_result before the next round's dispatch. The final answer segment has |
| 73 | // no entry — its end state is the run's final grade. |
| 74 | func roundEnds(path string) []int64 { |
| 75 | f, err := os.Open(path) |
| 76 | if err != nil { |
| 77 | return nil |
| 78 | } |
| 79 | defer f.Close() |
| 80 | var ends []int64 |
| 81 | var lastResult int64 |
| 82 | inModel := true |
| 83 | sc := bufio.NewScanner(f) |
| 84 | sc.Buffer(make([]byte, 0, 1<<20), 16<<20) |
| 85 | for sc.Scan() { |
| 86 | var rec trajectoryRecord |
| 87 | if err := json.Unmarshal(sc.Bytes(), &rec); err != nil { |
| 88 | continue |
| 89 | } |
| 90 | if rec.Event == nil || rec.Event.Tool == nil || rec.Event.Tool.ParentID != "" { |
| 91 | continue |
| 92 | } |
| 93 | switch rec.Event.Kind { |
| 94 | case "tool_dispatch": |
| 95 | if inModel && lastResult > 0 { |
| 96 | ends = append(ends, lastResult) |
| 97 | } |
| 98 | inModel = false |
| 99 | case "tool_result": |
| 100 | inModel = true |
| 101 | lastResult = rec.TS |
| 102 | } |
| 103 | } |
| 104 | if lastResult > 0 { |
| 105 | ends = append(ends, lastResult) |
| 106 | } |
| 107 | return ends |
| 108 | } |
| 109 |