| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "fmt" |
| 7 | "os" |
| 8 | "os/exec" |
| 9 | "path/filepath" |
| 10 | "sort" |
| 11 | "strings" |
| 12 | "time" |
| 13 | |
| 14 | "reasonix/internal/agent" |
| 15 | ) |
| 16 | |
| 17 | // forkRow is one continuation run from a frozen eligibility state: the pair |
| 18 | // analysis unit. Post-fork metrics need no alignment — the whole run IS the |
| 19 | // post-trigger tail, so its totals are Remaining-Cost-From-Eligibility. |
| 20 | type forkRow struct { |
| 21 | Task string `json:"task"` |
| 22 | Class string `json:"class"` |
| 23 | Arm string `json:"arm"` |
| 24 | Rep int `json:"rep"` |
| 25 | Passed bool `json:"passed"` |
| 26 | WallMs int64 `json:"wall_ms"` |
| 27 | Rounds int `json:"rounds"` |
| 28 | ReasoningTok int64 `json:"reasoning_tok"` |
| 29 | ExtraBlind int `json:"extra_blind"` |
| 30 | FirstCheckRnds int `json:"first_check_rounds"` // 0 = never |
| 31 | BlindAtFork int `json:"blind_at_fork"` |
| 32 | DebtAtFork int `json:"debt_at_fork"` |
| 33 | EligibleRound int `json:"eligible_round"` |
| 34 | } |
| 35 | |
| 36 | // runForkMode replays every captured bundle under the requested arms. Arms |
| 37 | // alternate order per rep so provider drift over the session cannot |
| 38 | // systematically favor one arm. |
| 39 | func runForkMode(bundlesDir, suite, arms string, reps int, cfg suiteConfig, trajDir, outMD, outJSON string) error { |
| 40 | entries, err := os.ReadDir(bundlesDir) |
| 41 | if err != nil { |
| 42 | return err |
| 43 | } |
| 44 | tasks, err := loadTasks(suite) |
| 45 | if err != nil { |
| 46 | return err |
| 47 | } |
| 48 | byID := map[string]task{} |
| 49 | for _, t := range tasks { |
| 50 | byID[t.ID] = t |
| 51 | } |
| 52 | armList := strings.Split(arms, ",") |
| 53 | var rows []forkRow |
| 54 | for _, e := range entries { |
| 55 | if !e.IsDir() { |
| 56 | continue |
| 57 | } |
| 58 | bdir := filepath.Join(bundlesDir, e.Name()) |
| 59 | bundle, err := agent.LoadForkBundle(filepath.Join(bdir, "bundle.json")) |
| 60 | if err != nil { |
| 61 | fmt.Fprintf(os.Stderr, "skip %s: %v\n", e.Name(), err) |
| 62 | continue |
| 63 | } |
| 64 | t, ok := byID[e.Name()] |
| 65 | if !ok { |
| 66 | fmt.Fprintf(os.Stderr, "skip %s: no such task in suite\n", e.Name()) |
| 67 | continue |
| 68 | } |
| 69 | for rep := 1; rep <= reps; rep++ { |
| 70 | order := append([]string(nil), armList...) |
| 71 | if rep%2 == 0 { |
| 72 | sort.Sort(sort.Reverse(sort.StringSlice(order))) |
| 73 | } |
| 74 | for _, arm := range order { |
| 75 | row, err := runForkContinuation(cfg, t, bundle, bdir, arm, rep, trajDir) |
| 76 | if err != nil { |
| 77 | fmt.Fprintf(os.Stderr, "%s/%s rep %d: %v\n", e.Name(), arm, rep, err) |
| 78 | continue |
| 79 | } |
| 80 | rows = append(rows, row) |
| 81 | fmt.Fprintf(os.Stderr, "fork %s %s rep%d: pass=%v wall=%s rounds=%d check@%d\n", |
| 82 | row.Task, row.Arm, rep, row.Passed, dur(row.WallMs), row.Rounds, row.FirstCheckRnds) |
| 83 | } |
| 84 | } |
| 85 | } |
| 86 | report := renderForkReport(rows) |
| 87 | emit(report, outMD, "") |
| 88 | if outJSON != "" { |
| 89 | data, _ := json.MarshalIndent(rows, "", " ") |
| 90 | if err := os.WriteFile(outJSON, data, 0o644); err != nil { |
| 91 | return err |
| 92 | } |
| 93 | } |
| 94 | return nil |
| 95 | } |
| 96 | |
| 97 | func runForkContinuation(cfg suiteConfig, t task, b *agent.ForkBundle, bdir, arm string, rep int, trajDir string) (forkRow, error) { |
| 98 | row := forkRow{Task: t.ID, Class: t.Class, Arm: arm, Rep: rep, |
| 99 | BlindAtFork: b.BlindAtFork, DebtAtFork: b.DebtAtFork, EligibleRound: b.EligibleRound} |
| 100 | work, err := os.MkdirTemp("", "fork-"+t.ID+"-") |
| 101 | if err != nil { |
| 102 | return row, err |
| 103 | } |
| 104 | defer os.RemoveAll(work) |
| 105 | if err := copyDir(filepath.Join(bdir, "workspace"), work); err != nil { |
| 106 | return row, err |
| 107 | } |
| 108 | trajPath := "" |
| 109 | if trajDir != "" { |
| 110 | trajPath = filepath.Join(trajDir, fmt.Sprintf("%s.%s.r%d.trajectory.jsonl", t.ID, arm, rep)) |
| 111 | } |
| 112 | ctx, cancel := context.WithTimeout(context.Background(), time.Duration(t.TimeoutSec)*time.Second) |
| 113 | defer cancel() |
| 114 | // Governor arms: low-effort cuts the whole continuation's thinking via |
| 115 | // the provider knob; act-first injects the shaping line instead. Both use |
| 116 | // the same frozen state as control. |
| 117 | runCfg, forkEnvArm := cfg, arm |
| 118 | switch arm { |
| 119 | case "low-effort": |
| 120 | runCfg.effort = "low" |
| 121 | forkEnvArm = "control" |
| 122 | case "act-first": |
| 123 | forkEnvArm = "actfirst" |
| 124 | } |
| 125 | args := buildRunTaskArgs(runCfg, filepath.Join(work, ".run-metrics.json"), trajPath, t.MaxSteps, b.Input) |
| 126 | cmd := exec.CommandContext(ctx, cfg.bin, args...) |
| 127 | cmd.Dir = work |
| 128 | cmd.Env = append(os.Environ(), |
| 129 | "REASONIX_EXPERIMENT_FORK_BUNDLE="+filepath.Join(bdir, "bundle.json"), |
| 130 | "REASONIX_EXPERIMENT_FORK_ARM="+forkEnvArm) |
| 131 | cmd.Stdout = os.Stderr |
| 132 | cmd.Stderr = os.Stderr |
| 133 | cmd.WaitDelay = 10 * time.Second |
| 134 | started := time.Now() |
| 135 | _ = cmd.Run() |
| 136 | row.WallMs = time.Since(started).Milliseconds() |
| 137 | row.Passed = grade(work, t.dir) |
| 138 | if trajPath != "" { |
| 139 | if scan, err := scanTrajectoryFile(trajPath); err == nil { |
| 140 | s := scan.finish() |
| 141 | row.Rounds = len(scan.outcomePoints) |
| 142 | row.ReasoningTok = s.ReasoningTokensTotal |
| 143 | for i, p := range scan.outcomePoints { |
| 144 | if p.discriminating > 0 { |
| 145 | row.FirstCheckRnds = i + 1 |
| 146 | break |
| 147 | } |
| 148 | row.ExtraBlind += p.churn |
| 149 | } |
| 150 | } |
| 151 | } |
| 152 | return row, nil |
| 153 | } |
| 154 | |
| 155 | // renderForkReport is deliberately per-pair first, per-class second, and never |
| 156 | // one blended score: the arms answer different questions per class. |
| 157 | func renderForkReport(rows []forkRow) string { |
| 158 | var b strings.Builder |
| 159 | b.WriteString("## Fork continuation report\n\n") |
| 160 | b.WriteString("| task | class | arm | rep | pass | wall | rounds | reasoning | extra blind | check@ |\n") |
| 161 | b.WriteString("|---|---|---|---|---|---|---|---|---|---|\n") |
| 162 | for _, r := range rows { |
| 163 | check := "never" |
| 164 | if r.FirstCheckRnds > 0 { |
| 165 | check = fmt.Sprint(r.FirstCheckRnds) |
| 166 | } |
| 167 | fmt.Fprintf(&b, "| %s | %s | %s | %d | %v | %s | %d | %s | %d | %s |\n", |
| 168 | r.Task, r.Class, r.Arm, r.Rep, r.Passed, dur(r.WallMs), r.Rounds, |
| 169 | comma(int(r.ReasoningTok)), r.ExtraBlind, check) |
| 170 | } |
| 171 | type agg struct { |
| 172 | n, pass, checks, extraBlind int |
| 173 | wall, reasoning int64 |
| 174 | } |
| 175 | byClassArm := map[string]*agg{} |
| 176 | for _, r := range rows { |
| 177 | key := r.Class + "/" + r.Arm |
| 178 | a := byClassArm[key] |
| 179 | if a == nil { |
| 180 | a = &agg{} |
| 181 | byClassArm[key] = a |
| 182 | } |
| 183 | a.n++ |
| 184 | if r.Passed { |
| 185 | a.pass++ |
| 186 | } |
| 187 | if r.FirstCheckRnds > 0 { |
| 188 | a.checks++ |
| 189 | } |
| 190 | a.extraBlind += r.ExtraBlind |
| 191 | a.wall += r.WallMs |
| 192 | a.reasoning += r.ReasoningTok |
| 193 | } |
| 194 | keys := make([]string, 0, len(byClassArm)) |
| 195 | for k := range byClassArm { |
| 196 | keys = append(keys, k) |
| 197 | } |
| 198 | sort.Strings(keys) |
| 199 | b.WriteString("\n| class/arm | n | pass | checked | avg wall | avg reasoning | avg extra blind |\n") |
| 200 | b.WriteString("|---|---|---|---|---|---|---|\n") |
| 201 | for _, k := range keys { |
| 202 | a := byClassArm[k] |
| 203 | fmt.Fprintf(&b, "| %s | %d | %s | %s | %s | %s | %.1f |\n", |
| 204 | k, a.n, pct(a.pass, a.n), pct(a.checks, a.n), |
| 205 | dur(a.wall/int64(a.n)), comma(int(a.reasoning/int64(a.n))), float64(a.extraBlind)/float64(a.n)) |
| 206 | } |
| 207 | return b.String() |
| 208 | } |
| 209 |