| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "encoding/json" |
| 5 | "fmt" |
| 6 | "strings" |
| 7 | |
| 8 | "reasonix/internal/ablation" |
| 9 | ) |
| 10 | |
| 11 | // SWE-bench mode runs the agent inside the official per-instance evaluation |
| 12 | // container, so it can execute the repo's tests exactly like the harnesses it |
| 13 | // is compared against, then hands the resulting patch to the official grader. |
| 14 | |
| 15 | type swebenchInstance struct { |
| 16 | InstanceID string `json:"instance_id"` |
| 17 | Repo string `json:"repo"` |
| 18 | BaseCommit string `json:"base_commit"` |
| 19 | Problem string `json:"problem_statement"` |
| 20 | Difficulty string `json:"difficulty"` |
| 21 | } |
| 22 | |
| 23 | // swebenchImage builds the registry name of an instance's evaluation image. |
| 24 | // The harness mangles "__" to "_1776_" only when a namespace is set, so the |
| 25 | // local image key it prints is not the name you can pull. |
| 26 | func swebenchImage(namespace, instanceID string) string { |
| 27 | return fmt.Sprintf("%s/sweb.eval.x86_64.%s:latest", |
| 28 | namespace, strings.ReplaceAll(instanceID, "__", "_1776_")) |
| 29 | } |
| 30 | |
| 31 | // swebenchContainer is the throwaway container name for one attempt. It is |
| 32 | // distinct from the grader's own container so a stale agent container can never |
| 33 | // be mistaken for an evaluation in progress. |
| 34 | func swebenchContainer(instanceID string) string { |
| 35 | return "rxagent." + strings.ReplaceAll(instanceID, "__", ".") |
| 36 | } |
| 37 | |
| 38 | // testbedShell wraps a command so it runs against the instance's conda |
| 39 | // environment. The images ship miniconda with the repo's dependencies in an env |
| 40 | // named "testbed"; a bare `docker exec` misses it and every import fails. |
| 41 | func testbedShell(command string) []string { |
| 42 | return []string{"bash", "-lc", |
| 43 | "source /opt/miniconda3/bin/activate && conda activate testbed && cd /testbed && " + command} |
| 44 | } |
| 45 | |
| 46 | // permissionFlag maps a benchmark permission posture onto the CLI flag. auto is |
| 47 | // the unattended default. The alternative posture exists because comparable |
| 48 | // harnesses run without the dynamic-shell gate, so measuring against them under |
| 49 | // the gate measures our permission policy rather than the agent. |
| 50 | func permissionFlag(mode string) (string, error) { |
| 51 | switch mode { |
| 52 | case "", "auto": |
| 53 | return "--permission-mode=auto", nil |
| 54 | case "yolo": |
| 55 | return "--permission-mode=bypassPermissions", nil |
| 56 | default: |
| 57 | return "", fmt.Errorf("unknown permission mode %q (want auto or yolo)", mode) |
| 58 | } |
| 59 | } |
| 60 | |
| 61 | func swebenchAgentArgs(metricsPath, model, profile, permission string, arm ablation.Set, maxSteps int, prompt string) []string { |
| 62 | posture, err := permissionFlag(permission) |
| 63 | if err != nil { |
| 64 | panic(err) // validated at flag-parse time; reaching here is a wiring bug |
| 65 | } |
| 66 | args := []string{"run", posture, "--metrics", metricsPath} |
| 67 | if model != "" { |
| 68 | args = append(args, "--model", model) |
| 69 | } |
| 70 | if maxSteps > 0 { |
| 71 | args = append(args, "--max-steps", fmt.Sprint(maxSteps)) |
| 72 | } |
| 73 | args = appendBenchmarkProfileArgs(args, profile) |
| 74 | if !arm.Empty() { |
| 75 | args = append(args, "--ablate", arm.String()) |
| 76 | } |
| 77 | return append(args, prompt) |
| 78 | } |
| 79 | |
| 80 | // swebenchPrompt is the task text the agent sees. It carries the issue and the |
| 81 | // working rules, and deliberately withholds the test patch and the |
| 82 | // FAIL_TO_PASS list — those are the answer key. |
| 83 | func swebenchPrompt(inst swebenchInstance) string { |
| 84 | var b strings.Builder |
| 85 | b.WriteString("Resolve the following issue in the repository at /testbed.\n\n") |
| 86 | b.WriteString("<issue>\n") |
| 87 | b.WriteString(strings.TrimSpace(inst.Problem)) |
| 88 | b.WriteString("\n</issue>\n\n") |
| 89 | b.WriteString("The repository is a git checkout at the commit where the issue reproduces. ") |
| 90 | b.WriteString("Edit the source to fix it, and run the project's own tests to check your work. ") |
| 91 | b.WriteString("Do not commit, and do not modify any test file — the fix is graded by tests you cannot see.\n") |
| 92 | return b.String() |
| 93 | } |
| 94 | |
| 95 | // swebenchPrediction is one line of the predictions file the official grader |
| 96 | // reads. Field names are the harness's, not ours. |
| 97 | type swebenchPrediction struct { |
| 98 | InstanceID string `json:"instance_id"` |
| 99 | Model string `json:"model_name_or_path"` |
| 100 | Patch string `json:"model_patch"` |
| 101 | } |
| 102 | |
| 103 | func encodePredictions(model string, patches map[string]string, order []string) (string, error) { |
| 104 | var b strings.Builder |
| 105 | for _, id := range order { |
| 106 | patch, ok := patches[id] |
| 107 | if !ok { |
| 108 | continue |
| 109 | } |
| 110 | line, err := json.Marshal(swebenchPrediction{InstanceID: id, Model: model, Patch: patch}) |
| 111 | if err != nil { |
| 112 | return "", err |
| 113 | } |
| 114 | b.Write(line) |
| 115 | b.WriteByte('\n') |
| 116 | } |
| 117 | return b.String(), nil |
| 118 | } |
| 119 | |
| 120 | // swebenchReport is the subset of the grader's JSON summary we consume. Unknown |
| 121 | // fields are ignored so a harness upgrade that adds counters does not break us. |
| 122 | type swebenchReport struct { |
| 123 | ResolvedIDs []string `json:"resolved_ids"` |
| 124 | UnresolvedIDs []string `json:"unresolved_ids"` |
| 125 | ErrorIDs []string `json:"error_ids"` |
| 126 | EmptyPatchIDs []string `json:"empty_patch_ids"` |
| 127 | IncompleteIDs []string `json:"incomplete_ids"` |
| 128 | } |
| 129 | |
| 130 | // swebenchReportPath is where run_evaluation writes its summary: the model name |
| 131 | // from the predictions file joined with the run id, in the working directory. |
| 132 | func swebenchReportPath(model, runID string) string { |
| 133 | return model + "." + runID + ".json" |
| 134 | } |
| 135 | |
| 136 | // gradedClass maps one instance's grader outcome onto our published failure |
| 137 | // taxonomy. An id the grader never mentions is reported as unknown rather than |
| 138 | // silently counted as unsolved. |
| 139 | func (r swebenchReport) gradedClass(instanceID string) string { |
| 140 | for _, id := range r.ResolvedIDs { |
| 141 | if id == instanceID { |
| 142 | return "solved" |
| 143 | } |
| 144 | } |
| 145 | for _, id := range r.EmptyPatchIDs { |
| 146 | if id == instanceID { |
| 147 | return "no_patch" |
| 148 | } |
| 149 | } |
| 150 | for _, id := range r.ErrorIDs { |
| 151 | if id == instanceID { |
| 152 | return "grader_error" |
| 153 | } |
| 154 | } |
| 155 | for _, id := range r.IncompleteIDs { |
| 156 | if id == instanceID { |
| 157 | return "eval_timeout" |
| 158 | } |
| 159 | } |
| 160 | for _, id := range r.UnresolvedIDs { |
| 161 | if id == instanceID { |
| 162 | return "wrong_patch" |
| 163 | } |
| 164 | } |
| 165 | return "ungraded" |
| 166 | } |
| 167 |