| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "encoding/json" |
| 5 | "fmt" |
| 6 | "slices" |
| 7 | "strings" |
| 8 | |
| 9 | "reasonix/internal/ablation" |
| 10 | ) |
| 11 | |
| 12 | // SWE-bench mode runs the agent inside the official per-instance evaluation |
| 13 | // container, so it can execute the repo's tests exactly like the harnesses it |
| 14 | // is compared against, then hands the resulting patch to the official grader. |
| 15 | |
| 16 | type swebenchInstance struct { |
| 17 | InstanceID string `json:"instance_id"` |
| 18 | Repo string `json:"repo"` |
| 19 | BaseCommit string `json:"base_commit"` |
| 20 | Problem string `json:"problem_statement"` |
| 21 | Difficulty string `json:"difficulty"` |
| 22 | } |
| 23 | |
| 24 | // swebenchImage builds the registry name of an instance's evaluation image. |
| 25 | // The harness mangles "__" to "_1776_" only when a namespace is set, so the |
| 26 | // local image key it prints is not the name you can pull. |
| 27 | func swebenchImage(namespace, instanceID string) string { |
| 28 | return fmt.Sprintf("%s/sweb.eval.x86_64.%s:latest", |
| 29 | namespace, strings.ReplaceAll(instanceID, "__", "_1776_")) |
| 30 | } |
| 31 | |
| 32 | // swebenchContainer is the throwaway container name for one attempt. It is |
| 33 | // distinct from the grader's own container so a stale agent container can never |
| 34 | // be mistaken for an evaluation in progress. |
| 35 | func swebenchContainer(instanceID string) string { |
| 36 | return "rxagent." + strings.ReplaceAll(instanceID, "__", ".") |
| 37 | } |
| 38 | |
| 39 | // testbedShell wraps a command so it runs against the instance's conda |
| 40 | // environment. The images ship miniconda with the repo's dependencies in an env |
| 41 | // named "testbed"; a bare `docker exec` misses it and every import fails. |
| 42 | func testbedShell(command string) []string { |
| 43 | return []string{"bash", "-lc", |
| 44 | "source /opt/miniconda3/bin/activate && conda activate testbed && cd /testbed && " + command} |
| 45 | } |
| 46 | |
| 47 | // permissionFlag maps the benchmark preset onto the same public CLI contract |
| 48 | // used by every other entry point. |
| 49 | func permissionFlag(mode string) (string, error) { |
| 50 | switch mode { |
| 51 | case "", "workspace-write": |
| 52 | return "--permission-mode=workspace-write", nil |
| 53 | case "read-only": |
| 54 | return "--permission-mode=read-only", nil |
| 55 | case "danger-full-access": |
| 56 | return "--permission-mode=danger-full-access", nil |
| 57 | default: |
| 58 | return "", fmt.Errorf("unknown permission preset %q (want read-only, workspace-write, or danger-full-access)", mode) |
| 59 | } |
| 60 | } |
| 61 | |
| 62 | func swebenchAgentArgs(metricsPath, model, permission string, arm ablation.Set, maxSteps int, prompt string) []string { |
| 63 | posture, err := permissionFlag(permission) |
| 64 | if err != nil { |
| 65 | panic(err) // validated at flag-parse time; reaching here is a wiring bug |
| 66 | } |
| 67 | args := []string{"run", posture, "--metrics", metricsPath} |
| 68 | if model != "" { |
| 69 | args = append(args, "--model", model) |
| 70 | } |
| 71 | if maxSteps > 0 { |
| 72 | args = append(args, "--max-steps", fmt.Sprint(maxSteps)) |
| 73 | } |
| 74 | if !arm.Empty() { |
| 75 | args = append(args, "--ablate", arm.String()) |
| 76 | } |
| 77 | return append(args, prompt) |
| 78 | } |
| 79 | |
| 80 | // swebenchPrompt is the task text the agent sees. It carries the issue and the |
| 81 | // working rules, and deliberately withholds the test patch and the |
| 82 | // FAIL_TO_PASS list — those are the answer key. |
| 83 | func swebenchPrompt(inst swebenchInstance) string { |
| 84 | var b strings.Builder |
| 85 | b.WriteString("Resolve the following issue in the repository at /testbed.\n\n") |
| 86 | b.WriteString("<issue>\n") |
| 87 | b.WriteString(strings.TrimSpace(inst.Problem)) |
| 88 | b.WriteString("\n</issue>\n\n") |
| 89 | b.WriteString("The repository is a git checkout at the commit where the issue reproduces. ") |
| 90 | b.WriteString("Edit the source to fix it, and run the project's own tests to check your work. ") |
| 91 | b.WriteString("Do not commit, and do not modify any test file — the fix is graded by tests you cannot see.\n") |
| 92 | return b.String() |
| 93 | } |
| 94 | |
| 95 | // swebenchPrediction is one line of the predictions file the official grader |
| 96 | // reads. Field names are the harness's, not ours. |
| 97 | type swebenchPrediction struct { |
| 98 | InstanceID string `json:"instance_id"` |
| 99 | Model string `json:"model_name_or_path"` |
| 100 | Patch string `json:"model_patch"` |
| 101 | } |
| 102 | |
| 103 | func encodePredictions(model string, patches map[string]string, order []string) (string, error) { |
| 104 | var b strings.Builder |
| 105 | for _, id := range order { |
| 106 | patch, ok := patches[id] |
| 107 | if !ok { |
| 108 | continue |
| 109 | } |
| 110 | line, err := json.Marshal(swebenchPrediction{InstanceID: id, Model: model, Patch: patch}) |
| 111 | if err != nil { |
| 112 | return "", err |
| 113 | } |
| 114 | b.Write(line) |
| 115 | b.WriteByte('\n') |
| 116 | } |
| 117 | return b.String(), nil |
| 118 | } |
| 119 | |
| 120 | // swebenchReport is the subset of the grader's JSON summary we consume. Unknown |
| 121 | // fields are ignored so a harness upgrade that adds counters does not break us. |
| 122 | type swebenchReport struct { |
| 123 | ResolvedIDs []string `json:"resolved_ids"` |
| 124 | UnresolvedIDs []string `json:"unresolved_ids"` |
| 125 | ErrorIDs []string `json:"error_ids"` |
| 126 | EmptyPatchIDs []string `json:"empty_patch_ids"` |
| 127 | IncompleteIDs []string `json:"incomplete_ids"` |
| 128 | } |
| 129 | |
| 130 | // swebenchReportPath is where run_evaluation writes its summary: the model name |
| 131 | // from the predictions file joined with the run id, in the working directory. |
| 132 | func swebenchReportPath(model, runID string) string { |
| 133 | return model + "." + runID + ".json" |
| 134 | } |
| 135 | |
| 136 | // gradedClass maps one instance's grader outcome onto our published failure |
| 137 | // taxonomy. An id the grader never mentions is reported as unknown rather than |
| 138 | // silently counted as unsolved. |
| 139 | func (r swebenchReport) gradedClass(instanceID string) string { |
| 140 | if slices.Contains(r.ResolvedIDs, instanceID) { |
| 141 | return "solved" |
| 142 | } |
| 143 | if slices.Contains(r.EmptyPatchIDs, instanceID) { |
| 144 | return "no_patch" |
| 145 | } |
| 146 | if slices.Contains(r.ErrorIDs, instanceID) { |
| 147 | return "grader_error" |
| 148 | } |
| 149 | if slices.Contains(r.IncompleteIDs, instanceID) { |
| 150 | return "eval_timeout" |
| 151 | } |
| 152 | if slices.Contains(r.UnresolvedIDs, instanceID) { |
| 153 | return "wrong_patch" |
| 154 | } |
| 155 | return "ungraded" |
| 156 | } |
| 157 |