| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "encoding/json" |
| 5 | "reflect" |
| 6 | "strings" |
| 7 | "testing" |
| 8 | |
| 9 | "github.com/BurntSushi/toml" |
| 10 | |
| 11 | "reasonix/internal/ablation" |
| 12 | "reasonix/internal/config" |
| 13 | ) |
| 14 | |
| 15 | // The harness prints an unmangled local image key but pulls a mangled one; the |
| 16 | // two differ and only the mangled form exists in the registry. |
| 17 | func TestSwebenchImageManglesDoubleUnderscore(t *testing.T) { |
| 18 | got := swebenchImage("swebench", "psf__requests-2317") |
| 19 | want := "swebench/sweb.eval.x86_64.psf_1776_requests-2317:latest" |
| 20 | if got != want { |
| 21 | t.Fatalf("image = %q, want %q", got, want) |
| 22 | } |
| 23 | if got := swebenchImage("swebench", "pylint-dev__pylint-7080"); !strings.Contains(got, "pylint-dev_1776_pylint-7080") { |
| 24 | t.Fatalf("hyphenated org mangled wrong: %q", got) |
| 25 | } |
| 26 | } |
| 27 | |
| 28 | func TestTestbedShellActivatesTheInstanceCondaEnv(t *testing.T) { |
| 29 | got := testbedShell("git diff") |
| 30 | if len(got) != 3 || got[0] != "bash" || got[1] != "-lc" { |
| 31 | t.Fatalf("shell wrapper = %v", got) |
| 32 | } |
| 33 | for _, want := range []string{"/opt/miniconda3/bin/activate", "conda activate testbed", "cd /testbed", "git diff"} { |
| 34 | if !strings.Contains(got[2], want) { |
| 35 | t.Errorf("command missing %q: %s", want, got[2]) |
| 36 | } |
| 37 | } |
| 38 | } |
| 39 | |
| 40 | func TestSwebenchPromptWithholdsTheAnswerKey(t *testing.T) { |
| 41 | prompt := swebenchPrompt(swebenchInstance{ |
| 42 | InstanceID: "psf__requests-2317", |
| 43 | Problem: "method = builtin_str(method) breaks binary strings", |
| 44 | }) |
| 45 | if !strings.Contains(prompt, "builtin_str(method)") { |
| 46 | t.Fatal("the issue text must reach the agent") |
| 47 | } |
| 48 | for _, leak := range []string{"FAIL_TO_PASS", "PASS_TO_PASS", "test_patch"} { |
| 49 | if strings.Contains(prompt, leak) { |
| 50 | t.Errorf("prompt leaks the answer key: %s", leak) |
| 51 | } |
| 52 | } |
| 53 | if !strings.Contains(prompt, "do not modify any test file") { |
| 54 | t.Error("the no-test-edit rule must be stated; the grader replaces test files anyway") |
| 55 | } |
| 56 | } |
| 57 | |
| 58 | func TestEncodePredictionsWritesOneHarnessRecordPerLine(t *testing.T) { |
| 59 | out, err := encodePredictions("reasonix", map[string]string{ |
| 60 | "a__a-1": "diff --git a/x b/x\n", |
| 61 | "b__b-2": "diff --git a/y b/y\n", |
| 62 | }, []string{"a__a-1", "missing__missing-9", "b__b-2"}) |
| 63 | if err != nil { |
| 64 | t.Fatalf("encode: %v", err) |
| 65 | } |
| 66 | lines := strings.Split(strings.TrimSpace(out), "\n") |
| 67 | if len(lines) != 2 { |
| 68 | t.Fatalf("lines = %d, want 2 (an instance with no patch is skipped, not emitted empty)", len(lines)) |
| 69 | } |
| 70 | for _, want := range []string{`"instance_id":"a__a-1"`, `"model_name_or_path":"reasonix"`, `"model_patch":"diff --git a/x b/x\n"`} { |
| 71 | if !strings.Contains(lines[0], want) { |
| 72 | t.Errorf("first record missing %q: %s", want, lines[0]) |
| 73 | } |
| 74 | } |
| 75 | } |
| 76 | |
| 77 | func TestGradedClassNeverGuessesForAnUnmentionedInstance(t *testing.T) { |
| 78 | report := swebenchReport{ |
| 79 | ResolvedIDs: []string{"a__a-1"}, |
| 80 | UnresolvedIDs: []string{"b__b-2"}, |
| 81 | ErrorIDs: []string{"c__c-3"}, |
| 82 | EmptyPatchIDs: []string{"d__d-4"}, |
| 83 | IncompleteIDs: []string{"f__f-6"}, |
| 84 | } |
| 85 | for id, want := range map[string]string{ |
| 86 | "a__a-1": "solved", |
| 87 | "b__b-2": "wrong_patch", |
| 88 | "c__c-3": "grader_error", |
| 89 | "d__d-4": "no_patch", |
| 90 | "f__f-6": "eval_timeout", |
| 91 | "e__e-5": "ungraded", |
| 92 | } { |
| 93 | if got := report.gradedClass(id); got != want { |
| 94 | t.Errorf("class(%s) = %q, want %q", id, got, want) |
| 95 | } |
| 96 | } |
| 97 | } |
| 98 | |
| 99 | // Captured from a real run on 2026-08-04: swebench 4.1.0, gold predictions, |
| 100 | // run_id goldpylint. Pins the field names and the report path we depend on. |
| 101 | func TestSwebenchReportParsesTheHarnessSummary(t *testing.T) { |
| 102 | const captured = `{"total_instances":1,"submitted_instances":500,"completed_instances":1, |
| 103 | "resolved_instances":1,"unresolved_instances":0,"empty_patch_instances":0,"error_instances":0, |
| 104 | "completed_ids":["pylint-dev__pylint-7080"],"incomplete_ids":[],"empty_patch_ids":[], |
| 105 | "resolved_ids":["pylint-dev__pylint-7080"],"unresolved_ids":[],"error_ids":[],"schema_version":2}` |
| 106 | |
| 107 | var report swebenchReport |
| 108 | if err := json.Unmarshal([]byte(captured), &report); err != nil { |
| 109 | t.Fatalf("unmarshal: %v", err) |
| 110 | } |
| 111 | if got := report.gradedClass("pylint-dev__pylint-7080"); got != "solved" { |
| 112 | t.Fatalf("gold patch classified as %q, want solved", got) |
| 113 | } |
| 114 | if got := swebenchReportPath("gold", "goldpylint"); got != "gold.goldpylint.json" { |
| 115 | t.Fatalf("report path = %q, want gold.goldpylint.json", got) |
| 116 | } |
| 117 | } |
| 118 | |
| 119 | // Issue text arrives verbatim from GitHub and routinely contains quotes, |
| 120 | // backticks, $ and newlines. It becomes one argv element inside a `bash -lc` |
| 121 | // string, so a quoting slip would let a problem statement run commands. |
| 122 | func TestShellQuoteAllContainsHostileIssueText(t *testing.T) { |
| 123 | hostile := "it's broken; `rm -rf /`; $(whoami)\n\"quoted\" && echo pwned" |
| 124 | got := shellQuoteAll([]string{"run", hostile}) |
| 125 | if !strings.HasPrefix(got, "'run' '") || !strings.HasSuffix(got, "'") { |
| 126 | t.Fatalf("every element must be single-quoted: %s", got) |
| 127 | } |
| 128 | // Inside single quotes the shell expands nothing, so the only way out is an |
| 129 | // unescaped apostrophe: every one in the payload must have been rewritten. |
| 130 | if strings.Count(got, `'\''`) != strings.Count(hostile, "'") { |
| 131 | t.Fatalf("apostrophes not all escaped: %s", got) |
| 132 | } |
| 133 | } |
| 134 | |
| 135 | func TestSwebenchAgentArgsKeepTheControlArmClean(t *testing.T) { |
| 136 | got := swebenchAgentArgs("/tmp/m.json", "e2e", "workspace-write", ablation.Set{}, 60, "fix it") |
| 137 | want := []string{"run", "--permission-mode=workspace-write", "--metrics", "/tmp/m.json", "--model", "e2e", "--max-steps", "60", "fix it"} |
| 138 | if !reflect.DeepEqual(got, want) { |
| 139 | t.Fatalf("control args = %v, want %v", got, want) |
| 140 | } |
| 141 | ablated := swebenchAgentArgs("/tmp/m.json", "", "workspace-write", ablation.New(ablation.Evidence), 0, "fix it") |
| 142 | if !reflect.DeepEqual(ablated, []string{"run", "--permission-mode=workspace-write", "--metrics", "/tmp/m.json", "--ablate", "evidence", "fix it"}) { |
| 143 | t.Fatalf("ablated args = %v", ablated) |
| 144 | } |
| 145 | } |
| 146 | |
| 147 | // The two postures must differ in exactly one argument. If anything else moved |
| 148 | // between arms, the published delta would not isolate the permission gate. |
| 149 | func TestPermissionPostureIsTheOnlyDifferenceBetweenArms(t *testing.T) { |
| 150 | a := swebenchAgentArgs("/m.json", "e2e", "workspace-write", ablation.Set{}, 60, "fix it") |
| 151 | b := swebenchAgentArgs("/m.json", "e2e", "danger-full-access", ablation.Set{}, 60, "fix it") |
| 152 | if len(a) != len(b) { |
| 153 | t.Fatalf("arms differ in argument count: %v vs %v", a, b) |
| 154 | } |
| 155 | diffs := 0 |
| 156 | for i := range a { |
| 157 | if a[i] != b[i] { |
| 158 | diffs++ |
| 159 | } |
| 160 | } |
| 161 | if diffs != 1 || a[1] != "--permission-mode=workspace-write" || b[1] != "--permission-mode=danger-full-access" { |
| 162 | t.Fatalf("arms must differ only in the posture flag: %v vs %v", a, b) |
| 163 | } |
| 164 | if _, err := permissionFlag("bypass"); err == nil { |
| 165 | t.Fatal("an unknown posture must fail loudly rather than silently running unattended") |
| 166 | } |
| 167 | } |
| 168 | |
| 169 | // The benchmark config must remain a default install, except for the required |
| 170 | // bash sandbox override; decode it with the real config type to pin the keys. |
| 171 | func TestSwebenchAgentConfigTunesNothingInTheAgentsFavor(t *testing.T) { |
| 172 | var cfg config.Config |
| 173 | meta, err := toml.Decode(swebenchAgentConfig, &cfg) |
| 174 | if err != nil { |
| 175 | t.Fatalf("container config must be valid TOML: %v", err) |
| 176 | } |
| 177 | if cfg.Sandbox.Bash != "off" { |
| 178 | t.Errorf("sandbox.bash = %q, want \"off\": bubblewrap is absent from the official images", cfg.Sandbox.Bash) |
| 179 | } |
| 180 | // Do not configure the benchmark's blocked network as an agent fact. |
| 181 | if cfg.Environment.Offline { |
| 182 | t.Error("the benchmark must not declare the environment offline: that configures the agent better than a default install") |
| 183 | } |
| 184 | for _, key := range meta.Keys() { |
| 185 | if meta.Type(key...) == "Hash" { |
| 186 | continue // table header, not a setting |
| 187 | } |
| 188 | if got := key.String(); got != "sandbox.bash" { |
| 189 | t.Errorf("unexpected benchmark-only setting %q: the container must run a default install", got) |
| 190 | } |
| 191 | } |
| 192 | } |
| 193 | |
| 194 | // Binary entries are omitted because their placeholder is not applicable; |
| 195 | // every text entry, including generated files, must remain in the patch. |
| 196 | func TestPatchFileListDropsBinariesAndNothingElse(t *testing.T) { |
| 197 | numstat := strings.Join([]string{ |
| 198 | "12\t4\tsphinx/directives/other.py", // the actual fix |
| 199 | "0\t9\tsphinx/old_helper.py", // text deletion |
| 200 | "-\t-\t_repro/_build/.doctrees/environment.pickle", // binary: drop |
| 201 | "-\t-\timg/probe.png", // new binary: drop |
| 202 | "3\t0\t_repro/_build/_static/basic.css", // build output the agent left: keep |
| 203 | "2\t0\tsklearn.egg-info/PKG-INFO", // packaging metadata: keep |
| 204 | "1\t0\tpkg/__pycache__/note.txt", // cache tree: keep |
| 205 | "5\t1\trepro.py", // agent scratch: keep |
| 206 | }, "\x00") + "\x00" |
| 207 | got := patchFileList(numstat) |
| 208 | want := []string{ |
| 209 | "sphinx/directives/other.py", |
| 210 | "sphinx/old_helper.py", |
| 211 | "_repro/_build/_static/basic.css", |
| 212 | "sklearn.egg-info/PKG-INFO", |
| 213 | "pkg/__pycache__/note.txt", |
| 214 | "repro.py", |
| 215 | } |
| 216 | if !reflect.DeepEqual(got, want) { |
| 217 | t.Fatalf("patchFileList = %v, want %v", got, want) |
| 218 | } |
| 219 | } |
| 220 | |
| 221 | func TestPatchFileListToleratesEmptyAndMalformedInput(t *testing.T) { |
| 222 | if got := patchFileList(""); got != nil { |
| 223 | t.Fatalf("empty numstat = %v, want nil", got) |
| 224 | } |
| 225 | if got := patchFileList("garbage-without-tabs\x00\x00"); got != nil { |
| 226 | t.Fatalf("malformed numstat = %v, want nil", got) |
| 227 | } |
| 228 | } |
| 229 | |
| 230 | func TestTestbedPatchDiffArgsTreatPathsLiterally(t *testing.T) { |
| 231 | got := testbedPatchDiffArgs("agent-123", []string{"source.py", ":(exclude)*"}) |
| 232 | want := []string{ |
| 233 | "exec", "agent-123", "git", "--literal-pathspecs", "-C", "/testbed", |
| 234 | "diff", "--cached", "--no-renames", "--", "source.py", ":(exclude)*", |
| 235 | } |
| 236 | if !reflect.DeepEqual(got, want) { |
| 237 | t.Fatalf("patch diff args = %v, want %v", got, want) |
| 238 | } |
| 239 | } |
| 240 | |
| 241 | func TestPatchFileBatchesBoundArgBytesAndPreserveOrder(t *testing.T) { |
| 242 | files := make([]string, 2000) |
| 243 | for i := range files { |
| 244 | files[i] = "generated.txt" |
| 245 | } |
| 246 | |
| 247 | batches, err := patchFileBatches("agent-123", files) |
| 248 | if err != nil { |
| 249 | t.Fatalf("patchFileBatches: %v", err) |
| 250 | } |
| 251 | if len(batches) < 2 { |
| 252 | t.Fatalf("batches = %d, want multiple batches", len(batches)) |
| 253 | } |
| 254 | |
| 255 | var got []string |
| 256 | for _, batch := range batches { |
| 257 | if bytes := argvBytes(testbedPatchDiffArgs("agent-123", batch)); bytes > patchArgBudget { |
| 258 | t.Fatalf("batch argv bytes = %d, budget = %d", bytes, patchArgBudget) |
| 259 | } |
| 260 | got = append(got, batch...) |
| 261 | } |
| 262 | if !reflect.DeepEqual(got, files) { |
| 263 | t.Fatalf("batched paths changed: got %d paths, want %d", len(got), len(files)) |
| 264 | } |
| 265 | } |
| 266 | |
| 267 | func TestPatchFileBatchesRejectOversizedPath(t *testing.T) { |
| 268 | path := strings.Repeat("x", patchArgBudget) |
| 269 | if _, err := patchFileBatches("agent-123", []string{path}); err == nil { |
| 270 | t.Fatal("oversized path must be rejected") |
| 271 | } |
| 272 | } |
| 273 |