| 1 | package evidence |
| 2 | |
| 3 | import ( |
| 4 | "encoding/json" |
| 5 | "strings" |
| 6 | "testing" |
| 7 | ) |
| 8 | |
| 9 | // TestVerificationCommandSummaryRecommendationsAreRecognized guards the |
| 10 | // single structured source used to render the summary and exercise the |
| 11 | // classifier. Every concrete example must render and remain accepted. |
| 12 | func TestVerificationCommandSummaryRecommendationsAreRecognized(t *testing.T) { |
| 13 | summary := VerificationCommandSummary() |
| 14 | for _, recommendation := range verificationCommandRecommendations() { |
| 15 | for _, command := range recommendation.examples { |
| 16 | if !strings.Contains(summary, command) { |
| 17 | t.Errorf("summary family %q missing concrete example %q: %s", recommendation.label, command, summary) |
| 18 | } |
| 19 | if !IsDeliveryVerificationCommand(command) { |
| 20 | t.Errorf("summary family %q advertises %q, but classifier rejects it", recommendation.label, command) |
| 21 | } |
| 22 | } |
| 23 | } |
| 24 | } |
| 25 | |
| 26 | func TestGoBuildAlwaysCountsAsMutation(t *testing.T) { |
| 27 | // Even ./... can expand to one main package and write a root executable. |
| 28 | // The static classifier cannot know package expansion or inherited GOFLAGS, |
| 29 | // so every go build form must fail closed as a mutation. |
| 30 | for _, command := range []string{ |
| 31 | "go build", |
| 32 | "go build .", |
| 33 | "go build ./cmd/reasonix", |
| 34 | "go build -race ./cmd/reasonix", |
| 35 | "go build ./...", |
| 36 | "go build -tags integration ./...", |
| 37 | "go build -tags=integration ./...", |
| 38 | "go build -race -trimpath ./...", |
| 39 | "go build -p 2 -mod=readonly ./...", |
| 40 | "go build -buildvcs ./...", |
| 41 | "go build -buildvcs=auto -- ./...", |
| 42 | "go build -tags ./... ./cmd/reasonix", |
| 43 | "go build -coverpkg ./... ./cmd/reasonix", |
| 44 | "go build -pkgdir ./... ./cmd/reasonix", |
| 45 | "go build -pkgdir=./... ./cmd/reasonix", |
| 46 | "go build -o reasonix ./...", |
| 47 | "go build -o=reasonix ./...", |
| 48 | "go build -mod=mod ./...", |
| 49 | "go build -n ./...", |
| 50 | "go build -work ./...", |
| 51 | "go build -future-flag ./...", |
| 52 | "go build ./... -o reasonix", |
| 53 | } { |
| 54 | if IsDeliveryVerificationCommand(command) { |
| 55 | t.Errorf("%q must not count as non-mutating verification", command) |
| 56 | } |
| 57 | args, err := json.Marshal(map[string]string{"command": command}) |
| 58 | if err != nil { |
| 59 | t.Fatal(err) |
| 60 | } |
| 61 | if !ToolCallMutates("bash", args, false) { |
| 62 | t.Errorf("%q must be classified as a mutation", command) |
| 63 | } |
| 64 | } |
| 65 | if strings.Contains(VerificationCommandSummary(), "go build") { |
| 66 | t.Fatal("recovery summary must not recommend go build as a first-line verifier") |
| 67 | } |
| 68 | } |
| 69 | |
| 70 | func TestTSCVerificationRequiresExplicitNoEmit(t *testing.T) { |
| 71 | for _, command := range []string{ |
| 72 | "tsc --noEmit", |
| 73 | "tsc --noEmit=true", |
| 74 | "tsc --project tsconfig.json --noEmit", |
| 75 | "tsc --noEmit --listFiles", |
| 76 | "tsc --noEmit --pretty false", |
| 77 | } { |
| 78 | if !IsDeliveryVerificationCommand(command) { |
| 79 | t.Errorf("%q should be recognized as an explicit no-emit type check", command) |
| 80 | } |
| 81 | args, err := json.Marshal(map[string]string{"command": command}) |
| 82 | if err != nil { |
| 83 | t.Fatal(err) |
| 84 | } |
| 85 | if ToolCallMutates("bash", args, false) { |
| 86 | t.Errorf("%q should remain a non-mutating verification", command) |
| 87 | } |
| 88 | } |
| 89 | |
| 90 | for _, command := range []string{ |
| 91 | "tsc", |
| 92 | "tsc --outDir dist", |
| 93 | "tsc --noEmit=false", |
| 94 | "tsc --noEmit false", |
| 95 | "tsc --noEmit=true --noEmit=false", |
| 96 | "tsc --noEmit --incremental --tsBuildInfoFile victim.ts", |
| 97 | "tsc --noEmit --incremental --tsBuildInfoFile=victim.ts", |
| 98 | "tsc --noEmit --generateTrace trace-dir", |
| 99 | "tsc --noEmit --generateTrace=trace-dir", |
| 100 | "tsc --noEmit --generateCpuProfile profile.cpuprofile", |
| 101 | "tsc --noEmit --generateCpuProfile=profile.cpuprofile", |
| 102 | "tsc --noEmit --init", |
| 103 | "tsc --noEmit --help", |
| 104 | "tsc --noEmit -h", |
| 105 | "tsc --noEmit -?", |
| 106 | "tsc --noEmit --all", |
| 107 | "tsc --noEmit --version", |
| 108 | "tsc --noEmit -v", |
| 109 | "tsc --noEmit --showConfig", |
| 110 | "tsc --noEmit --listFilesOnly", |
| 111 | "tsc --noEmit --noCheck", |
| 112 | "tsc --noEmit --watch", |
| 113 | "tsc --noEmit -w", |
| 114 | "tsc --build --noEmit", |
| 115 | "tsc -b --noEmit", |
| 116 | "tsc --build --clean --noEmit", |
| 117 | } { |
| 118 | if IsDeliveryVerificationCommand(command) { |
| 119 | t.Errorf("%q is not a bounded no-emit type check and must not count as verification", command) |
| 120 | } |
| 121 | args, err := json.Marshal(map[string]string{"command": command}) |
| 122 | if err != nil { |
| 123 | t.Fatal(err) |
| 124 | } |
| 125 | if !ToolCallMutates("bash", args, false) { |
| 126 | t.Errorf("%q must fail closed as a mutation", command) |
| 127 | } |
| 128 | } |
| 129 | |
| 130 | if !strings.Contains(VerificationCommandSummary(), "tsc --noEmit") { |
| 131 | t.Fatal("recovery summary must render the concrete no-emit TypeScript command") |
| 132 | } |
| 133 | } |
| 134 | |
| 135 | func TestSwiftTestRecognizedAsVerification(t *testing.T) { |
| 136 | // swift test runs the SwiftPM test suite; the build cache lands in the |
| 137 | // package's own .build directory, mirroring cargo test acceptance. Other |
| 138 | // swift subcommands emit binaries, run arbitrary code, or mutate the |
| 139 | // package graph and must fail closed as mutations. |
| 140 | for _, command := range []string{ |
| 141 | "swift test", |
| 142 | "swift test --parallel", |
| 143 | "swift test --filter SomeTests", |
| 144 | "swift test --enable-code-coverage", |
| 145 | } { |
| 146 | if !IsDeliveryVerificationCommand(command) { |
| 147 | t.Errorf("%q should be recognized as a read-only Swift test verifier", command) |
| 148 | } |
| 149 | args, err := json.Marshal(map[string]string{"command": command}) |
| 150 | if err != nil { |
| 151 | t.Fatal(err) |
| 152 | } |
| 153 | if ToolCallMutates("bash", args, false) { |
| 154 | t.Errorf("%q should remain a non-mutating verification", command) |
| 155 | } |
| 156 | } |
| 157 | |
| 158 | for _, command := range []string{ |
| 159 | "swift", |
| 160 | "swift build", |
| 161 | "swift build -c release", |
| 162 | "swift run", |
| 163 | "swift package resolve", |
| 164 | "swift package update", |
| 165 | "swift test --xunit-output report.xml", |
| 166 | "swift test --xunit-output=report.xml", |
| 167 | "swift test --scratch-path /tmp/out", |
| 168 | "swift test --build-path /tmp/out", |
| 169 | "swift test --build-path=/tmp/out", |
| 170 | "swift test --event-stream-output-path /tmp/events.json", |
| 171 | "swift test --experimental-event-stream-output /tmp/events.json", |
| 172 | "swift test --attachments-path /tmp/attachments", |
| 173 | "swift test --experimental-attachments-path /tmp/attachments", |
| 174 | "swift test --cache-path /tmp/cache", |
| 175 | "swift test --help", |
| 176 | "swift test --list-tests", |
| 177 | } { |
| 178 | if IsDeliveryVerificationCommand(command) { |
| 179 | t.Errorf("%q can build, run, or mutate the package and must not count as verification", command) |
| 180 | } |
| 181 | args, err := json.Marshal(map[string]string{"command": command}) |
| 182 | if err != nil { |
| 183 | t.Fatal(err) |
| 184 | } |
| 185 | if !ToolCallMutates("bash", args, false) { |
| 186 | t.Errorf("%q must fail closed as a mutation", command) |
| 187 | } |
| 188 | } |
| 189 | |
| 190 | if !strings.Contains(VerificationCommandSummary(), "swift test") { |
| 191 | t.Fatal("recovery summary must render the concrete Swift test command") |
| 192 | } |
| 193 | } |
| 194 | |
| 195 | func TestPythonCompileallAlwaysCountsAsMutation(t *testing.T) { |
| 196 | for _, command := range []string{ |
| 197 | "python -m compileall .", |
| 198 | "python3 -m compileall -q src/", |
| 199 | "python -m compileall -b package.py", |
| 200 | } { |
| 201 | if IsDeliveryVerificationCommand(command) { |
| 202 | t.Errorf("%q writes bytecode and must not count as verification", command) |
| 203 | } |
| 204 | args, err := json.Marshal(map[string]string{"command": command}) |
| 205 | if err != nil { |
| 206 | t.Fatal(err) |
| 207 | } |
| 208 | if !ToolCallMutates("bash", args, false) { |
| 209 | t.Errorf("%q writes bytecode and must be classified as a mutation", command) |
| 210 | } |
| 211 | } |
| 212 | if strings.Contains(VerificationCommandSummary(), "compileall") { |
| 213 | t.Fatal("recovery summary must not recommend bytecode-emitting compileall") |
| 214 | } |
| 215 | } |
| 216 | |
| 217 | func TestNpxVerificationUsesSafeKnownRunners(t *testing.T) { |
| 218 | for _, command := range []string{ |
| 219 | "npx vitest run src/lib/foo.test.ts", |
| 220 | "npx vitest@1.6.0 run src/lib/foo.test.ts", |
| 221 | "npx jest src/lib/foo.test.ts", |
| 222 | "npx mocha test/", |
| 223 | "npx ava", |
| 224 | "npx eslint src/", |
| 225 | "npx prettier --check .", |
| 226 | "npx prettier --list-different src/", |
| 227 | "npx tsc --noEmit", |
| 228 | "npx tsc --project tsconfig.json --noEmit", |
| 229 | "npx mocha test/ 2>&1 | tail -40", |
| 230 | } { |
| 231 | if !IsDeliveryVerificationCommand(command) { |
| 232 | t.Errorf("%q should be recognized as a known read-only npx verifier", command) |
| 233 | } |
| 234 | args, err := json.Marshal(map[string]string{"command": command}) |
| 235 | if err != nil { |
| 236 | t.Fatal(err) |
| 237 | } |
| 238 | if ToolCallMutates("bash", args, false) { |
| 239 | t.Errorf("%q should remain non-mutating verification", command) |
| 240 | } |
| 241 | } |
| 242 | |
| 243 | for _, command := range []string{ |
| 244 | "npx --yes vitest run", |
| 245 | "npx eslint@npm:evil src/", |
| 246 | "npx vitest@file:../fake run", |
| 247 | "npx ./eslint src/", |
| 248 | "npx /tmp/eslint src/", |
| 249 | "npx playwright test", |
| 250 | "npx cypress run", |
| 251 | "npx tsx check.ts", |
| 252 | "npx ts-node check.ts", |
| 253 | "npx eslint src/ --fix", |
| 254 | "npx eslint src/ --output-file report.txt", |
| 255 | "npx prettier --write .", |
| 256 | "npx prettier .", |
| 257 | "npx tsc", |
| 258 | "npx tsc --noEmit --tsBuildInfoFile victim.ts", |
| 259 | "npx tsc --noEmit --generateTrace trace-dir", |
| 260 | "npx tsc --noEmit --init", |
| 261 | "npx tsc --noEmit --showConfig", |
| 262 | "npx tsc --noEmit --listFilesOnly", |
| 263 | "npx tsc --noEmit --noCheck", |
| 264 | "npx tsc --noEmit --watch", |
| 265 | "npx tsc --build --noEmit", |
| 266 | } { |
| 267 | if IsDeliveryVerificationCommand(command) { |
| 268 | t.Errorf("%q can install, execute, or write output and must fail closed", command) |
| 269 | } |
| 270 | args, err := json.Marshal(map[string]string{"command": command}) |
| 271 | if err != nil { |
| 272 | t.Fatal(err) |
| 273 | } |
| 274 | if !ToolCallMutates("bash", args, false) { |
| 275 | t.Errorf("%q must remain a mutation", command) |
| 276 | } |
| 277 | } |
| 278 | |
| 279 | if strings.Contains(VerificationCommandSummary(), "npx") { |
| 280 | t.Fatal("self-installing npx commands must not be first-line recovery recommendations") |
| 281 | } |
| 282 | } |
| 283 | |
| 284 | // TestVerificationCommandSummaryNamesTheDeadlockTraps ensures the summary |
| 285 | // explicitly warns about the two command shapes that used to send models into |
| 286 | // the readiness deadlock loop: inline interpreters (blocked before execution) |
| 287 | // and read-only inspection commands (executed but never classified as |
| 288 | // verification). |
| 289 | func TestVerificationCommandSummaryNamesTheDeadlockTraps(t *testing.T) { |
| 290 | s := VerificationCommandSummary() |
| 291 | for _, want := range []string{ |
| 292 | "python -c", |
| 293 | "node -e", |
| 294 | "grep/find/cat/wc", |
| 295 | "NOT verification", |
| 296 | "blocked in delivery mode", |
| 297 | } { |
| 298 | if !strings.Contains(s, want) { |
| 299 | t.Errorf("summary should warn about trap %q, got: %s", want, s) |
| 300 | } |
| 301 | } |
| 302 | } |
| 303 | |
| 304 | // TestVerificationCommandSummaryAcceptedPipeline ensures the escape hatch |
| 305 | // advertised by the summary (a read-only extraction pipeline ending in a |
| 306 | // recognized verifier) really is accepted by the classifier. |
| 307 | func TestVerificationCommandSummaryAcceptedPipeline(t *testing.T) { |
| 308 | for _, cmd := range []string{ |
| 309 | "tail -n +1 out.json | node --check -", |
| 310 | "cat file.js | node --check -", |
| 311 | } { |
| 312 | if !IsDeliveryVerificationCommand(cmd) { |
| 313 | t.Errorf("advertised read-only pipeline %q rejected by classifier", cmd) |
| 314 | } |
| 315 | } |
| 316 | } |
| 317 |