| 1 | package agent |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "fmt" |
| 7 | "os" |
| 8 | "path/filepath" |
| 9 | "strings" |
| 10 | "testing" |
| 11 | |
| 12 | "reasonix/internal/event" |
| 13 | "reasonix/internal/provider" |
| 14 | "reasonix/internal/tool" |
| 15 | "reasonix/internal/tool/builtin" |
| 16 | ) |
| 17 | |
| 18 | func TestOrdinaryModeBlocksMixedMutationAndVerification(t *testing.T) { |
| 19 | // Preflight runs before Execute, so a fake bash is enough — the process |
| 20 | // must never start for a mixed mutation+verification command. `;` is the |
| 21 | // shape that matters: the verifier's exit status replaces go generate's. |
| 22 | reg := tool.NewRegistry() |
| 23 | reg.Add(fakeTool{name: "bash", readOnly: false}) |
| 24 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 25 | {toolCallChunk("m1", "bash", `{"command":"go generate ./... ; go test ./..."}`), {Type: provider.ChunkDone}}, |
| 26 | {{Type: provider.ChunkText, Text: "ok"}, {Type: provider.ChunkDone}}, |
| 27 | }} |
| 28 | a := New(prov, reg, NewSession(""), Options{}, event.Discard) |
| 29 | if err := a.Run(context.Background(), "test"); err != nil { |
| 30 | t.Fatal(err) |
| 31 | } |
| 32 | got := toolResultByID(a.session, "m1") |
| 33 | if strings.Contains(got, "bash done") { |
| 34 | t.Fatal("mixed command was executed") |
| 35 | } |
| 36 | if !strings.Contains(got, "state-changing segment") { |
| 37 | t.Fatalf("result = %q, want ordinary-mode mixed block", got) |
| 38 | } |
| 39 | for _, msg := range a.session.Snapshot() { |
| 40 | if msg.ToolCallID != "m1" { |
| 41 | continue |
| 42 | } |
| 43 | if msg.ToolExecution == nil || msg.ToolExecution.State != tool.ShellStateNotRun { |
| 44 | t.Fatalf("execution = %+v, want not_run", msg.ToolExecution) |
| 45 | } |
| 46 | if msg.ToolExecution.FailurePhase != tool.ShellPhasePreflight { |
| 47 | t.Fatalf("phase = %q", msg.ToolExecution.FailurePhase) |
| 48 | } |
| 49 | return |
| 50 | } |
| 51 | t.Fatal("tool result missing") |
| 52 | } |
| 53 | |
| 54 | // TestOrdinaryModeRunsShortCircuitBuildAndVerify guards the everyday shape the |
| 55 | // preflight must not touch. `go build ./... && go test ./...` cannot report a |
| 56 | // false success: bash stops at the failing build and returns its status. Only |
| 57 | // Delivery blocks it, because there a mutation invalidates the verification |
| 58 | // receipt regardless of exit status. |
| 59 | func TestOrdinaryModeRunsShortCircuitBuildAndVerify(t *testing.T) { |
| 60 | commands := []string{ |
| 61 | "go build ./... && go test ./...", |
| 62 | "npm install && npm test", |
| 63 | "mkdir -p out && go test ./...", |
| 64 | } |
| 65 | for _, command := range commands { |
| 66 | t.Run(command, func(t *testing.T) { |
| 67 | reg := tool.NewRegistry() |
| 68 | reg.Add(fakeTool{name: "bash", readOnly: false}) |
| 69 | args, err := json.Marshal(map[string]string{"command": command}) |
| 70 | if err != nil { |
| 71 | t.Fatal(err) |
| 72 | } |
| 73 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 74 | {toolCallChunk("m1", "bash", string(args)), {Type: provider.ChunkDone}}, |
| 75 | {{Type: provider.ChunkText, Text: "ok"}, {Type: provider.ChunkDone}}, |
| 76 | }} |
| 77 | a := New(prov, reg, NewSession(""), Options{}, event.Discard) |
| 78 | if err := a.Run(context.Background(), "test"); err != nil { |
| 79 | t.Fatal(err) |
| 80 | } |
| 81 | got := toolResultByID(a.session, "m1") |
| 82 | if strings.Contains(got, "blocked:") { |
| 83 | t.Fatalf("ordinary mode blocked %q: %s", command, got) |
| 84 | } |
| 85 | if !strings.Contains(got, "bash done") { |
| 86 | t.Fatalf("command did not run: result = %q", got) |
| 87 | } |
| 88 | }) |
| 89 | } |
| 90 | } |
| 91 | |
| 92 | func TestOrdinaryModeBlocksMaskedVerifierExit(t *testing.T) { |
| 93 | reg := tool.NewRegistry() |
| 94 | reg.Add(fakeTool{name: "bash", readOnly: false}) |
| 95 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 96 | {toolCallChunk("m1", "bash", `{"command":"go test ./...; echo $?"}`), {Type: provider.ChunkDone}}, |
| 97 | {{Type: provider.ChunkText, Text: "ok"}, {Type: provider.ChunkDone}}, |
| 98 | }} |
| 99 | a := New(prov, reg, NewSession(""), Options{}, event.Discard) |
| 100 | if err := a.Run(context.Background(), "test"); err != nil { |
| 101 | t.Fatal(err) |
| 102 | } |
| 103 | got := toolResultByID(a.session, "m1") |
| 104 | if strings.Contains(got, "bash done") { |
| 105 | t.Fatal("masked exit command was executed") |
| 106 | } |
| 107 | if !strings.Contains(got, "masks") && !strings.Contains(got, "exit status") { |
| 108 | t.Fatalf("result = %q, want mask block", got) |
| 109 | } |
| 110 | } |
| 111 | |
| 112 | func TestOrdinaryModeBlocksNonTerminalInlineInterpreter(t *testing.T) { |
| 113 | reg := tool.NewRegistry() |
| 114 | reg.Add(fakeTool{name: "bash", readOnly: false}) |
| 115 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 116 | {toolCallChunk("m1", "bash", `{"command":"python3 -c 'open(\"x\",\"w\").write(\"y\")' ; node verify_frontend_logic.js"}`), {Type: provider.ChunkDone}}, |
| 117 | // A `&&` variant of the same pair is covered by the allow-list test above. |
| 118 | {{Type: provider.ChunkText, Text: "ok"}, {Type: provider.ChunkDone}}, |
| 119 | }} |
| 120 | a := New(prov, reg, NewSession(""), Options{}, event.Discard) |
| 121 | if err := a.Run(context.Background(), "test"); err != nil { |
| 122 | t.Fatal(err) |
| 123 | } |
| 124 | got := toolResultByID(a.session, "m1") |
| 125 | if strings.Contains(got, "bash done") { |
| 126 | t.Fatal("non-terminal inline interpreter was executed") |
| 127 | } |
| 128 | if !strings.Contains(got, "inline interpreter") { |
| 129 | t.Fatalf("result = %q, want non-terminal inline block", got) |
| 130 | } |
| 131 | } |
| 132 | |
| 133 | func TestBatchDependencyBarrierSkipsVerificationAfterFailedMutation(t *testing.T) { |
| 134 | dir := t.TempDir() |
| 135 | path := filepath.Join(dir, "x.txt") |
| 136 | if err := os.WriteFile(path, []byte("a\n"), 0o600); err != nil { |
| 137 | t.Fatal(err) |
| 138 | } |
| 139 | reg := tool.NewRegistry() |
| 140 | for _, tl := range (builtin.Workspace{Dir: dir}).Tools("edit_file") { |
| 141 | reg.Add(tl) |
| 142 | } |
| 143 | // Verification would return "bash done" if it ran — the barrier must prevent that. |
| 144 | reg.Add(fakeTool{name: "bash", readOnly: false}) |
| 145 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 146 | { |
| 147 | toolCallChunk("e1", "edit_file", `{"path":"x.txt","old_string":"missing","new_string":"b"}`), |
| 148 | toolCallChunk("v1", "bash", `{"command":"go test ./..."}`), |
| 149 | {Type: provider.ChunkDone}, |
| 150 | }, |
| 151 | {{Type: provider.ChunkText, Text: "done"}, {Type: provider.ChunkDone}}, |
| 152 | }} |
| 153 | a := New(prov, reg, NewSession(""), Options{}, event.Discard) |
| 154 | if err := a.Run(context.Background(), "edit then verify"); err != nil { |
| 155 | t.Fatal(err) |
| 156 | } |
| 157 | if got := toolResultByID(a.session, "v1"); !strings.Contains(got, "earlier modification") { |
| 158 | t.Fatalf("verify result = %q, want dependency skip", got) |
| 159 | } |
| 160 | if strings.Contains(toolResultByID(a.session, "v1"), "bash done") { |
| 161 | t.Fatal("verification process should not have started") |
| 162 | } |
| 163 | for _, msg := range a.session.Snapshot() { |
| 164 | if msg.ToolCallID != "v1" { |
| 165 | continue |
| 166 | } |
| 167 | if msg.ToolExecution == nil { |
| 168 | t.Fatal("missing execution metadata on skipped verify") |
| 169 | } |
| 170 | if msg.ToolExecution.State != tool.ShellStateNotRun || msg.ToolExecution.FailurePhase != tool.ShellPhaseDependency { |
| 171 | t.Fatalf("execution = %+v", msg.ToolExecution) |
| 172 | } |
| 173 | if msg.ToolExecution.Verification != tool.ShellVerificationNotRun { |
| 174 | t.Fatalf("verification = %q, want not_run (not failed)", msg.ToolExecution.Verification) |
| 175 | } |
| 176 | return |
| 177 | } |
| 178 | t.Fatal("verify tool result missing") |
| 179 | } |
| 180 | |
| 181 | // TestBatchDependencyBarrierIgnoresFailedNonMutationMetaTool keeps bookkeeping |
| 182 | // writers out of the barrier. todo_write, complete_step, ask, bash_output and |
| 183 | // wait all report ReadOnly()==false, but evidence.ToolCallMutates deliberately |
| 184 | // exempts them: they never touch workspace state. A failed todo update must not |
| 185 | // block the real edits queued behind it in the same batch. |
| 186 | func TestBatchDependencyBarrierIgnoresFailedNonMutationMetaTool(t *testing.T) { |
| 187 | dir := t.TempDir() |
| 188 | path := filepath.Join(dir, "x.txt") |
| 189 | if err := os.WriteFile(path, []byte("a\n"), 0o600); err != nil { |
| 190 | t.Fatal(err) |
| 191 | } |
| 192 | reg := tool.NewRegistry() |
| 193 | for _, tl := range (builtin.Workspace{Dir: dir}).Tools("edit_file") { |
| 194 | reg.Add(tl) |
| 195 | } |
| 196 | reg.Add(fakeTool{name: "todo_write", readOnly: false, err: fmt.Errorf("todo store unavailable")}) |
| 197 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 198 | { |
| 199 | toolCallChunk("t1", "todo_write", `{"todos":[]}`), |
| 200 | toolCallChunk("e1", "edit_file", `{"path":"x.txt","old_string":"a","new_string":"b"}`), |
| 201 | {Type: provider.ChunkDone}, |
| 202 | }, |
| 203 | {{Type: provider.ChunkText, Text: "done"}, {Type: provider.ChunkDone}}, |
| 204 | }} |
| 205 | a := New(prov, reg, NewSession(""), Options{}, event.Discard) |
| 206 | if err := a.Run(context.Background(), "track then edit"); err != nil { |
| 207 | t.Fatal(err) |
| 208 | } |
| 209 | if got := toolResultByID(a.session, "e1"); strings.Contains(got, "earlier modification") { |
| 210 | t.Fatalf("edit was blocked by a failed todo_write: %s", got) |
| 211 | } |
| 212 | got, err := os.ReadFile(path) |
| 213 | if err != nil { |
| 214 | t.Fatal(err) |
| 215 | } |
| 216 | if string(got) != "b\n" { |
| 217 | t.Fatalf("file = %q, want the edit to have been applied", string(got)) |
| 218 | } |
| 219 | } |
| 220 | |
| 221 | // TestBatchDependencyBarrierStopsAfterFailedWorkspaceWrite is the other half of |
| 222 | // the same boundary: a genuine workspace mutation failing still stops the batch. |
| 223 | func TestBatchDependencyBarrierStopsAfterFailedWorkspaceWrite(t *testing.T) { |
| 224 | dir := t.TempDir() |
| 225 | if err := os.WriteFile(filepath.Join(dir, "x.txt"), []byte("a\n"), 0o600); err != nil { |
| 226 | t.Fatal(err) |
| 227 | } |
| 228 | reg := tool.NewRegistry() |
| 229 | for _, tl := range (builtin.Workspace{Dir: dir}).Tools("edit_file") { |
| 230 | reg.Add(tl) |
| 231 | } |
| 232 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 233 | { |
| 234 | toolCallChunk("e1", "edit_file", `{"path":"x.txt","old_string":"missing","new_string":"b"}`), |
| 235 | toolCallChunk("e2", "edit_file", `{"path":"x.txt","old_string":"a","new_string":"c"}`), |
| 236 | {Type: provider.ChunkDone}, |
| 237 | }, |
| 238 | {{Type: provider.ChunkText, Text: "done"}, {Type: provider.ChunkDone}}, |
| 239 | }} |
| 240 | a := New(prov, reg, NewSession(""), Options{}, event.Discard) |
| 241 | if err := a.Run(context.Background(), "two edits"); err != nil { |
| 242 | t.Fatal(err) |
| 243 | } |
| 244 | if got := toolResultByID(a.session, "e2"); !strings.Contains(got, "earlier modification") { |
| 245 | t.Fatalf("second edit result = %q, want dependency skip", got) |
| 246 | } |
| 247 | got, err := os.ReadFile(filepath.Join(dir, "x.txt")) |
| 248 | if err != nil { |
| 249 | t.Fatal(err) |
| 250 | } |
| 251 | if string(got) != "a\n" { |
| 252 | t.Fatalf("file = %q, want it untouched after the barrier", string(got)) |
| 253 | } |
| 254 | } |
| 255 | |
| 256 | // writerProxy is a use_capability-shaped CallResolver: schema ReadOnly is true, |
| 257 | // but ResolveCall points at a real writer. The batch barrier must not let this |
| 258 | // run after an earlier mutation failed. |
| 259 | type writerProxy struct { |
| 260 | target tool.Tool |
| 261 | resolves *int |
| 262 | } |
| 263 | |
| 264 | func (writerProxy) Name() string { return "use_capability" } |
| 265 | func (writerProxy) Description() string { return "proxy" } |
| 266 | func (writerProxy) Schema() json.RawMessage { |
| 267 | return json.RawMessage(`{"type":"object","properties":{"action":{"type":"string"}}}`) |
| 268 | } |
| 269 | func (writerProxy) ReadOnly() bool { return true } |
| 270 | func (p writerProxy) Execute(context.Context, json.RawMessage) (string, error) { |
| 271 | return "", fmt.Errorf("proxy Execute must not run; ResolveCall provides the target") |
| 272 | } |
| 273 | func (p writerProxy) ResolveCall(_ context.Context, args json.RawMessage) (tool.ResolvedCall, error) { |
| 274 | if p.resolves != nil { |
| 275 | (*p.resolves)++ |
| 276 | } |
| 277 | return tool.ResolvedCall{ |
| 278 | DisplayName: "use_capability", |
| 279 | TargetName: p.target.Name(), |
| 280 | Args: args, |
| 281 | Target: p.target, |
| 282 | ReadOnly: false, |
| 283 | ProxyAction: "call", |
| 284 | CapabilityID: "mcp-tool:test/write", |
| 285 | }, nil |
| 286 | } |
| 287 | |
| 288 | type capturingWriter struct { |
| 289 | name string |
| 290 | path string |
| 291 | calls *int |
| 292 | } |
| 293 | |
| 294 | func (c *capturingWriter) Name() string { return c.name } |
| 295 | func (c *capturingWriter) Description() string { return "" } |
| 296 | func (c *capturingWriter) Schema() json.RawMessage { return json.RawMessage(`{"type":"object"}`) } |
| 297 | func (c *capturingWriter) ReadOnly() bool { return false } |
| 298 | func (c *capturingWriter) Execute(context.Context, json.RawMessage) (string, error) { |
| 299 | if c.calls != nil { |
| 300 | *c.calls++ |
| 301 | } |
| 302 | if c.path != "" { |
| 303 | _ = os.WriteFile(c.path, []byte("proxy-wrote\n"), 0o600) |
| 304 | } |
| 305 | return "wrote", nil |
| 306 | } |
| 307 | |
| 308 | func TestBatchDependencyBarrierBlocksResolvedMCPWriterAfterFailedMutation(t *testing.T) { |
| 309 | dir := t.TempDir() |
| 310 | path := filepath.Join(dir, "x.txt") |
| 311 | if err := os.WriteFile(path, []byte("a\n"), 0o600); err != nil { |
| 312 | t.Fatal(err) |
| 313 | } |
| 314 | proxyWrote := filepath.Join(dir, "proxy-out.txt") |
| 315 | var writerCalls int |
| 316 | var resolves int |
| 317 | writer := &capturingWriter{name: "mcp__test__write", path: proxyWrote, calls: &writerCalls} |
| 318 | reg := tool.NewRegistry() |
| 319 | for _, tl := range (builtin.Workspace{Dir: dir}).Tools("edit_file") { |
| 320 | reg.Add(tl) |
| 321 | } |
| 322 | reg.Add(writerProxy{target: writer, resolves: &resolves}) |
| 323 | reg.Add(writer) // real target available for ResolveCall |
| 324 | |
| 325 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 326 | { |
| 327 | toolCallChunk("e1", "edit_file", `{"path":"x.txt","old_string":"missing","new_string":"b"}`), |
| 328 | toolCallChunk("m1", "use_capability", `{"action":"call","capability_id":"mcp-tool:test/write"}`), |
| 329 | {Type: provider.ChunkDone}, |
| 330 | }, |
| 331 | {{Type: provider.ChunkText, Text: "done"}, {Type: provider.ChunkDone}}, |
| 332 | }} |
| 333 | a := New(prov, reg, NewSession(""), Options{}, event.Discard) |
| 334 | if err := a.Run(context.Background(), "fail then mcp write"); err != nil { |
| 335 | t.Fatal(err) |
| 336 | } |
| 337 | if writerCalls != 0 { |
| 338 | t.Fatalf("MCP writer Execute ran %d times; dependency barrier must block after failed edit", writerCalls) |
| 339 | } |
| 340 | if resolves != 1 { |
| 341 | t.Fatalf("proxy ResolveCall ran %d times, want exactly once before the dependency barrier", resolves) |
| 342 | } |
| 343 | if _, err := os.Stat(proxyWrote); err == nil { |
| 344 | t.Fatal("proxy writer mutated disk after failed edit") |
| 345 | } |
| 346 | got := toolResultByID(a.session, "m1") |
| 347 | if !strings.Contains(got, "earlier modification") { |
| 348 | t.Fatalf("proxy result = %q, want dependency skip", got) |
| 349 | } |
| 350 | } |
| 351 | |
| 352 | func TestBatchDependencyBarrierAllowsReadOnlyDiagnosisAfterFailedMutation(t *testing.T) { |
| 353 | // After a mutating failure, host-proven read-only diagnosis must still run. |
| 354 | // Only subsequent mutations and verification commands are skipped. |
| 355 | dir := t.TempDir() |
| 356 | path := filepath.Join(dir, "x.txt") |
| 357 | if err := os.WriteFile(path, []byte("a\n"), 0o600); err != nil { |
| 358 | t.Fatal(err) |
| 359 | } |
| 360 | reg := tool.NewRegistry() |
| 361 | for _, name := range []string{"edit_file", "read_file"} { |
| 362 | for _, tl := range (builtin.Workspace{Dir: dir}).Tools(name) { |
| 363 | reg.Add(tl) |
| 364 | } |
| 365 | } |
| 366 | reg.Add(fakeTool{name: "bash", readOnly: false}) |
| 367 | prov := &scriptedProvider{name: "p", turns: [][]provider.Chunk{ |
| 368 | { |
| 369 | toolCallChunk("e1", "edit_file", `{"path":"x.txt","old_string":"missing","new_string":"b"}`), |
| 370 | toolCallChunk("r1", "read_file", `{"path":"x.txt"}`), |
| 371 | toolCallChunk("v1", "bash", `{"command":"go test ./..."}`), |
| 372 | {Type: provider.ChunkDone}, |
| 373 | }, |
| 374 | {{Type: provider.ChunkText, Text: "done"}, {Type: provider.ChunkDone}}, |
| 375 | }} |
| 376 | a := New(prov, reg, NewSession(""), Options{}, event.Discard) |
| 377 | if err := a.Run(context.Background(), "fail then diagnose"); err != nil { |
| 378 | t.Fatal(err) |
| 379 | } |
| 380 | readOut := toolResultByID(a.session, "r1") |
| 381 | if strings.Contains(readOut, "earlier modification") { |
| 382 | t.Fatalf("read_file was incorrectly dependency-skipped: %q", readOut) |
| 383 | } |
| 384 | trimmed := strings.TrimSpace(readOut) |
| 385 | if strings.HasPrefix(trimmed, "error:") || strings.HasPrefix(trimmed, "blocked:") { |
| 386 | t.Fatalf("read_file should have executed successfully, got %q", readOut) |
| 387 | } |
| 388 | if !strings.Contains(readOut, "a") { |
| 389 | t.Fatalf("read_file body missing original file content: %q", readOut) |
| 390 | } |
| 391 | if got := toolResultByID(a.session, "v1"); !strings.Contains(got, "earlier modification") { |
| 392 | t.Fatalf("verification should be dependency-skipped, got %q", got) |
| 393 | } |
| 394 | if strings.Contains(toolResultByID(a.session, "v1"), "bash done") { |
| 395 | t.Fatal("verification process must not start after failed mutation") |
| 396 | } |
| 397 | } |
| 398 | |
| 399 | func TestModelMessagesStripsToolExecution(t *testing.T) { |
| 400 | code := 1 |
| 401 | in := []provider.Message{ |
| 402 | {Role: provider.RoleUser, Content: "hi"}, |
| 403 | {Role: provider.RoleAssistant, Content: "", ToolCalls: []provider.ToolCall{{ID: "c1", Name: "bash", Arguments: `{"command":"false"}`}}}, |
| 404 | {Role: provider.RoleTool, ToolCallID: "c1", Name: "bash", Content: "error", ToolExecution: &provider.ToolExecution{ |
| 405 | Kind: "shell", Shell: "bash", State: "failed", ExitCode: &code, FailurePhase: "execution", |
| 406 | }}, |
| 407 | } |
| 408 | out := provider.ModelMessages(in) |
| 409 | if len(out) != 3 { |
| 410 | t.Fatalf("len = %d", len(out)) |
| 411 | } |
| 412 | if out[2].ToolExecution != nil { |
| 413 | t.Fatalf("ToolExecution leaked into model messages: %+v", out[2].ToolExecution) |
| 414 | } |
| 415 | if in[2].ToolExecution == nil { |
| 416 | t.Fatal("session copy was mutated") |
| 417 | } |
| 418 | } |
| 419 |