| 1 | package agent |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "path/filepath" |
| 7 | "strconv" |
| 8 | "strings" |
| 9 | "sync/atomic" |
| 10 | "testing" |
| 11 | |
| 12 | "reasonix/internal/event" |
| 13 | "reasonix/internal/evidence" |
| 14 | "reasonix/internal/provider" |
| 15 | "reasonix/internal/runtimepolicy" |
| 16 | "reasonix/internal/tool" |
| 17 | ) |
| 18 | |
| 19 | func setTurnConstraints(a *Agent, raw string) { |
| 20 | c := runtimepolicy.ParseConstraints(runtimepolicy.StripQuotedConstraints(raw)) |
| 21 | a.turn.constraints = c |
| 22 | a.turn.engine = runtimepolicy.NewEngine(c) |
| 23 | } |
| 24 | |
| 25 | func TestExplicitFullVerificationRemainsModelInstruction(t *testing.T) { |
| 26 | a := New(nil, tool.NewRegistry(), NewSession(""), Options{}, event.Discard) |
| 27 | setTurnConstraints(a, "请完整验证并交付") |
| 28 | if result := a.ReadinessResult(); !result.Ready || len(result.Missing) != 0 { |
| 29 | t.Fatalf("natural language created host quality requirements: %+v", result) |
| 30 | } |
| 31 | } |
| 32 | |
| 33 | func TestTaskPolicyUsesStructuredCommandEffects(t *testing.T) { |
| 34 | var calls int32 |
| 35 | reg := tool.NewRegistry() |
| 36 | reg.Add(fakeTool{name: "bash", readOnly: false, calls: &calls}) |
| 37 | a := New(&scriptedProvider{name: "p"}, reg, NewSession("sys"), Options{}, event.Discard) |
| 38 | a.turn.constraints = runtimepolicy.Constraints{ForbidMutation: true} |
| 39 | a.turn.engine = runtimepolicy.NewEngine(a.turn.constraints) |
| 40 | |
| 41 | listing := a.executeOne(context.Background(), &a.turn, provider.ToolCall{Name: "bash", Arguments: `{"command":"git branch -a"}`}) |
| 42 | if listing.blocked || listing.errMsg != "" { |
| 43 | t.Fatalf("branch listing outcome = %+v, want execution", listing) |
| 44 | } |
| 45 | if got := atomic.LoadInt32(&calls); got != 1 { |
| 46 | t.Fatalf("branch listing Execute calls = %d, want 1", got) |
| 47 | } |
| 48 | |
| 49 | tests := []struct { |
| 50 | name string |
| 51 | command string |
| 52 | wantDomain string |
| 53 | secret string |
| 54 | }{ |
| 55 | {name: "tag creation", command: "git tag v1.2.3", wantDomain: "repository metadata", secret: "v1.2.3"}, |
| 56 | {name: "host clock", command: "date --set tomorrow", wantDomain: "host state", secret: "tomorrow"}, |
| 57 | {name: "audit fix", command: "npm audit fix", wantDomain: "workspace content", secret: "fix"}, |
| 58 | {name: "config edit", command: "git config --edit", wantDomain: "repository metadata", secret: "--edit"}, |
| 59 | } |
| 60 | for _, tt := range tests { |
| 61 | t.Run(tt.name, func(t *testing.T) { |
| 62 | args, err := json.Marshal(map[string]string{"command": tt.command}) |
| 63 | if err != nil { |
| 64 | t.Fatal(err) |
| 65 | } |
| 66 | got := a.executeOne(context.Background(), &a.turn, provider.ToolCall{Name: "bash", Arguments: string(args)}) |
| 67 | if !got.blocked || !strings.Contains(got.output, "forbid") { |
| 68 | t.Fatalf("command %q outcome = %+v, want mutation block", tt.command, got) |
| 69 | } |
| 70 | if strings.Contains(got.errMsg, tt.secret) && tt.secret != "--edit" { |
| 71 | t.Fatalf("policy error leaked command operand %q: %q", tt.secret, got.errMsg) |
| 72 | } |
| 73 | }) |
| 74 | } |
| 75 | if got := atomic.LoadInt32(&calls); got != 1 { |
| 76 | t.Fatalf("blocked writers reached Execute: calls=%d, want 1", got) |
| 77 | } |
| 78 | } |
| 79 | |
| 80 | func TestTaskPolicyEnforcesVerificationAllowlist(t *testing.T) { |
| 81 | var calls int32 |
| 82 | reg := tool.NewRegistry() |
| 83 | reg.Add(fakeTool{name: "bash", readOnly: true, calls: &calls}) |
| 84 | a := New(&scriptedProvider{name: "p"}, reg, NewSession("sys"), Options{}, event.Discard) |
| 85 | setTurnConstraints(a, "fix it; only run go test ./internal/parser") |
| 86 | |
| 87 | for _, command := range []string{"npm test", "go vet ./...", "golangci-lint run", "npm run typecheck"} { |
| 88 | blocked := a.executeOne(context.Background(), &a.turn, provider.ToolCall{ |
| 89 | Name: "bash", Arguments: `{"command":` + strconv.Quote(command) + `}`, |
| 90 | }) |
| 91 | if !blocked.blocked || !strings.Contains(blocked.errMsg, "allowlist") { |
| 92 | t.Fatalf("%s outcome = %+v, want allowlist block", command, blocked) |
| 93 | } |
| 94 | } |
| 95 | if got := atomic.LoadInt32(&calls); got != 0 { |
| 96 | t.Fatalf("disallowed verification commands executed %d times, want 0", got) |
| 97 | } |
| 98 | allowed := a.executeOne(context.Background(), &a.turn, provider.ToolCall{Name: "bash", Arguments: `{"command":"go test ./internal/parser"}`}) |
| 99 | if allowed.blocked || allowed.errMsg != "" { |
| 100 | t.Fatalf("allowed go test outcome = %+v", allowed) |
| 101 | } |
| 102 | if got := atomic.LoadInt32(&calls); got != 1 { |
| 103 | t.Fatalf("allowed verification command executed %d times, want 1", got) |
| 104 | } |
| 105 | } |
| 106 | |
| 107 | func TestTaskPolicyForbidTestsBlocksEveryVerifier(t *testing.T) { |
| 108 | var calls int32 |
| 109 | reg := tool.NewRegistry() |
| 110 | reg.Add(fakeTool{name: "bash", readOnly: true, calls: &calls}) |
| 111 | a := New(&scriptedProvider{name: "p"}, reg, NewSession("sys"), Options{}, event.Discard) |
| 112 | setTurnConstraints(a, "fix it; don't run tests") |
| 113 | |
| 114 | for _, command := range []string{"go test ./...", "go vet ./...", "golangci-lint run", "npm run typecheck"} { |
| 115 | got := a.executeOne(context.Background(), &a.turn, provider.ToolCall{ |
| 116 | Name: "bash", Arguments: `{"command":` + strconv.Quote(command) + `}`, |
| 117 | }) |
| 118 | if !got.blocked || !strings.Contains(got.output, "forbid") { |
| 119 | t.Fatalf("%s outcome = %+v, want user-constraint block", command, got) |
| 120 | } |
| 121 | } |
| 122 | if got := atomic.LoadInt32(&calls); got != 0 { |
| 123 | t.Fatalf("forbidden verification commands executed %d times, want 0", got) |
| 124 | } |
| 125 | } |
| 126 | |
| 127 | func TestTaskPolicyBlocksExternalActionCommandVariants(t *testing.T) { |
| 128 | reg := tool.NewRegistry() |
| 129 | reg.Add(fakeTool{name: "bash", readOnly: false}) |
| 130 | a := New(&scriptedProvider{name: "p"}, reg, NewSession("sys"), Options{}, event.Discard) |
| 131 | setTurnConstraints(a, "fix it, but don't push") |
| 132 | a.setTodoState([]evidence.TodoItem{{Content: "fix it", Status: "in_progress"}}) |
| 133 | |
| 134 | for _, command := range []string{ |
| 135 | "git -C ../repo push origin HEAD", |
| 136 | "npm --workspace pkg publish", |
| 137 | "kubectl -n production apply -f deploy.yaml", |
| 138 | } { |
| 139 | args, err := json.Marshal(map[string]string{"command": command}) |
| 140 | if err != nil { |
| 141 | t.Fatal(err) |
| 142 | } |
| 143 | got := a.executeOne(context.Background(), &a.turn, provider.ToolCall{Name: "bash", Arguments: string(args)}) |
| 144 | if !got.blocked || !(strings.Contains(got.output, "external") || strings.Contains(got.output, "push") || strings.Contains(got.output, "publish") || strings.Contains(got.output, "deploy")) { |
| 145 | t.Fatalf("command %q outcome = %+v, want task-policy block", command, got) |
| 146 | } |
| 147 | } |
| 148 | } |
| 149 | |
| 150 | func TestTaskPolicyBlocksResolvedExternalCapability(t *testing.T) { |
| 151 | calls := 0 |
| 152 | target := readOnlyBoundaryTarget{name: "mcp__vercel__deploy_project", readOnly: false, calls: &calls} |
| 153 | proxy := readOnlyBoundaryProxy{resolved: tool.ResolvedCall{ |
| 154 | ProxyAction: "call", TargetName: target.Name(), Target: target, ReadOnly: false, Args: json.RawMessage(`{}`), |
| 155 | }} |
| 156 | reg := tool.NewRegistry() |
| 157 | reg.Add(proxy) |
| 158 | a := New(nil, reg, NewSession("sys"), Options{}, event.Discard) |
| 159 | setTurnConstraints(a, "prepare the release, but don't deploy") |
| 160 | |
| 161 | got := a.executeOne(context.Background(), &a.turn, provider.ToolCall{ |
| 162 | ID: "deploy-1", Name: "use_capability", Arguments: `{"action":"call","capability_id":"mcp-tool:vercel/deploy_project"}`, |
| 163 | }) |
| 164 | if !got.blocked || !strings.Contains(got.output, "deploy") { |
| 165 | t.Fatalf("resolved deploy outcome = %+v, want deploy block", got) |
| 166 | } |
| 167 | if calls != 0 { |
| 168 | t.Fatalf("resolved deploy Execute calls = %d, want 0", calls) |
| 169 | } |
| 170 | } |
| 171 | |
| 172 | func TestTaskPolicyReportsPostMutationVerificationGapWithoutBlockingTargetedTurn(t *testing.T) { |
| 173 | reg := tool.NewRegistry() |
| 174 | reg.Add(fakeTool{name: "bash", readOnly: true}) |
| 175 | writer := evidence.Receipt{ToolName: "write_file", Success: true, Write: true, Mutation: true, Paths: []string{"notes.txt"}} |
| 176 | check := evidence.Receipt{ToolName: "bash", Success: true, Command: "go test ./..."} |
| 177 | a := &Agent{ |
| 178 | task: taskRuntime{ledger: readinessLedger(check, writer)}, |
| 179 | svc: agentServices{tools: reg}, |
| 180 | turn: turnRuntime{engine: runtimepolicy.NewEngine(runtimepolicy.Constraints{})}, |
| 181 | } |
| 182 | if got := a.ReadinessResult(); got.Reason != "" { |
| 183 | t.Fatalf("targeted readiness = %+v, want quality gap to remain non-blocking", got) |
| 184 | } |
| 185 | a.task.ledger.Record(check) |
| 186 | if got := a.ReadinessResult(); got.Reason != "" { |
| 187 | t.Fatalf("readiness after verification = %+v, want ready", got) |
| 188 | } |
| 189 | } |
| 190 | |
| 191 | func TestPolicyEscalatesBeforeFirstSensitiveMutation(t *testing.T) { |
| 192 | var calls int32 |
| 193 | reg := tool.NewRegistry() |
| 194 | reg.Add(fakeTool{name: "edit_file", readOnly: false, calls: &calls}) |
| 195 | permission := &stubGate{deny: map[string]bool{}} |
| 196 | a := New(nil, reg, NewSession("sys"), Options{Gate: permission}, event.Discard) |
| 197 | setTurnConstraints(a, "fix the typo in README.md") |
| 198 | |
| 199 | got := a.executeOne(context.Background(), &a.turn, provider.ToolCall{ |
| 200 | Name: "edit_file", |
| 201 | Arguments: `{"path":"internal/auth/session.go","old_string":"old","new_string":"new"}`, |
| 202 | }) |
| 203 | if got.blocked || got.errMsg != "" || atomic.LoadInt32(&calls) != 1 || len(permission.checked) != 1 { |
| 204 | t.Fatalf("sensitive path must use ordinary permissioned execution: %+v calls=%d permission=%v", got, calls, permission.checked) |
| 205 | } |
| 206 | } |
| 207 | |
| 208 | func TestPolicyEscalatesDeepAbsoluteSensitiveMutationBeforeExecution(t *testing.T) { |
| 209 | var calls int32 |
| 210 | root := t.TempDir() |
| 211 | reg := tool.NewRegistry() |
| 212 | reg.Add(fakeTool{name: "edit_file", readOnly: false, calls: &calls}) |
| 213 | permission := &stubGate{deny: map[string]bool{}} |
| 214 | a := New(nil, reg, NewSession("sys"), Options{Gate: permission, WriteWorkspaceRoot: root}, event.Discard) |
| 215 | setTurnConstraints(a, "fix this file") |
| 216 | args, err := json.Marshal(map[string]string{ |
| 217 | "path": filepath.Join(root, "internal", "provider", "openai", "responses", "client.go"), |
| 218 | "old_string": "old", |
| 219 | "new_string": "new", |
| 220 | }) |
| 221 | if err != nil { |
| 222 | t.Fatal(err) |
| 223 | } |
| 224 | |
| 225 | got := a.executeOne(context.Background(), &a.turn, provider.ToolCall{Name: "edit_file", Arguments: string(args)}) |
| 226 | if got.blocked { |
| 227 | t.Fatalf("ordinary production file must not be pre-classified as schema/auth: %+v", got) |
| 228 | } |
| 229 | } |
| 230 | |
| 231 | func TestPlannedLowRiskMutationKeepsOrdinaryPath(t *testing.T) { |
| 232 | var calls int32 |
| 233 | reg := tool.NewRegistry() |
| 234 | reg.Add(fakeTool{name: "edit_file", readOnly: false, calls: &calls}) |
| 235 | permission := &stubGate{deny: map[string]bool{}} |
| 236 | a := New(nil, reg, NewSession("sys"), Options{Gate: permission}, event.Discard) |
| 237 | setTurnConstraints(a, "fix the typo in README.md") |
| 238 | |
| 239 | got := a.executeOne(context.Background(), &a.turn, provider.ToolCall{ |
| 240 | Name: "edit_file", |
| 241 | Arguments: `{"path":"README.md","old_string":"teh","new_string":"the"}`, |
| 242 | }) |
| 243 | if got.blocked || got.errMsg != "" { |
| 244 | t.Fatalf("low-risk mutation outcome = %+v, want ordinary execution", got) |
| 245 | } |
| 246 | if got := atomic.LoadInt32(&calls); got != 1 { |
| 247 | t.Fatalf("low-risk writer calls = %d, want 1", got) |
| 248 | } |
| 249 | if len(permission.checked) != 1 { |
| 250 | t.Fatalf("permission checks = %v, want one ordinary check", permission.checked) |
| 251 | } |
| 252 | if a.closedLoopActive() { |
| 253 | t.Fatal("README typo must not create a closed-loop contract") |
| 254 | } |
| 255 | } |
| 256 |