| 1 | package evidence |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "reflect" |
| 7 | "strings" |
| 8 | "testing" |
| 9 | ) |
| 10 | |
| 11 | func TestLedgerRecordsSuccessAndFailureReceipts(t *testing.T) { |
| 12 | ledger := NewLedger() |
| 13 | ledger.Record(Receipt{ |
| 14 | ToolName: "bash", |
| 15 | Args: json.RawMessage(`{"command":"go test ./..."}`), |
| 16 | Success: true, |
| 17 | Command: "go test ./...", |
| 18 | }) |
| 19 | ledger.Record(Receipt{ |
| 20 | ToolName: "bash", |
| 21 | Args: json.RawMessage(`{"command":"go test ./internal/..."}`), |
| 22 | Success: false, |
| 23 | Command: "go test ./internal/...", |
| 24 | }) |
| 25 | |
| 26 | if !ledger.HasSuccessfulCommand("go test ./...") { |
| 27 | t.Fatal("successful bash command should verify") |
| 28 | } |
| 29 | if ledger.HasSuccessfulCommand("go test ./internal/...") { |
| 30 | t.Fatal("failed bash command must not verify") |
| 31 | } |
| 32 | } |
| 33 | |
| 34 | func TestLedgerDistinguishesWorkflowSpecificMutation(t *testing.T) { |
| 35 | ledger := NewLedger() |
| 36 | ledger.Record(ReceiptFromToolCall("remember", json.RawMessage(`{"body":"ORBIT-42"}`), true, false)) |
| 37 | if !ledger.HasSuccessfulToolReceipt("remember") { |
| 38 | t.Fatal("successful remember receipt was not found") |
| 39 | } |
| 40 | if ledger.HasSuccessfulMutationOtherThan("remember") { |
| 41 | t.Fatal("remember-only mutation was treated as an unrelated workspace mutation") |
| 42 | } |
| 43 | ledger.Record(ReceiptFromToolCall("edit_file", json.RawMessage(`{"path":"internal/a.go"}`), true, false)) |
| 44 | if !ledger.HasSuccessfulMutationOtherThan("remember") { |
| 45 | t.Fatal("workspace mutation was hidden by the remember allowance") |
| 46 | } |
| 47 | } |
| 48 | |
| 49 | func TestLedgerMatchesFreshSuccessfulReviewReceipt(t *testing.T) { |
| 50 | ledger := NewLedger() |
| 51 | ledger.Record(ReceiptFromToolCall("review", json.RawMessage(`{"task":"review changes"}`), true, true)) |
| 52 | if !ledger.HasCompletedReview() { |
| 53 | t.Fatal("successful dedicated review tool should verify review evidence") |
| 54 | } |
| 55 | legacy := NewLedger() |
| 56 | legacy.Record(ReceiptFromToolCall("task", json.RawMessage(`{"profile":"review"}`), true, true)) |
| 57 | if !legacy.HasCompletedReview() { |
| 58 | t.Fatal("successful task(profile=review) should verify review evidence") |
| 59 | } |
| 60 | failed := NewLedger() |
| 61 | failed.Record(ReceiptFromToolCall("task", json.RawMessage(`{"profile":"review"}`), false, true)) |
| 62 | if failed.HasCompletedReview() { |
| 63 | t.Fatal("failed review task must not verify review evidence") |
| 64 | } |
| 65 | background := NewLedger() |
| 66 | background.Record(ReceiptFromToolCall("task", json.RawMessage(`{"profile":"review","run_in_background":true}`), true, true)) |
| 67 | if background.HasCompletedReview() { |
| 68 | t.Fatal("starting a background review must not count as a completed review") |
| 69 | } |
| 70 | } |
| 71 | |
| 72 | func TestLedgerRejectsReviewBeforeLatestMutation(t *testing.T) { |
| 73 | ledger := NewLedger() |
| 74 | ledger.Record(ReceiptFromToolCall("review", json.RawMessage(`{"task":"review changes"}`), true, true)) |
| 75 | ledger.Record(ReceiptFromToolCall("edit_file", json.RawMessage(`{"path":"changed.go"}`), true, false)) |
| 76 | if ledger.HasCompletedReview() { |
| 77 | t.Fatal("review evidence before the latest mutation must be stale") |
| 78 | } |
| 79 | } |
| 80 | |
| 81 | func TestLedgerRequiresReviewCoverageAfterMutation(t *testing.T) { |
| 82 | fresh := NewLedger() |
| 83 | fresh.Record(ReceiptFromToolCall("edit_file", json.RawMessage(`{"path":"changed.go"}`), true, false)) |
| 84 | fresh.Record(ReceiptFromToolCall("read_file", json.RawMessage(`{"path":"changed.go"}`), true, true)) |
| 85 | fresh.Record(ReceiptFromToolCall("review", json.RawMessage(`{"task":"review changes"}`), true, true)) |
| 86 | if !fresh.HasCompletedReview() { |
| 87 | t.Fatal("review after reading the latest changed path should count") |
| 88 | } |
| 89 | |
| 90 | unrelated := NewLedger() |
| 91 | unrelated.Record(ReceiptFromToolCall("edit_file", json.RawMessage(`{"path":"changed.go"}`), true, false)) |
| 92 | unrelated.Record(ReceiptFromToolCall("read_file", json.RawMessage(`{"path":"other.go"}`), true, true)) |
| 93 | unrelated.Record(ReceiptFromToolCall("review", json.RawMessage(`{"task":"review something else"}`), true, true)) |
| 94 | if unrelated.HasCompletedReview() { |
| 95 | t.Fatal("review that did not inspect the latest changed path must not count") |
| 96 | } |
| 97 | } |
| 98 | |
| 99 | func TestLedgerHostReviewCoverageRequiresContentForEveryPath(t *testing.T) { |
| 100 | ledger := NewLedger() |
| 101 | ledger.Record(ReceiptFromToolCall("edit_file", json.RawMessage(`{"path":"internal/a.go"}`), true, false)) |
| 102 | ledger.Record(ReceiptFromToolCall("edit_file", json.RawMessage(`{"path":"internal/b.go"}`), true, false)) |
| 103 | mutation, ok := ledger.LatestSuccessfulMutationIndex() |
| 104 | if !ok { |
| 105 | t.Fatal("missing mutation index") |
| 106 | } |
| 107 | ledger.Record(ReceiptFromToolCall("read_file", json.RawMessage(`{"path":"internal/a.go"}`), true, true)) |
| 108 | if ledger.HasHostReviewCoverageAfter(mutation, []string{"internal/a.go", "internal/b.go"}) { |
| 109 | t.Fatal("one path read must not cover a two-path change set") |
| 110 | } |
| 111 | ledger.Record(ReceiptFromToolCall("read_file", json.RawMessage(`{"path":"internal/b.go"}`), true, true)) |
| 112 | if !ledger.HasHostReviewCoverageAfter(mutation, []string{"internal/a.go", "internal/b.go"}) { |
| 113 | t.Fatal("fresh reads of every changed path should prove host review coverage") |
| 114 | } |
| 115 | |
| 116 | diff := NewLedger() |
| 117 | diff.Record(ReceiptFromToolCall("edit_file", json.RawMessage(`{"path":"internal/a.go"}`), true, false)) |
| 118 | diffMutation, ok := diff.LatestSuccessfulMutationIndex() |
| 119 | if !ok { |
| 120 | t.Fatal("missing diff mutation index") |
| 121 | } |
| 122 | diff.Record(Receipt{ToolName: "bash", Success: true, Command: "git diff", OutputBytes: 200}) |
| 123 | if !diff.HasHostReviewCoverageAfter(diffMutation, []string{"internal/a.go", "internal/b.go"}) { |
| 124 | t.Fatal("an output-producing whole git diff should cover the current change set") |
| 125 | } |
| 126 | for _, command := range []string{"git status --short", "git diff --check"} { |
| 127 | insufficient := NewLedger() |
| 128 | insufficient.Record(ReceiptFromToolCall("edit_file", json.RawMessage(`{"path":"internal/a.go"}`), true, false)) |
| 129 | idx, ok := insufficient.LatestSuccessfulMutationIndex() |
| 130 | if !ok { |
| 131 | t.Fatal("missing insufficient mutation index") |
| 132 | } |
| 133 | insufficient.Record(Receipt{ToolName: "bash", Success: true, Command: command, OutputBytes: 200}) |
| 134 | if insufficient.HasHostReviewCoverageAfter(idx, []string{"internal/a.go"}) { |
| 135 | t.Fatalf("%q must not count as content review", command) |
| 136 | } |
| 137 | } |
| 138 | } |
| 139 | |
| 140 | func TestLedgerAcceptsCollectedStructuredReviewReport(t *testing.T) { |
| 141 | ledger := NewLedger() |
| 142 | ledger.Record(ReceiptFromToolCall("edit_file", json.RawMessage(`{"path":"changed.go"}`), true, false)) |
| 143 | ledger.Record(Receipt{ToolName: "review_report", Success: true, Args: json.RawMessage(`{ |
| 144 | "kind":"review", |
| 145 | "verdict":"block", |
| 146 | "reviewed_paths":["changed.go"], |
| 147 | "findings":[{"severity":"critical","summary":"must fix","path":"changed.go"}] |
| 148 | }`)}) |
| 149 | if !ledger.HasCompletedReview() { |
| 150 | t.Fatal("a structured report completes the review activity even when its verdict requires follow-up") |
| 151 | } |
| 152 | } |
| 153 | |
| 154 | func TestLedgerHasWriteOrCommandSince(t *testing.T) { |
| 155 | ledger := NewLedger() |
| 156 | ledger.Record(Receipt{ToolName: "todo_write", Success: true, Todos: []TodoItem{{Content: "edit", Status: "in_progress"}}}) |
| 157 | ledger.Record(Receipt{ToolName: "write_file", Success: true, Write: true, Paths: []string{"a.go"}}) |
| 158 | ledger.Record(Receipt{ToolName: "bash", Success: false, Command: "go test ./..."}) |
| 159 | ledger.Record(Receipt{ToolName: "complete_step", Success: true, Step: "edit"}) |
| 160 | |
| 161 | if got := ledger.Len(); got != 4 { |
| 162 | t.Fatalf("Len() = %d, want 4", got) |
| 163 | } |
| 164 | if !ledger.HasWriteOrCommandSince(0) { |
| 165 | t.Fatal("write receipt at index 1 should count from index 0") |
| 166 | } |
| 167 | if !ledger.HasWriteOrCommandSince(-1) { |
| 168 | t.Fatal("negative index should behave like 0") |
| 169 | } |
| 170 | if ledger.HasWriteOrCommandSince(2) { |
| 171 | t.Fatal("failed command and bookkeeping receipts must not count as progress") |
| 172 | } |
| 173 | ledger.Record(Receipt{ToolName: "bash", Success: true, Command: "go test ./..."}) |
| 174 | if !ledger.HasWriteOrCommandSince(2) { |
| 175 | t.Fatal("successful command receipt after index should count as progress") |
| 176 | } |
| 177 | var nilLedger *Ledger |
| 178 | if nilLedger.HasWriteOrCommandSince(0) { |
| 179 | t.Fatal("nil ledger must report no progress") |
| 180 | } |
| 181 | } |
| 182 | |
| 183 | func TestLedgerMatchesFileReadAndWriteReceipts(t *testing.T) { |
| 184 | ledger := NewLedger() |
| 185 | ledger.Record(Receipt{ToolName: "read_file", Success: true, Paths: []string{`internal/tool/builtin/completestep.go`}, Read: true}) |
| 186 | ledger.Record(Receipt{ToolName: "write_file", Success: true, Paths: []string{`internal/evidence/evidence.go`}, Write: true}) |
| 187 | ledger.Record(Receipt{ToolName: "edit_file", Success: false, Paths: []string{`failed.go`}, Write: true}) |
| 188 | |
| 189 | if !ledger.HasSuccessfulReadOrWrite([]string{`internal\tool\builtin\completestep.go`}) { |
| 190 | t.Fatal("read receipt should verify the same path across separators") |
| 191 | } |
| 192 | if !ledger.HasSuccessfulWrite([]string{`internal/evidence/evidence.go`}) { |
| 193 | t.Fatal("write receipt should verify written path") |
| 194 | } |
| 195 | if ledger.HasSuccessfulWrite([]string{`failed.go`}) { |
| 196 | t.Fatal("failed write receipt must not verify") |
| 197 | } |
| 198 | } |
| 199 | |
| 200 | func TestLedgerReportsFinalReadinessReceiptsAfterWriter(t *testing.T) { |
| 201 | ledger := NewLedger() |
| 202 | ledger.Record(Receipt{ToolName: "bash", Success: true, Command: "go test ./..."}) |
| 203 | ledger.Record(Receipt{ToolName: "write_file", Success: true, Paths: []string{"changed.go"}, Write: true}) |
| 204 | ledger.Record(Receipt{ToolName: "bash", Success: false, Command: "git diff --check"}) |
| 205 | ledger.Record(Receipt{ToolName: "todo_write", Success: true, Todos: []TodoItem{{Content: "Edit code", Status: "in_progress"}}}) |
| 206 | |
| 207 | writer, ok := ledger.LatestSuccessfulWriterIndex() |
| 208 | if !ok { |
| 209 | t.Fatal("expected latest successful writer") |
| 210 | } |
| 211 | if ledger.HasSuccessfulCommandAfter("go test ./...", writer) { |
| 212 | t.Fatal("command before latest writer must not satisfy final readiness") |
| 213 | } |
| 214 | if ledger.HasSuccessfulCommandAfter("git diff --check", writer) { |
| 215 | t.Fatal("failed command must not satisfy final readiness") |
| 216 | } |
| 217 | if ledger.HasSuccessfulCompleteStepAfter(writer) { |
| 218 | t.Fatal("missing complete_step must not satisfy final readiness") |
| 219 | } |
| 220 | if !ledger.HasSuccessfulTodoWrite() { |
| 221 | t.Fatal("successful todo_write receipt should be reported") |
| 222 | } |
| 223 | |
| 224 | ledger.Record(Receipt{ToolName: "bash", Success: true, Command: "git diff --check"}) |
| 225 | ledger.Record(Receipt{ToolName: "complete_step", Success: true, Step: "Edit code"}) |
| 226 | if !ledger.HasSuccessfulCommandAfter("git diff --check", writer) { |
| 227 | t.Fatal("command after latest writer should satisfy final readiness") |
| 228 | } |
| 229 | if !ledger.HasSuccessfulCompleteStepAfter(writer) { |
| 230 | t.Fatal("complete_step after latest writer should satisfy final readiness") |
| 231 | } |
| 232 | } |
| 233 | |
| 234 | func TestLedgerResetClearsTurnReceipts(t *testing.T) { |
| 235 | ledger := NewLedger() |
| 236 | ledger.Record(Receipt{ToolName: "bash", Success: true, Command: "go test ./..."}) |
| 237 | |
| 238 | ledger.Reset() |
| 239 | |
| 240 | if ledger.HasSuccessfulCommand("go test ./...") { |
| 241 | t.Fatal("reset should clear prior-turn evidence") |
| 242 | } |
| 243 | } |
| 244 | |
| 245 | func TestContextCarriesLedger(t *testing.T) { |
| 246 | ledger := NewLedger() |
| 247 | ctx := WithLedger(context.Background(), ledger) |
| 248 | |
| 249 | got, ok := FromContext(ctx) |
| 250 | if !ok { |
| 251 | t.Fatal("ledger missing from context") |
| 252 | } |
| 253 | if got != ledger { |
| 254 | t.Fatal("context returned a different ledger") |
| 255 | } |
| 256 | } |
| 257 | |
| 258 | func TestReceiptFromToolCallExtractsEvidenceFields(t *testing.T) { |
| 259 | bash := ReceiptFromToolCall("bash", json.RawMessage(`{"command":"git diff --check"}`), true, false) |
| 260 | if bash.Command != "git diff --check" { |
| 261 | t.Fatalf("bash command = %q", bash.Command) |
| 262 | } |
| 263 | if bash.Write { |
| 264 | t.Fatal("bash should not be treated as a verified file writer") |
| 265 | } |
| 266 | |
| 267 | write := ReceiptFromToolCall("write_file", json.RawMessage(`{"path":"internal/evidence/evidence.go","content":"x"}`), true, false) |
| 268 | if !write.Write || len(write.Paths) != 1 || write.Paths[0] != `internal/evidence/evidence.go` { |
| 269 | t.Fatalf("write receipt not extracted: %+v", write) |
| 270 | } |
| 271 | |
| 272 | read := ReceiptFromToolCall("read_file", json.RawMessage(`{"path":"internal/tool/builtin/completestep.go"}`), true, true) |
| 273 | if !read.Read || len(read.Paths) != 1 { |
| 274 | t.Fatalf("read receipt not extracted: %+v", read) |
| 275 | } |
| 276 | |
| 277 | glob := ReceiptFromToolCall("glob", json.RawMessage(`{"pattern":"**/*.go"}`), true, true) |
| 278 | if !glob.Read { |
| 279 | t.Fatalf("generic read-only tool should be treated as read context: %+v", glob) |
| 280 | } |
| 281 | } |
| 282 | |
| 283 | func TestReceiptFromToolCallExtractsTodoWriteItems(t *testing.T) { |
| 284 | receipt := ReceiptFromToolCall("todo_write", json.RawMessage(`{"todos":[ |
| 285 | {"content":"Add parser","status":"in_progress","activeForm":"Adding parser"}, |
| 286 | {"content":"Wire parser","status":"pending","level":1} |
| 287 | ]}`), true, true) |
| 288 | |
| 289 | if len(receipt.Todos) != 2 { |
| 290 | t.Fatalf("todos not extracted: %+v", receipt) |
| 291 | } |
| 292 | if receipt.Todos[0].Content != "Add parser" || receipt.Todos[0].Status != "in_progress" || receipt.Todos[0].ActiveForm != "Adding parser" { |
| 293 | t.Fatalf("first todo not extracted: %+v", receipt.Todos[0]) |
| 294 | } |
| 295 | if receipt.Todos[1].Level != 1 { |
| 296 | t.Fatalf("todo level not extracted: %+v", receipt.Todos[1]) |
| 297 | } |
| 298 | } |
| 299 | |
| 300 | func TestReceiptFromToolCallExtractsCompleteStep(t *testing.T) { |
| 301 | receipt := ReceiptFromToolCall("complete_step", json.RawMessage(`{ |
| 302 | "step":"Add parser", |
| 303 | "result":"parser added", |
| 304 | "evidence":[{"kind":"manual","summary":"checked manually"}] |
| 305 | }`), true, true) |
| 306 | |
| 307 | if receipt.Step != "Add parser" { |
| 308 | t.Fatalf("complete_step step = %q", receipt.Step) |
| 309 | } |
| 310 | if !receipt.StepProof { |
| 311 | t.Fatalf("complete_step evidence proof not extracted: %+v", receipt) |
| 312 | } |
| 313 | if receipt.Read { |
| 314 | t.Fatalf("complete_step should not be treated as read-only context: %+v", receipt) |
| 315 | } |
| 316 | |
| 317 | missingResult := ReceiptFromToolCall("complete_step", json.RawMessage(`{ |
| 318 | "step":"Add parser", |
| 319 | "evidence":[{"kind":"manual","summary":"checked manually"}] |
| 320 | }`), false, true) |
| 321 | if missingResult.StepProof { |
| 322 | t.Fatalf("complete_step without result should not count as proof: %+v", missingResult) |
| 323 | } |
| 324 | |
| 325 | missingCommand := ReceiptFromToolCall("complete_step", json.RawMessage(`{ |
| 326 | "step":"Add parser", |
| 327 | "result":"parser added", |
| 328 | "evidence":[{"kind":"verification","summary":"checked manually"}] |
| 329 | }`), false, true) |
| 330 | if missingCommand.StepProof { |
| 331 | t.Fatalf("verification evidence without command should not count as proof: %+v", missingCommand) |
| 332 | } |
| 333 | |
| 334 | emptyProof := ReceiptFromToolCall("complete_step", json.RawMessage(`{ |
| 335 | "step":"Add parser", |
| 336 | "result":"parser added", |
| 337 | "evidence":[] |
| 338 | }`), false, true) |
| 339 | if emptyProof.StepProof { |
| 340 | t.Fatalf("empty complete_step evidence should not count as proof: %+v", emptyProof) |
| 341 | } |
| 342 | } |
| 343 | |
| 344 | func TestReceiptFromToolCallExtractsCompleteStepIndex(t *testing.T) { |
| 345 | receipt := ReceiptFromToolCall("complete_step", json.RawMessage(`{ |
| 346 | "step_index":2, |
| 347 | "result":"done", |
| 348 | "evidence":[{"kind":"manual","summary":"checked"}] |
| 349 | }`), true, true) |
| 350 | |
| 351 | if receipt.Step != "2" { |
| 352 | t.Fatalf("step index not extracted as step identity: %+v", receipt) |
| 353 | } |
| 354 | if !receipt.StepProof { |
| 355 | t.Fatalf("step proof not detected: %+v", receipt) |
| 356 | } |
| 357 | } |
| 358 | |
| 359 | func TestLedgerMatchesLatestSuccessfulTodoStep(t *testing.T) { |
| 360 | ledger := NewLedger() |
| 361 | ledger.Record(Receipt{ |
| 362 | ToolName: "todo_write", |
| 363 | Success: false, |
| 364 | Todos: []TodoItem{{Content: "Failed only", Status: "in_progress"}}, |
| 365 | }) |
| 366 | ledger.Record(Receipt{ |
| 367 | ToolName: "todo_write", |
| 368 | Success: true, |
| 369 | Todos: []TodoItem{ |
| 370 | {Content: "Add parser", Status: "in_progress", ActiveForm: "Adding parser"}, |
| 371 | {Content: "Wire parser", Status: "completed"}, |
| 372 | {Content: "Document parser", Status: "pending"}, |
| 373 | }, |
| 374 | }) |
| 375 | |
| 376 | for _, step := range []string{"Add parser", "Adding parser", "2"} { |
| 377 | match, ok := ledger.MatchLatestTodoStep(step) |
| 378 | if !ok { |
| 379 | t.Fatalf("latest todo receipt missing for %q", step) |
| 380 | } |
| 381 | if !match.Found { |
| 382 | t.Fatalf("step %q did not match latest todo list", step) |
| 383 | } |
| 384 | if step == "2" && match.Content != "Wire parser" { |
| 385 | t.Fatalf("numeric step matched %q, want Wire parser", match.Content) |
| 386 | } |
| 387 | } |
| 388 | |
| 389 | match, ok := ledger.MatchLatestTodoStep("Failed only") |
| 390 | if !ok { |
| 391 | t.Fatal("successful todo receipt should exist") |
| 392 | } |
| 393 | if match.Found { |
| 394 | t.Fatal("failed todo_write receipt must not match") |
| 395 | } |
| 396 | } |
| 397 | |
| 398 | func TestMatchTodoStepToleratesCitationDrift(t *testing.T) { |
| 399 | // Verbatim shape from discussion #3970: todo authored with a fullwidth |
| 400 | // colon, cited back with a halfwidth one — and stuck forever. |
| 401 | ledger := NewLedger() |
| 402 | ledger.Record(Receipt{ |
| 403 | ToolName: "todo_write", |
| 404 | Success: true, |
| 405 | Todos: []TodoItem{ |
| 406 | {Content: "Phase 4：环境准备", Status: "completed"}, |
| 407 | {Content: "Phase 5：脚本编辑与执行代码", Status: "in_progress"}, |
| 408 | {Content: "Review notes", Status: "pending"}, |
| 409 | }, |
| 410 | }) |
| 411 | |
| 412 | matches := map[string]int{ |
| 413 | "Phase 5: 脚本编辑与执行代码": 2, |
| 414 | "phase 5：脚本编辑与执行代码": 2, |
| 415 | " Phase 5：脚本编辑与执行代码": 2, |
| 416 | "脚本编辑与执行代码": 2, |
| 417 | "Phase 4：环境": 1, |
| 418 | "REVIEW NOTES": 3, |
| 419 | "２": 2, |
| 420 | } |
| 421 | for step, want := range matches { |
| 422 | match, ok := ledger.MatchLatestTodoStep(step) |
| 423 | if !ok || !match.Found { |
| 424 | t.Fatalf("step %q should match todo %d, got found=%v", step, want, match.Found) |
| 425 | } |
| 426 | if match.Index != want { |
| 427 | t.Errorf("step %q matched todo %d, want %d", step, match.Index, want) |
| 428 | } |
| 429 | } |
| 430 | |
| 431 | for _, step := range []string{"deploy backend", "代码", "Phase 9：不存在的阶段"} { |
| 432 | if match, _ := ledger.MatchLatestTodoStep(step); match.Found { |
| 433 | t.Errorf("step %q should not match, got todo %d (%q)", step, match.Index, match.Content) |
| 434 | } |
| 435 | } |
| 436 | } |
| 437 | |
| 438 | func TestMatchTodoStepAmbiguousContainmentStaysUnmatched(t *testing.T) { |
| 439 | ledger := NewLedger() |
| 440 | ledger.Record(Receipt{ |
| 441 | ToolName: "todo_write", |
| 442 | Success: true, |
| 443 | Todos: []TodoItem{ |
| 444 | {Content: "Deploy backend service", Status: "in_progress"}, |
| 445 | {Content: "Deploy backend worker", Status: "pending"}, |
| 446 | }, |
| 447 | }) |
| 448 | if match, _ := ledger.MatchLatestTodoStep("Deploy backend"); match.Found { |
| 449 | t.Fatalf("ambiguous citation should stay unmatched, got todo %d (%q)", match.Index, match.Content) |
| 450 | } |
| 451 | if match, _ := ledger.MatchLatestTodoStep("Deploy backend worker"); !match.Found || match.Index != 2 { |
| 452 | t.Fatal("exact citation must still resolve despite shared prefix") |
| 453 | } |
| 454 | } |
| 455 | |
| 456 | func TestLedgerRequiresCompleteStepForNewCompletedTodos(t *testing.T) { |
| 457 | ledger := NewLedger() |
| 458 | ledger.Record(Receipt{ |
| 459 | ToolName: "todo_write", |
| 460 | Success: true, |
| 461 | Todos: []TodoItem{ |
| 462 | {Content: "Add parser", Status: "in_progress"}, |
| 463 | {Content: "Already done", Status: "completed"}, |
| 464 | }, |
| 465 | }) |
| 466 | |
| 467 | current := []TodoItem{ |
| 468 | {Content: "Add parser", Status: "completed"}, |
| 469 | {Content: "Already done", Status: "completed"}, |
| 470 | } |
| 471 | missing, hasBaseline := ledger.UnverifiedCompletedTodos(current) |
| 472 | if !hasBaseline { |
| 473 | t.Fatal("expected prior todo_write baseline") |
| 474 | } |
| 475 | if len(missing) != 1 || missing[0].Content != "Add parser" { |
| 476 | t.Fatalf("missing = %+v, want only Add parser", missing) |
| 477 | } |
| 478 | |
| 479 | ledger.Record(Receipt{ToolName: "complete_step", Success: false, Step: "Add parser"}) |
| 480 | missing, hasBaseline = ledger.UnverifiedCompletedTodos(current) |
| 481 | if !hasBaseline { |
| 482 | t.Fatal("expected prior todo_write baseline after failed complete_step") |
| 483 | } |
| 484 | if len(missing) != 1 || missing[0].Content != "Add parser" { |
| 485 | t.Fatalf("failed complete_step without proof-bearing recovery should not authorize completion, missing = %+v", missing) |
| 486 | } |
| 487 | |
| 488 | ledger.Record(Receipt{ToolName: "complete_step", Success: true, Step: "Add parser"}) |
| 489 | missing, hasBaseline = ledger.UnverifiedCompletedTodos(current) |
| 490 | if !hasBaseline { |
| 491 | t.Fatal("expected prior todo_write baseline after successful complete_step") |
| 492 | } |
| 493 | if len(missing) != 0 { |
| 494 | t.Fatalf("successful complete_step should authorize completion, missing = %+v", missing) |
| 495 | } |
| 496 | } |
| 497 | |
| 498 | func TestLedgerMatchesCompletionByActiveFormAndNumber(t *testing.T) { |
| 499 | ledger := NewLedger() |
| 500 | ledger.Record(Receipt{ |
| 501 | ToolName: "todo_write", |
| 502 | Success: true, |
| 503 | Todos: []TodoItem{ |
| 504 | {Content: "Add parser", Status: "in_progress", ActiveForm: "Adding parser"}, |
| 505 | {Content: "Wire parser", Status: "in_progress"}, |
| 506 | }, |
| 507 | }) |
| 508 | |
| 509 | current := []TodoItem{ |
| 510 | {Content: "Add parser", Status: "completed", ActiveForm: "Adding parser"}, |
| 511 | {Content: "Wire parser", Status: "in_progress"}, |
| 512 | } |
| 513 | ledger.Record(Receipt{ToolName: "complete_step", Success: true, Step: "Adding parser"}) |
| 514 | missing, hasBaseline := ledger.UnverifiedCompletedTodos(current) |
| 515 | if !hasBaseline { |
| 516 | t.Fatal("expected prior todo_write baseline") |
| 517 | } |
| 518 | if len(missing) != 0 { |
| 519 | t.Fatalf("activeForm complete_step should authorize completion, missing = %+v", missing) |
| 520 | } |
| 521 | |
| 522 | current = []TodoItem{ |
| 523 | {Content: "Add parser", Status: "completed", ActiveForm: "Adding parser"}, |
| 524 | {Content: "Wire parser", Status: "completed"}, |
| 525 | } |
| 526 | ledger.Record(Receipt{ToolName: "complete_step", Success: true, Step: "2"}) |
| 527 | missing, hasBaseline = ledger.UnverifiedCompletedTodos(current) |
| 528 | if !hasBaseline { |
| 529 | t.Fatal("expected prior todo_write baseline") |
| 530 | } |
| 531 | if len(missing) != 0 { |
| 532 | t.Fatalf("numeric complete_step should authorize completion, missing = %+v", missing) |
| 533 | } |
| 534 | } |
| 535 | |
| 536 | func TestLedgerNumericCompleteStepDoesNotAuthorizeReplacedTodo(t *testing.T) { |
| 537 | ledger := NewLedger() |
| 538 | ledger.Record(Receipt{ |
| 539 | ToolName: "todo_write", |
| 540 | Success: true, |
| 541 | Todos: []TodoItem{{Content: "Add parser", Status: "in_progress"}}, |
| 542 | }) |
| 543 | ledger.Record(Receipt{ToolName: "complete_step", Success: true, Step: "1"}) |
| 544 | |
| 545 | missing, hasBaseline := ledger.UnverifiedCompletedTodos([]TodoItem{ |
| 546 | {Content: "Ship parser", Status: "completed"}, |
| 547 | }) |
| 548 | if !hasBaseline { |
| 549 | t.Fatal("expected prior todo_write baseline") |
| 550 | } |
| 551 | if len(missing) != 1 || missing[0].Content != "Ship parser" { |
| 552 | t.Fatalf("numeric complete_step should not authorize a replaced todo, missing = %+v", missing) |
| 553 | } |
| 554 | } |
| 555 | |
| 556 | func TestLedgerNumericCompleteStepFollowsReorderedSignedTodo(t *testing.T) { |
| 557 | ledger := NewLedger() |
| 558 | ledger.Record(Receipt{ |
| 559 | ToolName: "todo_write", |
| 560 | Success: true, |
| 561 | Todos: []TodoItem{ |
| 562 | {Content: "Add parser", Status: "in_progress"}, |
| 563 | {Content: "Write tests", Status: "pending"}, |
| 564 | }, |
| 565 | }) |
| 566 | ledger.Record(Receipt{ToolName: "complete_step", Success: true, Step: "1"}) |
| 567 | |
| 568 | missing, hasBaseline := ledger.UnverifiedCompletedTodos([]TodoItem{ |
| 569 | {Content: "Write tests", Status: "pending"}, |
| 570 | {Content: "Add parser", Status: "completed"}, |
| 571 | }) |
| 572 | if !hasBaseline { |
| 573 | t.Fatal("expected prior todo_write baseline") |
| 574 | } |
| 575 | if len(missing) != 0 { |
| 576 | t.Fatalf("numeric complete_step should follow the signed todo identity after reorder, missing = %+v", missing) |
| 577 | } |
| 578 | } |
| 579 | |
| 580 | func TestLedgerNoBaselineDoesNotConstrainCompletedTodos(t *testing.T) { |
| 581 | ledger := NewLedger() |
| 582 | missing, hasBaseline := ledger.UnverifiedCompletedTodos([]TodoItem{ |
| 583 | {Content: "Add parser", Status: "completed"}, |
| 584 | }) |
| 585 | |
| 586 | if hasBaseline { |
| 587 | t.Fatal("empty ledger should not report a prior todo_write baseline") |
| 588 | } |
| 589 | if len(missing) != 0 { |
| 590 | t.Fatalf("no baseline should not report missing completions, got %+v", missing) |
| 591 | } |
| 592 | } |
| 593 | |
| 594 | func TestValidateSerialTodosAllowsModelReportedOrdering(t *testing.T) { |
| 595 | for _, todos := range [][]TodoItem{ |
| 596 | {{Content: "first", Status: "in_progress"}, {Content: "second", Status: "completed"}}, |
| 597 | {{Content: "first", Status: "pending"}}, |
| 598 | {{Content: "done", Status: "completed"}, {Content: "current", Status: "in_progress"}, {Content: "later", Status: "pending"}}, |
| 599 | } { |
| 600 | if err := ValidateSerialTodos(todos); err != nil { |
| 601 | t.Fatalf("model-reported todo ordering rejected: %v", err) |
| 602 | } |
| 603 | } |
| 604 | if err := ValidateSerialTodos([]TodoItem{ |
| 605 | {Content: "first", Status: "in_progress"}, |
| 606 | {Content: "second", Status: "in_progress"}, |
| 607 | }); err == nil || !strings.Contains(err.Error(), "second in_progress") { |
| 608 | t.Fatalf("multiple current items error = %v", err) |
| 609 | } |
| 610 | } |
| 611 | |
| 612 | func TestNormalizeSerialTodosRepairsLegacyOutOfOrderState(t *testing.T) { |
| 613 | got := NormalizeSerialTodos([]TodoItem{ |
| 614 | {Content: "first", Status: "in_progress"}, |
| 615 | {Content: "second", Status: "completed"}, |
| 616 | {Content: "third", Status: "in_progress"}, |
| 617 | }) |
| 618 | want := []string{"in_progress", "pending", "pending"} |
| 619 | for i := range want { |
| 620 | if got[i].Status != want[i] { |
| 621 | t.Fatalf("todo %d status = %q, want %q: %+v", i+1, got[i].Status, want[i], got) |
| 622 | } |
| 623 | } |
| 624 | } |
| 625 | |
| 626 | func TestValidateSerialTodosAcceptsPhaseChains(t *testing.T) { |
| 627 | tests := []struct { |
| 628 | name string |
| 629 | todos []TodoItem |
| 630 | }{ |
| 631 | { |
| 632 | name: "entered phase with an active sub-step", |
| 633 | todos: []TodoItem{ |
| 634 | {Content: "Phase", Status: "pending"}, |
| 635 | {Content: "sub one", Status: "in_progress", Level: 1}, |
| 636 | }, |
| 637 | }, |
| 638 | { |
| 639 | name: "single current item mid-chain", |
| 640 | todos: []TodoItem{ |
| 641 | {Content: "Phase", Status: "pending"}, |
| 642 | {Content: "sub one", Status: "completed", Level: 1}, |
| 643 | {Content: "sub two", Status: "in_progress", Level: 1}, |
| 644 | {Content: "sub three", Status: "pending", Level: 1}, |
| 645 | {Content: "Later phase", Status: "pending"}, |
| 646 | {Content: "later sub", Status: "pending", Level: 1}, |
| 647 | }, |
| 648 | }, |
| 649 | { |
| 650 | name: "phase awaiting sign-off after its sub-steps", |
| 651 | todos: []TodoItem{ |
| 652 | {Content: "Phase", Status: "in_progress"}, |
| 653 | {Content: "sub one", Status: "completed", Level: 1}, |
| 654 | {Content: "sub two", Status: "completed", Level: 1}, |
| 655 | {Content: "next", Status: "pending"}, |
| 656 | }, |
| 657 | }, |
| 658 | { |
| 659 | name: "completed phase segment before the current one", |
| 660 | todos: []TodoItem{ |
| 661 | {Content: "Phase", Status: "completed"}, |
| 662 | {Content: "sub one", Status: "completed", Level: 1}, |
| 663 | {Content: "Second phase", Status: "pending"}, |
| 664 | {Content: "sub two", Status: "in_progress", Level: 1}, |
| 665 | }, |
| 666 | }, |
| 667 | } |
| 668 | for _, tc := range tests { |
| 669 | t.Run(tc.name, func(t *testing.T) { |
| 670 | if err := ValidateSerialTodos(tc.todos); err != nil { |
| 671 | t.Fatalf("valid phase chain rejected: %v", err) |
| 672 | } |
| 673 | }) |
| 674 | } |
| 675 | } |
| 676 | |
| 677 | func TestValidateSerialTodosOnlyRejectsInvalidPhaseShape(t *testing.T) { |
| 678 | for _, todos := range [][]TodoItem{ |
| 679 | {{Content: "Phase", Status: "completed"}, {Content: "sub one", Status: "in_progress", Level: 1}}, |
| 680 | {{Content: "Phase", Status: "in_progress"}, {Content: "sub one", Status: "pending", Level: 1}}, |
| 681 | {{Content: "Phase", Status: "pending"}, {Content: "sub one", Status: "in_progress", Level: 1}, {Content: "Second", Status: "completed"}}, |
| 682 | } { |
| 683 | if err := ValidateSerialTodos(todos); err != nil { |
| 684 | t.Fatalf("model-reported phase status rejected: %v", err) |
| 685 | } |
| 686 | } |
| 687 | tests := []struct { |
| 688 | name string |
| 689 | todos []TodoItem |
| 690 | want string |
| 691 | }{ |
| 692 | { |
| 693 | name: "phase and sub-step both in_progress", |
| 694 | todos: []TodoItem{ |
| 695 | {Content: "Phase", Status: "in_progress"}, |
| 696 | {Content: "sub one", Status: "in_progress", Level: 1}, |
| 697 | }, |
| 698 | want: "second in_progress item", |
| 699 | }, |
| 700 | { |
| 701 | name: "orphan sub-step with no phase above it", |
| 702 | todos: []TodoItem{ |
| 703 | {Content: "sub one", Status: "in_progress", Level: 1}, |
| 704 | {Content: "Next", Status: "pending"}, |
| 705 | }, |
| 706 | want: "no phase above it", |
| 707 | }, |
| 708 | } |
| 709 | for _, tc := range tests { |
| 710 | t.Run(tc.name, func(t *testing.T) { |
| 711 | if err := ValidateSerialTodos(tc.todos); err == nil || !strings.Contains(err.Error(), tc.want) { |
| 712 | t.Fatalf("ValidateSerialTodos() error = %v, want %q", err, tc.want) |
| 713 | } |
| 714 | }) |
| 715 | } |
| 716 | } |
| 717 | |
| 718 | func TestNormalizeSerialTodosRepairsPhaseChains(t *testing.T) { |
| 719 | got := NormalizeSerialTodos([]TodoItem{ |
| 720 | {Content: "Phase", Status: "completed"}, |
| 721 | {Content: "sub one", Status: "completed", Level: 1}, |
| 722 | {Content: "sub two", Status: "pending", Level: 1}, |
| 723 | {Content: "Second phase", Status: "completed"}, |
| 724 | {Content: "sub three", Status: "completed", Level: 1}, |
| 725 | }) |
| 726 | want := []string{"pending", "completed", "in_progress", "pending", "pending"} |
| 727 | for i := range want { |
| 728 | if got[i].Status != want[i] { |
| 729 | t.Fatalf("todo %d status = %q, want %q: %+v", i+1, got[i].Status, want[i], got) |
| 730 | } |
| 731 | } |
| 732 | |
| 733 | signable := NormalizeSerialTodos([]TodoItem{ |
| 734 | {Content: "Phase", Status: "pending"}, |
| 735 | {Content: "sub one", Status: "completed", Level: 1}, |
| 736 | {Content: "sub two", Status: "completed", Level: 1}, |
| 737 | }) |
| 738 | if signable[0].Status != "in_progress" { |
| 739 | t.Fatalf("phase with completed sub-steps should normalize to in_progress for sign-off: %+v", signable) |
| 740 | } |
| 741 | } |
| 742 | |
| 743 | func TestAdvanceSerialTodoWalksPhaseChain(t *testing.T) { |
| 744 | todos := []TodoItem{ |
| 745 | {Content: "Phase", Status: "pending"}, |
| 746 | {Content: "sub one", Status: "in_progress", Level: 1}, |
| 747 | {Content: "sub two", Status: "pending", Level: 1}, |
| 748 | {Content: "Second phase", Status: "pending"}, |
| 749 | {Content: "sub three", Status: "pending", Level: 1}, |
| 750 | } |
| 751 | statuses := func() []string { |
| 752 | out := make([]string, len(todos)) |
| 753 | for i, todo := range todos { |
| 754 | out[i] = todo.Status |
| 755 | } |
| 756 | return out |
| 757 | } |
| 758 | |
| 759 | if AdvanceSerialTodo(todos, 0) { |
| 760 | t.Fatalf("pending phase completed ahead of its sub-steps: %v", statuses()) |
| 761 | } |
| 762 | if !AdvanceSerialTodo(todos, 1) { |
| 763 | t.Fatal("current sub-step did not complete") |
| 764 | } |
| 765 | if got, want := statuses(), []string{"pending", "completed", "in_progress", "pending", "pending"}; !reflect.DeepEqual(got, want) { |
| 766 | t.Fatalf("after first sub-step statuses = %v, want %v", got, want) |
| 767 | } |
| 768 | if !AdvanceSerialTodo(todos, 2) { |
| 769 | t.Fatal("last sub-step did not complete") |
| 770 | } |
| 771 | if got := todos[0].Status; got != "in_progress" { |
| 772 | t.Fatalf("phase status after its sub-steps = %q, want in_progress for sign-off", got) |
| 773 | } |
| 774 | if !AdvanceSerialTodo(todos, 0) { |
| 775 | t.Fatal("phase with completed sub-steps did not complete") |
| 776 | } |
| 777 | if got, want := statuses(), []string{"completed", "completed", "completed", "pending", "in_progress"}; !reflect.DeepEqual(got, want) { |
| 778 | t.Fatalf("after phase sign-off statuses = %v, want next sub-step promoted under its pending phase: %v", got, want) |
| 779 | } |
| 780 | if !AdvanceSerialTodo(todos, 4) { |
| 781 | t.Fatal("second chain sub-step did not complete") |
| 782 | } |
| 783 | if got := todos[3].Status; got != "in_progress" { |
| 784 | t.Fatalf("second phase after its sub-step = %q, want in_progress", got) |
| 785 | } |
| 786 | if !AdvanceSerialTodo(todos, 3) { |
| 787 | t.Fatal("second phase did not sign off") |
| 788 | } |
| 789 | for i, todo := range todos { |
| 790 | if todo.Status != "completed" { |
| 791 | t.Fatalf("todo %d = %+v, want completed", i+1, todo) |
| 792 | } |
| 793 | } |
| 794 | } |
| 795 | |
| 796 | func TestAdvanceSerialTodoAdvancesOrphanSubStep(t *testing.T) { |
| 797 | todos := []TodoItem{ |
| 798 | {Content: "orphan sub", Status: "in_progress", Level: 1}, |
| 799 | {Content: "Next step", Status: "pending"}, |
| 800 | } |
| 801 | if !AdvanceSerialTodo(todos, 0) { |
| 802 | t.Fatal("orphan sub-step did not complete") |
| 803 | } |
| 804 | if todos[0].Status != "completed" || todos[1].Status != "in_progress" { |
| 805 | t.Fatalf("orphan completion must promote the next pending unit: %+v", todos) |
| 806 | } |
| 807 | |
| 808 | chained := []TodoItem{ |
| 809 | {Content: "orphan sub", Status: "in_progress", Level: 1}, |
| 810 | {Content: "Phase", Status: "pending"}, |
| 811 | {Content: "sub one", Status: "pending", Level: 1}, |
| 812 | } |
| 813 | if !AdvanceSerialTodo(chained, 0) { |
| 814 | t.Fatal("orphan sub-step before a phase did not complete") |
| 815 | } |
| 816 | if chained[1].Status != "pending" || chained[2].Status != "in_progress" { |
| 817 | t.Fatalf("orphan completion before a phase must promote the phase's first sub-step: %+v", chained) |
| 818 | } |
| 819 | } |
| 820 | |
| 821 | func TestSuccessfulProgressSignaturesIgnoreExactRepeatsAndTrackTodoClear(t *testing.T) { |
| 822 | ledger := NewLedger() |
| 823 | read := ReceiptFromToolCall("read_file", json.RawMessage(`{"path":"a.go"}`), true, true) |
| 824 | read.OutputBytes = 10 |
| 825 | ledger.Record(read) |
| 826 | ledger.Record(read) |
| 827 | ledger.Record(ReceiptFromToolCall("todo_write", json.RawMessage(`{"todos":[]}`), true, true)) |
| 828 | ledger.Record(ReceiptFromToolCall("edit_file", json.RawMessage(`{"path":"a.go","old_string":"a","new_string":"b"}`), true, false)) |
| 829 | ledger.Record(ReceiptFromToolCall("bash", json.RawMessage(`{"command":"go test ./..."}`), true, false)) |
| 830 | |
| 831 | sigs := ledger.SuccessfulProgressSignaturesSince(0) |
| 832 | if len(sigs) != 5 { |
| 833 | t.Fatalf("progress signatures = %d, want two reads plus todo clear, mutation, and command", len(sigs)) |
| 834 | } |
| 835 | if sigs[0] != sigs[1] { |
| 836 | t.Fatalf("exact repeated reads should have the same signature: %q != %q", sigs[0], sigs[1]) |
| 837 | } |
| 838 | if sigs[1] == sigs[2] || sigs[2] == sigs[3] || sigs[3] == sigs[4] { |
| 839 | t.Fatalf("distinct host work collapsed to one signature: %v", sigs) |
| 840 | } |
| 841 | } |
| 842 | |
| 843 | func TestLedgerNumericCompleteStepAuthorizesRephrasedTodo(t *testing.T) { |
| 844 | ledger := NewLedger() |
| 845 | ledger.Record(Receipt{ |
| 846 | ToolName: "todo_write", |
| 847 | Success: true, |
| 848 | Todos: []TodoItem{ |
| 849 | {Content: "Add parser", Status: "in_progress"}, |
| 850 | {Content: "Write tests", Status: "pending"}, |
| 851 | }, |
| 852 | }) |
| 853 | ledger.Record(Receipt{ToolName: "complete_step", Success: true, Step: "1"}) |
| 854 | |
| 855 | // The model rephrased item 1 (added detail) but it's the same task. |
| 856 | missing, hasBaseline := ledger.UnverifiedCompletedTodos([]TodoItem{ |
| 857 | {Content: "Add parser with streaming support", Status: "completed"}, |
| 858 | {Content: "Write tests", Status: "pending"}, |
| 859 | }) |
| 860 | if !hasBaseline { |
| 861 | t.Fatal("expected prior todo_write baseline") |
| 862 | } |
| 863 | if len(missing) != 0 { |
| 864 | t.Fatalf("rephrased todo at same index should be authorized by content overlap, missing = %+v", missing) |
| 865 | } |
| 866 | |
| 867 | // The model also rephrased item 2; still ok because the new text contains the old. |
| 868 | missing, hasBaseline = ledger.UnverifiedCompletedTodos([]TodoItem{ |
| 869 | {Content: "Add parser with streaming support", Status: "completed"}, |
| 870 | {Content: "Write tests and benchmarks", Status: "completed"}, |
| 871 | }) |
| 872 | if !hasBaseline { |
| 873 | t.Fatal("expected prior todo_write baseline for second rephrase") |
| 874 | } |
| 875 | // Item 1 is already authorized; item 2 is also rephrased but lacks a |
| 876 | // complete_step — so it should still be flagged. |
| 877 | if len(missing) != 1 || missing[0].Content != "Write tests and benchmarks" { |
| 878 | t.Fatalf("rephrased todo without complete_step should still be missing, got %+v", missing) |
| 879 | } |
| 880 | } |
| 881 | |
| 882 | func TestToolCallMutatesForDeliveryProfile(t *testing.T) { |
| 883 | tests := []struct { |
| 884 | name string |
| 885 | toolName string |
| 886 | args string |
| 887 | readOnly bool |
| 888 | want bool |
| 889 | }{ |
| 890 | {name: "trusted reader", toolName: "read_file", args: `{"path":"a.go"}`, readOnly: true}, |
| 891 | {name: "file writer", toolName: "edit_file", args: `{"path":"a.go"}`, want: true}, |
| 892 | {name: "delegated task meta", toolName: "task", args: `{"prompt":"fix it"}`}, |
| 893 | {name: "run_skill meta", toolName: "run_skill", args: `{"name":"review"}`}, |
| 894 | {name: "review meta", toolName: "review", args: `{"task":"review changes"}`}, |
| 895 | {name: "security_review meta", toolName: "security_review", args: `{"task":"security"}`}, |
| 896 | {name: "use_capability meta", toolName: "use_capability", args: `{"action":"inspect","capability_id":"mcp-server:github"}`}, |
| 897 | {name: "test command", toolName: "bash", args: `{"command":"go test ./..."}`}, |
| 898 | {name: "npx vitest pipeline", toolName: "bash", args: `{"command":"npx vitest run src/lib/foo.test.ts 2>&1 | tail -40"}`}, |
| 899 | {name: "node syntax check", toolName: "bash", args: `{"command":"node --check app.js"}`}, |
| 900 | {name: "node syntax check pipeline", toolName: "bash", args: `{"command":"tail -n +2 app.html | head -n 20 | node --check"}`}, |
| 901 | {name: "node eval stays opaque", toolName: "bash", args: `{"command":"node -e 'console.log(1)'"}`, want: true}, |
| 902 | {name: "node conditions flag stays opaque", toolName: "bash", args: `{"command":"node -C production server.js"}`, want: true}, |
| 903 | {name: "node test runner", toolName: "bash", args: `{"command":"node --test"}`}, |
| 904 | {name: "node test snapshot update stays opaque", toolName: "bash", args: `{"command":"node --test --test-update-snapshots"}`, want: true}, |
| 905 | {name: "node test reporter file stays opaque", toolName: "bash", args: `{"command":"node --test --test-reporter=junit --test-reporter-destination=result.txt"}`, want: true}, |
| 906 | {name: "node test rerun state stays opaque", toolName: "bash", args: `{"command":"node --test --test-rerun-failures=state.json"}`, want: true}, |
| 907 | {name: "node test cpu profile stays opaque", toolName: "bash", args: `{"command":"node --test --cpu-prof"}`, want: true}, |
| 908 | {name: "diff review", toolName: "bash", args: `{"command":"git diff --check"}`}, |
| 909 | {name: "PowerShell network probe does not write workspace", toolName: "bash", args: `{"command":"Test-NetConnection -ComputerName example.com -Port 443"}`}, |
| 910 | {name: "resolved read-only substitution", toolName: "bash", args: `{"command":"basename \"$(pwd)\""}`}, |
| 911 | {name: "writer in substitution", toolName: "bash", args: `{"command":"basename \"$(touch out)\""}`, want: true}, |
| 912 | {name: "formatter write", toolName: "bash", args: `{"command":"gofmt -w internal/a.go"}`, want: true}, |
| 913 | {name: "file redirect", toolName: "bash", args: `{"command":"printf x > generated.txt"}`, want: true}, |
| 914 | {name: "compound verification", toolName: "bash", args: `{"command":"go test ./... 2>&1 | tail -20"}`}, |
| 915 | {name: "pytest snapshot update stays opaque", toolName: "bash", args: `{"command":"pytest --snapshot-update"}`, want: true}, |
| 916 | {name: "pytest junitxml report stays opaque", toolName: "bash", args: `{"command":"pytest --junitxml=report.xml"}`, want: true}, |
| 917 | {name: "gotestsum junitfile stays opaque", toolName: "bash", args: `{"command":"gotestsum --junitfile out.xml ./..."}`, want: true}, |
| 918 | {name: "go test coverprofile stays opaque", toolName: "bash", args: `{"command":"go test -coverprofile=cover.out ./..."}`, want: true}, |
| 919 | {name: "go test blockprofile stays opaque", toolName: "bash", args: `{"command":"go test -blockprofile=block.out ./..."}`, want: true}, |
| 920 | {name: "go test trace stays opaque", toolName: "bash", args: `{"command":"go test -trace trace.out ./..."}`, want: true}, |
| 921 | {name: "go test compile binary stays opaque", toolName: "bash", args: `{"command":"go test -c ./internal/evidence"}`, want: true}, |
| 922 | {name: "go test dotted cpuprofile stays opaque", toolName: "bash", args: `{"command":"go test ./internal/evidence -test.cpuprofile=cpu.out -count=1"}`, want: true}, |
| 923 | {name: "go test artifacts stays opaque", toolName: "bash", args: `{"command":"go test -artifacts ./..."}`, want: true}, |
| 924 | {name: "jest output file stays opaque", toolName: "bash", args: `{"command":"npm test -- --json --outputFile=result.json"}`, want: true}, |
| 925 | {name: "mypy report stays opaque", toolName: "bash", args: `{"command":"mypy --txt-report reports src/"}`, want: true}, |
| 926 | {name: "plain pytest", toolName: "bash", args: `{"command":"pytest"}`}, |
| 927 | } |
| 928 | for _, tt := range tests { |
| 929 | t.Run(tt.name, func(t *testing.T) { |
| 930 | if got := ToolCallMutates(tt.toolName, json.RawMessage(tt.args), tt.readOnly); got != tt.want { |
| 931 | t.Fatalf("ToolCallMutates(%q, %s) = %v, want %v", tt.toolName, tt.args, got, tt.want) |
| 932 | } |
| 933 | }) |
| 934 | } |
| 935 | } |
| 936 | |
| 937 | func TestRunnerWriteOutputFlagsCannotMasqueradeAsVerification(t *testing.T) { |
| 938 | // Snapshot flags rewrite checked-in fixtures and report/profile flags |
| 939 | // write explicit output paths; both must stay opaque mutations so the |
| 940 | // files they produce still require review and sign-off. |
| 941 | for _, command := range []string{ |
| 942 | "pytest --snapshot-update", |
| 943 | "pytest --junitxml=report.xml", |
| 944 | "mypy --junit-xml report.xml src/", |
| 945 | "gotestsum --junitfile out.xml ./...", |
| 946 | "go test -coverprofile=cover.out ./...", |
| 947 | "go test --coverprofile cover.out ./...", |
| 948 | "go test -blockprofile=block.out ./...", |
| 949 | "go test -mutexprofile mutex.out ./...", |
| 950 | "go test -trace trace.out ./...", |
| 951 | "go test -c ./internal/evidence", |
| 952 | "go test -o evidence.test -c ./internal/evidence", |
| 953 | "go test ./internal/evidence -test.cpuprofile=cpu.out -count=1", |
| 954 | "go test -test.trace trace.out ./...", |
| 955 | "go test -artifacts ./...", |
| 956 | "go test ./... -args -test.testlogfile=log.txt", |
| 957 | "go test -test.gocoverdir=covdir ./...", |
| 958 | "gotestsum -- -test.coverprofile=cover.out ./...", |
| 959 | "npm test -- --updateSnapshot", |
| 960 | "npx vitest run --update", |
| 961 | "npx vitest run --update=true", |
| 962 | "npx vitest run -u=true", |
| 963 | "npx jest --updateSnapshot=true", |
| 964 | "npx vitest run --coverage", |
| 965 | "npx vitest run --outputFile=result.json", |
| 966 | "npx --yes vitest run", |
| 967 | "npm test -- --json --outputFile=result.json", |
| 968 | "yarn test --outputFile.json=result.json", |
| 969 | "pytest --report-log=log.jsonl", |
| 970 | "mypy --txt-report reports src/", |
| 971 | "mypy --html-report html src/", |
| 972 | "mypy --xml-report=reports src/", |
| 973 | "mypy --cobertura-xml-report reports src/", |
| 974 | } { |
| 975 | if bashCommandIsVerification(command) { |
| 976 | t.Fatalf("%q writes files and must not be classified as verification", command) |
| 977 | } |
| 978 | if !ToolCallMutates("bash", json.RawMessage(`{"command":"`+command+`"}`), false) { |
| 979 | t.Fatalf("%q must remain an opaque mutation", command) |
| 980 | } |
| 981 | } |
| 982 | for _, command := range []string{ |
| 983 | "pytest", |
| 984 | "gotestsum ./...", |
| 985 | "go test -cover ./...", |
| 986 | "go test -count=1 ./...", |
| 987 | "go test -test.v -test.run TestFoo ./...", |
| 988 | "pytest --trace", |
| 989 | "npm test -- --json", |
| 990 | "npx vitest run src/lib/foo.test.ts 2>&1 | tail -40", |
| 991 | "npx jest src/lib/foo.test.ts", |
| 992 | "mypy src/", |
| 993 | "mypy --strict src/", |
| 994 | } { |
| 995 | if !bashCommandIsVerification(command) { |
| 996 | t.Fatalf("%q should remain a verification command", command) |
| 997 | } |
| 998 | } |
| 999 | } |
| 1000 | |
| 1001 | func TestToolCallRequiresAcceptanceCriteriaForExecutionCommands(t *testing.T) { |
| 1002 | if !ToolCallRequiresAcceptanceCriteria("bash", json.RawMessage(`{"command":"go test ./..."}`), false) { |
| 1003 | t.Fatal("verification command should require delivery acceptance criteria") |
| 1004 | } |
| 1005 | if !ToolCallRequiresAcceptanceCriteria("bash", json.RawMessage(`{"command":"npm run test"}`), false) { |
| 1006 | t.Fatal("npm run test should require delivery acceptance criteria") |
| 1007 | } |
| 1008 | if !ToolCallRequiresAcceptanceCriteria("bash", json.RawMessage(`{"command":"git diff --check"}`), false) { |
| 1009 | t.Fatal("git diff --check is a verification command and should require acceptance criteria") |
| 1010 | } |
| 1011 | if !ToolCallRequiresAcceptanceCriteria("bash", json.RawMessage(`{"command":"node --check app.js"}`), false) { |
| 1012 | t.Fatal("node --check is a verification command and should require acceptance criteria") |
| 1013 | } |
| 1014 | } |
| 1015 | |
| 1016 | func TestBashToolCallUsesOpaqueInlineInterpreter(t *testing.T) { |
| 1017 | tests := []struct { |
| 1018 | command string |
| 1019 | want bool |
| 1020 | }{ |
| 1021 | {command: `node -e 'require("fs").readFileSync("snake.html")'`, want: true}, |
| 1022 | {command: `node --input-type=module --eval 'console.log(1)'`, want: true}, |
| 1023 | {command: `python3 -c 'print(open("snake.html").read())'`, want: true}, |
| 1024 | {command: `ruby -e 'puts 1'`, want: true}, |
| 1025 | {command: `php -r 'echo 1;'`, want: true}, |
| 1026 | {command: `deno eval 'console.log(1)'`, want: true}, |
| 1027 | {command: "tail -n +2 snake.html | node --check -"}, |
| 1028 | {command: "node --test"}, |
| 1029 | {command: "python3 -m pytest"}, |
| 1030 | {command: "node scripts/check.js"}, |
| 1031 | } |
| 1032 | for _, tt := range tests { |
| 1033 | args, err := json.Marshal(map[string]string{"command": tt.command}) |
| 1034 | if err != nil { |
| 1035 | t.Fatal(err) |
| 1036 | } |
| 1037 | if got := BashToolCallUsesOpaqueInlineInterpreter(args); got != tt.want { |
| 1038 | t.Errorf("BashToolCallUsesOpaqueInlineInterpreter(%q) = %v, want %v", tt.command, got, tt.want) |
| 1039 | } |
| 1040 | } |
| 1041 | } |
| 1042 | |
| 1043 | func TestLedgerDeliverySignoffAcceptsNodeSyntaxCheckAfterMutation(t *testing.T) { |
| 1044 | ledger := NewLedger() |
| 1045 | ledger.Record(ReceiptFromToolCall("edit_file", json.RawMessage(`{"path":"app.js"}`), true, false)) |
| 1046 | mutation, ok := ledger.LatestSuccessfulMutationIndex() |
| 1047 | if !ok { |
| 1048 | t.Fatal("expected mutation receipt") |
| 1049 | } |
| 1050 | ledger.Record(ReceiptFromToolCall("read_file", json.RawMessage(`{"path":"app.js"}`), true, true)) |
| 1051 | command := "node --check app.js" |
| 1052 | ledger.Record(ReceiptFromToolCall("bash", json.RawMessage(`{"command":"node --check app.js"}`), true, false)) |
| 1053 | ledger.Record(ReceiptFromToolCall("complete_step", json.RawMessage(`{ |
| 1054 | "step":"Check JavaScript", |
| 1055 | "result":"syntax valid", |
| 1056 | "evidence":[{"kind":"verification","summary":"syntax valid","command":"node --check app.js"}] |
| 1057 | }`), true, true)) |
| 1058 | |
| 1059 | if !IsVerificationCommand(command) { |
| 1060 | t.Fatal("node --check should be recognized as a delivery verification") |
| 1061 | } |
| 1062 | if latest, ok := ledger.LatestSuccessfulMutationIndex(); !ok || latest != mutation { |
| 1063 | t.Fatalf("node --check moved latest mutation from %d to %d (ok=%v)", mutation, latest, ok) |
| 1064 | } |
| 1065 | if !ledger.HasSuccessfulReviewAfter(mutation) { |
| 1066 | t.Fatal("expected post-mutation read to satisfy review") |
| 1067 | } |
| 1068 | if !ledger.HasSuccessfulDeliverySignoffAfter(mutation) { |
| 1069 | t.Fatal("expected node --check to satisfy delivery sign-off") |
| 1070 | } |
| 1071 | } |
| 1072 | |
| 1073 | func TestNodeEvalCannotMasqueradeAsDeliveryVerification(t *testing.T) { |
| 1074 | command := `node -e 'require("fs").readFileSync("app.js")'` |
| 1075 | if IsVerificationCommand(command) { |
| 1076 | t.Fatal("arbitrary node eval must not be recognized as delivery verification") |
| 1077 | } |
| 1078 | if !ToolCallMutates("bash", json.RawMessage(`{"command":"node -e 'require(\"fs\").readFileSync(\"app.js\")'"}`), false) { |
| 1079 | t.Fatal("arbitrary node eval must remain an opaque mutation") |
| 1080 | } |
| 1081 | } |
| 1082 | |
| 1083 | func TestNodeConditionsFlagCannotMasqueradeAsDeliveryVerification(t *testing.T) { |
| 1084 | // Node CLI flags are case-sensitive: -C is --conditions and executes the |
| 1085 | // target script, unlike the syntax-only -c/--check. |
| 1086 | command := "node -C production server.js" |
| 1087 | if IsVerificationCommand(command) { |
| 1088 | t.Fatal("node -C (--conditions) executes the script and must not be recognized as delivery verification") |
| 1089 | } |
| 1090 | if !ToolCallMutates("bash", json.RawMessage(`{"command":"node -C production server.js"}`), false) { |
| 1091 | t.Fatal("node -C (--conditions) must remain an opaque mutation") |
| 1092 | } |
| 1093 | } |
| 1094 | |
| 1095 | func TestNodeTestRunnerWriteFlagsCannotMasqueradeAsDeliveryVerification(t *testing.T) { |
| 1096 | if !IsVerificationCommand("node --test") { |
| 1097 | t.Fatal("plain node --test should be recognized as a delivery verification") |
| 1098 | } |
| 1099 | // Test-runner state/report flags and Node runtime profiling/tracing flags |
| 1100 | // create or update files. They must stay opaque mutations so those files |
| 1101 | // still require review and sign-off. |
| 1102 | for _, command := range []string{ |
| 1103 | "node --test --test-update-snapshots", |
| 1104 | "node --test --test-reporter=junit --test-reporter-destination=result.txt", |
| 1105 | "node --test --test-reporter junit --test-reporter-destination result.txt", |
| 1106 | "node --test --test-rerun-failures=state.json", |
| 1107 | "node --test --test-rerun-failures state.json", |
| 1108 | "node --test --cpu-prof", |
| 1109 | "node --test --heap-prof", |
| 1110 | "node --test --heapsnapshot-near-heap-limit=1", |
| 1111 | "node --test --heapsnapshot-signal=SIGUSR2", |
| 1112 | "node --test --localstorage-file=localstorage.json", |
| 1113 | "node --test --perf-basic-prof", |
| 1114 | "node --test --perf-basic-prof-only-functions", |
| 1115 | "node --test --perf-prof", |
| 1116 | "node --test --prof", |
| 1117 | "node --test --redirect-warnings=warnings.log", |
| 1118 | "node --test --report-on-fatalerror", |
| 1119 | "node --test --report-on-signal", |
| 1120 | "node --test --report-uncaught-exception", |
| 1121 | "node --test --tls-keylog=tls.log", |
| 1122 | "node --test --trace-events-enabled", |
| 1123 | } { |
| 1124 | if IsVerificationCommand(command) { |
| 1125 | t.Fatalf("%q writes files and must not be recognized as delivery verification", command) |
| 1126 | } |
| 1127 | } |
| 1128 | } |
| 1129 | |
| 1130 | func TestLedgerReviewAfterRestoredCheckpointBaseline(t *testing.T) { |
| 1131 | // A negative index is the restored-checkpoint baseline: the mutation |
| 1132 | // happened before a controller rebuild or cold resume, so its receipt (and |
| 1133 | // touched paths) are not in this ledger. Fresh review-shaped receipts must |
| 1134 | // still be able to satisfy the review gate. |
| 1135 | if NewLedger().HasSuccessfulReviewAfter(-1) { |
| 1136 | t.Fatal("an empty ledger must not satisfy the checkpoint-baseline review") |
| 1137 | } |
| 1138 | |
| 1139 | read := NewLedger() |
| 1140 | read.Record(ReceiptFromToolCall("read_file", json.RawMessage(`{"path":"internal/parser.go"}`), true, true)) |
| 1141 | if !read.HasSuccessfulReviewAfter(-1) { |
| 1142 | t.Fatal("a successful read must satisfy review for a restored mutation baseline") |
| 1143 | } |
| 1144 | |
| 1145 | diff := NewLedger() |
| 1146 | diff.Record(ReceiptFromToolCall("bash", json.RawMessage(`{"command":"git diff"}`), true, false)) |
| 1147 | if !diff.HasSuccessfulReviewAfter(-1) { |
| 1148 | t.Fatal("a git diff inspection must satisfy review for a restored mutation baseline") |
| 1149 | } |
| 1150 | |
| 1151 | failed := NewLedger() |
| 1152 | failed.Record(ReceiptFromToolCall("read_file", json.RawMessage(`{"path":"internal/parser.go"}`), false, true)) |
| 1153 | if failed.HasSuccessfulReviewAfter(-1) { |
| 1154 | t.Fatal("a failed read must not satisfy the checkpoint-baseline review") |
| 1155 | } |
| 1156 | |
| 1157 | opaque := NewLedger() |
| 1158 | opaque.Record(ReceiptFromToolCall("bash", json.RawMessage(`{"command":"echo done"}`), true, false)) |
| 1159 | if opaque.HasSuccessfulReviewAfter(-1) { |
| 1160 | t.Fatal("a non-review command must not satisfy the checkpoint-baseline review") |
| 1161 | } |
| 1162 | } |
| 1163 | |
| 1164 | func TestLedgerDeliverySignoffRequiresPostMutationVerificationAndReview(t *testing.T) { |
| 1165 | ledger := NewLedger() |
| 1166 | ledger.Record(ReceiptFromToolCall("todo_write", json.RawMessage(`{"todos":[{"content":"Ship parser","status":"in_progress"}]}`), true, true)) |
| 1167 | ledger.Record(ReceiptFromToolCall("edit_file", json.RawMessage(`{"path":"internal/parser.go"}`), true, false)) |
| 1168 | mutation, ok := ledger.LatestSuccessfulMutationIndex() |
| 1169 | if !ok { |
| 1170 | t.Fatal("expected mutation receipt") |
| 1171 | } |
| 1172 | ledger.Record(ReceiptFromToolCall("read_file", json.RawMessage(`{"path":"internal/parser.go"}`), true, true)) |
| 1173 | ledger.Record(ReceiptFromToolCall("bash", json.RawMessage(`{"command":"go test ./internal/..."}`), true, false)) |
| 1174 | ledger.Record(ReceiptFromToolCall("complete_step", json.RawMessage(`{ |
| 1175 | "step":"Ship parser", |
| 1176 | "result":"parser shipped", |
| 1177 | "evidence":[{"kind":"verification","summary":"tests passed","command":"go test ./internal/..."}] |
| 1178 | }`), true, true)) |
| 1179 | |
| 1180 | if !ledger.HasSuccessfulAcceptanceCriteria() { |
| 1181 | t.Fatal("expected non-empty todo_write to establish acceptance criteria") |
| 1182 | } |
| 1183 | if !ledger.HasSuccessfulReviewAfter(mutation) { |
| 1184 | t.Fatal("expected post-mutation read to satisfy review") |
| 1185 | } |
| 1186 | if !ledger.HasSuccessfulDeliverySignoffAfter(mutation) { |
| 1187 | t.Fatal("expected post-mutation verification cited by complete_step") |
| 1188 | } |
| 1189 | } |
| 1190 | |
| 1191 | func TestLedgerDeliverySignoffRejectsPreMutationVerification(t *testing.T) { |
| 1192 | ledger := NewLedger() |
| 1193 | ledger.Record(ReceiptFromToolCall("bash", json.RawMessage(`{"command":"go test ./..."}`), true, false)) |
| 1194 | ledger.Record(ReceiptFromToolCall("edit_file", json.RawMessage(`{"path":"main.go"}`), true, false)) |
| 1195 | mutation, ok := ledger.LatestSuccessfulMutationIndex() |
| 1196 | if !ok { |
| 1197 | t.Fatal("expected mutation receipt") |
| 1198 | } |
| 1199 | ledger.Record(ReceiptFromToolCall("complete_step", json.RawMessage(`{ |
| 1200 | "step":"change", |
| 1201 | "result":"changed", |
| 1202 | "evidence":[{"kind":"verification","summary":"tests passed before edit","command":"go test ./..."}] |
| 1203 | }`), true, true)) |
| 1204 | if ledger.HasSuccessfulDeliverySignoffAfter(mutation) { |
| 1205 | t.Fatal("pre-mutation verification must not sign off changed work") |
| 1206 | } |
| 1207 | } |
| 1208 | |
| 1209 | func TestLedgerDeliverySignoffRejectsInspectionCommandMasqueradingAsVerification(t *testing.T) { |
| 1210 | ledger := NewLedger() |
| 1211 | ledger.Record(ReceiptFromToolCall("edit_file", json.RawMessage(`{"path":"main.go"}`), true, false)) |
| 1212 | mutation, ok := ledger.LatestSuccessfulMutationIndex() |
| 1213 | if !ok { |
| 1214 | t.Fatal("expected mutation receipt") |
| 1215 | } |
| 1216 | ledger.Record(ReceiptFromToolCall("bash", json.RawMessage(`{"command":"git status --short"}`), true, false)) |
| 1217 | ledger.Record(ReceiptFromToolCall("complete_step", json.RawMessage(`{ |
| 1218 | "step":"change", |
| 1219 | "result":"changed", |
| 1220 | "evidence":[{"kind":"verification","summary":"claimed verification","command":"git status --short"}] |
| 1221 | }`), true, true)) |
| 1222 | if ledger.HasSuccessfulDeliverySignoffAfter(mutation) { |
| 1223 | t.Fatal("inspection-only git status must not count as delivery verification") |
| 1224 | } |
| 1225 | } |
| 1226 |