| 1 | { |
| 2 | "schema_version": 2, |
| 3 | "cases": [ |
| 4 | { |
| 5 | "id": "last-round-tools-kept", |
| 6 | "expect": "pass", |
| 7 | "last_round_start": 0, |
| 8 | "original": [ |
| 9 | {"role": "user", "content": [{"type": "text", "text": "Run the failing test."}]}, |
| 10 | {"role": "assistant", "content": [{"type": "text", "text": "Running."}]}, |
| 11 | {"role": "assistant", "content": [{"type": "tool_use", "id": "live", "name": "Bash", "input": {"command": "cargo test"}}]}, |
| 12 | {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "live", "content": "test session_store::roundtrip ... FAILED"}]} |
| 13 | ], |
| 14 | "replacement": [ |
| 15 | {"role": "user", "content": [{"type": "text", "text": "Run the failing test."}]}, |
| 16 | {"role": "assistant", "content": [{"type": "text", "text": "Running."}]}, |
| 17 | {"role": "assistant", "content": [{"type": "tool_use", "id": "live", "name": "Bash", "input": {"command": "cargo test"}}]}, |
| 18 | {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "live", "content": "test session_store::roundtrip ... FAILED"}]}, |
| 19 | {"role": "user", "content": [{"type": "text", "text": "Another language model started to solve this problem: suite still failing"}]} |
| 20 | ] |
| 21 | }, |
| 22 | { |
| 23 | "id": "last-round-tools-dropped", |
| 24 | "expect": "fail", |
| 25 | "last_round_start": 0, |
| 26 | "original": [ |
| 27 | {"role": "user", "content": [{"type": "text", "text": "Run the failing test."}]}, |
| 28 | {"role": "assistant", "content": [{"type": "text", "text": "Running."}]}, |
| 29 | {"role": "assistant", "content": [{"type": "tool_use", "id": "live", "name": "Bash", "input": {"command": "cargo test"}}]}, |
| 30 | {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "live", "content": "test session_store::roundtrip ... FAILED"}]} |
| 31 | ], |
| 32 | "replacement": [ |
| 33 | {"role": "user", "content": [{"type": "text", "text": "Run the failing test."}]}, |
| 34 | {"role": "user", "content": [{"type": "text", "text": "Another language model started to solve this problem: and kept going"}]} |
| 35 | ] |
| 36 | }, |
| 37 | { |
| 38 | "id": "last-round-assistant-dropped", |
| 39 | "expect": "fail", |
| 40 | "last_round_start": 0, |
| 41 | "original": [ |
| 42 | {"role": "user", "content": [{"type": "text", "text": "What failed?"}]}, |
| 43 | {"role": "assistant", "content": [{"type": "text", "text": "session_store::roundtrip panics on reload."}]} |
| 44 | ], |
| 45 | "replacement": [ |
| 46 | {"role": "user", "content": [{"type": "text", "text": "What failed?"}]}, |
| 47 | {"role": "user", "content": [{"type": "text", "text": "Another language model started to solve this problem: missing assistant"}]} |
| 48 | ] |
| 49 | }, |
| 50 | { |
| 51 | "id": "toolless-tail-keeps-prior-tools", |
| 52 | "expect": "pass", |
| 53 | "last_round_start": 0, |
| 54 | "original": [ |
| 55 | {"role": "user", "content": [{"type": "text", "text": "Run the failing test."}]}, |
| 56 | {"role": "assistant", "content": [{"type": "text", "text": "Running."}]}, |
| 57 | {"role": "assistant", "content": [{"type": "tool_use", "id": "live", "name": "Bash", "input": {"command": "cargo test"}}]}, |
| 58 | {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "live", "content": "test session_store::roundtrip ... FAILED"}]}, |
| 59 | {"role": "user", "content": [{"type": "text", "text": "ok thanks"}]}, |
| 60 | {"role": "assistant", "content": [{"type": "text", "text": "you're welcome"}]}, |
| 61 | {"role": "user", "content": [{"type": "text", "text": "one more thing"}]}, |
| 62 | {"role": "assistant", "content": [{"type": "text", "text": "sure"}]} |
| 63 | ], |
| 64 | "replacement": [ |
| 65 | {"role": "user", "content": [{"type": "text", "text": "Run the failing test."}]}, |
| 66 | {"role": "assistant", "content": [{"type": "text", "text": "Running."}]}, |
| 67 | {"role": "assistant", "content": [{"type": "tool_use", "id": "live", "name": "Bash", "input": {"command": "cargo test"}}]}, |
| 68 | {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "live", "content": "test session_store::roundtrip ... FAILED"}]}, |
| 69 | {"role": "user", "content": [{"type": "text", "text": "ok thanks"}]}, |
| 70 | {"role": "assistant", "content": [{"type": "text", "text": "you're welcome"}]}, |
| 71 | {"role": "user", "content": [{"type": "text", "text": "one more thing"}]}, |
| 72 | {"role": "assistant", "content": [{"type": "text", "text": "sure"}]}, |
| 73 | {"role": "user", "content": [{"type": "text", "text": "Another language model started to solve this problem: keep the failing test result"}]} |
| 74 | ] |
| 75 | }, |
| 76 | { |
| 77 | "id": "toolless-tail-drops-prior-tools", |
| 78 | "expect": "fail", |
| 79 | "last_round_start": 0, |
| 80 | "original": [ |
| 81 | {"role": "user", "content": [{"type": "text", "text": "Run the failing test."}]}, |
| 82 | {"role": "assistant", "content": [{"type": "text", "text": "Running."}]}, |
| 83 | {"role": "assistant", "content": [{"type": "tool_use", "id": "live", "name": "Bash", "input": {"command": "cargo test"}}]}, |
| 84 | {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "live", "content": "test session_store::roundtrip ... FAILED"}]}, |
| 85 | {"role": "user", "content": [{"type": "text", "text": "ok thanks"}]}, |
| 86 | {"role": "assistant", "content": [{"type": "text", "text": "you're welcome"}]}, |
| 87 | {"role": "user", "content": [{"type": "text", "text": "one more thing"}]}, |
| 88 | {"role": "assistant", "content": [{"type": "text", "text": "sure"}]} |
| 89 | ], |
| 90 | "replacement": [ |
| 91 | {"role": "user", "content": [{"type": "text", "text": "one more thing"}]}, |
| 92 | {"role": "assistant", "content": [{"type": "text", "text": "sure"}]}, |
| 93 | {"role": "user", "content": [{"type": "text", "text": "Another language model started to solve this problem: dropped the tool round"}]} |
| 94 | ] |
| 95 | }, |
| 96 | { |
| 97 | "id": "chat-only-last-round", |
| 98 | "expect": "pass", |
| 99 | "last_round_start": 2, |
| 100 | "original": [ |
| 101 | {"role": "user", "content": [{"type": "text", "text": "hello"}]}, |
| 102 | {"role": "assistant", "content": [{"type": "text", "text": "hi"}]}, |
| 103 | {"role": "user", "content": [{"type": "text", "text": "how are you"}]}, |
| 104 | {"role": "assistant", "content": [{"type": "text", "text": "fine"}]} |
| 105 | ], |
| 106 | "replacement": [ |
| 107 | {"role": "user", "content": [{"type": "text", "text": "how are you"}]}, |
| 108 | {"role": "assistant", "content": [{"type": "text", "text": "fine"}]}, |
| 109 | {"role": "user", "content": [{"type": "text", "text": "Another language model started to solve this problem: continued the chat"}]} |
| 110 | ] |
| 111 | }, |
| 112 | { |
| 113 | "id": "dropped-anchor", |
| 114 | "expect": "fail", |
| 115 | "anchors": "ship 0.9.12", |
| 116 | "last_round_start": 0, |
| 117 | "original": [ |
| 118 | {"role": "user", "content": [{"type": "text", "text": "Keep the pin."}]}, |
| 119 | {"role": "assistant", "content": [{"type": "text", "text": "Anchored."}]} |
| 120 | ], |
| 121 | "replacement": [ |
| 122 | {"role": "user", "content": [{"type": "text", "text": "Keep the pin."}]}, |
| 123 | {"role": "assistant", "content": [{"type": "text", "text": "Anchored."}]}, |
| 124 | {"role": "user", "content": [{"type": "text", "text": "Another language model started to solve this problem: progress without the pin"}]} |
| 125 | ] |
| 126 | }, |
| 127 | { |
| 128 | "id": "missing-receipt", |
| 129 | "expect": "fail", |
| 130 | "anchors": "ship 0.9.12", |
| 131 | "last_round_start": 0, |
| 132 | "original": [ |
| 133 | {"role": "user", "content": [{"type": "text", "text": "Keep the pin."}]}, |
| 134 | {"role": "assistant", "content": [{"type": "text", "text": "Anchored."}]} |
| 135 | ], |
| 136 | "replacement": [ |
| 137 | {"role": "user", "content": [{"type": "text", "text": "Keep the pin."}]}, |
| 138 | {"role": "assistant", "content": [{"type": "text", "text": "Anchored."}]} |
| 139 | ] |
| 140 | }, |
| 141 | { |
| 142 | "id": "duplicate-checkpoint", |
| 143 | "expect": "fail", |
| 144 | "last_round_start": 1, |
| 145 | "original": [ |
| 146 | {"role": "user", "content": [{"type": "text", "text": "older"}]}, |
| 147 | {"role": "user", "content": [{"type": "text", "text": "Run the suite now."}]}, |
| 148 | {"role": "assistant", "content": [{"type": "text", "text": "Rerunning."}]}, |
| 149 | {"role": "assistant", "content": [{"type": "tool_use", "id": "live", "name": "Bash", "input": {"command": "cargo test"}}]}, |
| 150 | {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "live", "content": "ok"}]}, |
| 151 | {"role": "user", "content": [{"type": "text", "text": "Another language model started to solve this problem: first handoff"}]} |
| 152 | ], |
| 153 | "replacement": [ |
| 154 | {"role": "user", "content": [{"type": "text", "text": "Run the suite now."}]}, |
| 155 | {"role": "assistant", "content": [{"type": "text", "text": "Rerunning."}]}, |
| 156 | {"role": "assistant", "content": [{"type": "tool_use", "id": "live", "name": "Bash", "input": {"command": "cargo test"}}]}, |
| 157 | {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "live", "content": "ok"}]}, |
| 158 | {"role": "user", "content": [{"type": "text", "text": "Another language model started to solve this problem: first handoff"}]}, |
| 159 | {"role": "user", "content": [{"type": "text", "text": "Another language model started to solve this problem: second handoff"}]} |
| 160 | ] |
| 161 | }, |
| 162 | {"id":"continuous-round-summarizes-old-exchanges","expect":"pass","last_round_start":0,"original":[{"role":"user","content":[{"type":"text","text":"Continue this task without publishing."}]},{"role":"assistant","content":[{"type":"tool_use","id":"old","name":"File","input":{"path":"old"}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"old","content":"evidence old"}]},{"role":"user","content":[{"type":"text","text":"Another language model started to solve this problem. Old step completed; latest checks remain."}]},{"role":"assistant","content":[{"type":"tool_use","id":"new-a","name":"File","input":{"path":"new-a"}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"new-a","content":"evidence new-a"}]},{"role":"assistant","content":[{"type":"tool_use","id":"new-b","name":"File","input":{"path":"new-b"}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"new-b","content":"evidence new-b"}]}],"replacement":[{"role":"user","content":[{"type":"text","text":"Continue this task without publishing."}]},{"role":"assistant","content":[{"type":"tool_use","id":"new-a","name":"File","input":{"path":"new-a"}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"new-a","content":"evidence new-a"}]},{"role":"assistant","content":[{"type":"tool_use","id":"new-b","name":"File","input":{"path":"new-b"}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"new-b","content":"evidence new-b"}]},{"role":"user","content":[{"type":"text","text":"Another language model started to solve this problem. Old step completed; latest checks remain."}]}]}, |
| 163 | {"id":"continuous-round-must-keep-post-checkpoint-results","expect":"fail","original":[{"role":"user","content":[{"type":"text","text":"Continue this task without publishing."}]},{"role":"assistant","content":[{"type":"tool_use","id":"old","name":"File","input":{"path":"old"}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"old","content":"evidence old"}]},{"role":"user","content":[{"type":"text","text":"Another language model started to solve this problem. Old step completed; latest checks remain."}]},{"role":"assistant","content":[{"type":"tool_use","id":"new-a","name":"File","input":{"path":"new-a"}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"new-a","content":"evidence new-a"}]},{"role":"assistant","content":[{"type":"tool_use","id":"new-b","name":"File","input":{"path":"new-b"}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"new-b","content":"evidence new-b"}]}],"replacement":[{"role":"user","content":[{"type":"text","text":"Continue this task without publishing."}]},{"role":"assistant","content":[{"type":"tool_use","id":"old","name":"File","input":{"path":"old"}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"old","content":"evidence old"}]},{"role":"user","content":[{"type":"text","text":"Another language model started to solve this problem. Old step completed; latest checks remain."}]}]}, |
| 164 | {"id":"continuous-round-must-keep-tool-pairs","expect":"fail","original":[{"role":"user","content":[{"type":"text","text":"Continue this task without publishing."}]},{"role":"assistant","content":[{"type":"tool_use","id":"old","name":"File","input":{"path":"old"}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"old","content":"evidence old"}]},{"role":"user","content":[{"type":"text","text":"Another language model started to solve this problem. Old step completed; latest checks remain."}]},{"role":"assistant","content":[{"type":"tool_use","id":"new-a","name":"File","input":{"path":"new-a"}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"new-a","content":"evidence new-a"}]},{"role":"assistant","content":[{"type":"tool_use","id":"new-b","name":"File","input":{"path":"new-b"}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"new-b","content":"evidence new-b"}]}],"replacement":[{"role":"user","content":[{"type":"text","text":"Continue this task without publishing."}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"new-a","content":"evidence new-a"}]},{"role":"assistant","content":[{"type":"tool_use","id":"new-b","name":"File","input":{"path":"new-b"}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"new-b","content":"evidence new-b"}]},{"role":"user","content":[{"type":"text","text":"Another language model started to solve this problem. Old step completed; latest checks remain."}]}]} |
| 165 | ] |
| 166 | } |
| 167 |