| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "fmt" |
| 5 | "slices" |
| 6 | "strings" |
| 7 | "unicode" |
| 8 | |
| 9 | "reasonix/internal/agent" |
| 10 | "reasonix/internal/provider" |
| 11 | "reasonix/internal/tool" |
| 12 | ) |
| 13 | |
| 14 | // probeAnswerContract rides with every probe question. A context full of tool |
| 15 | // calls invites the model to answer with another one, which scores as a lost |
| 16 | // fact when it is really a harness artifact — both arms get the same nudge. |
| 17 | const probeAnswerContract = "Answer using only the conversation above. Reply with the answer itself in plain text: no tool calls, no tool-call syntax, no explanation." |
| 18 | |
| 19 | // noAnswerMarker and toolCallMarker label a reply that never answered, so the |
| 20 | // report can separate what compaction lost from what the harness failed to ask. |
| 21 | const ( |
| 22 | noAnswerMarker = "<no answer" |
| 23 | toolCallMarker = "DSML" |
| 24 | toolCallInvalid = "<tool-call syntax instead of an answer>" |
| 25 | ) |
| 26 | |
| 27 | // invalidAnswer reports whether a reply failed to answer at all, rather than |
| 28 | // answering wrongly. These are excluded from the survival rate and counted. |
| 29 | func invalidAnswer(s string) bool { |
| 30 | return strings.HasPrefix(s, noAnswerMarker) || s == toolCallInvalid |
| 31 | } |
| 32 | |
| 33 | // A probe is a fact planted in history and a question only that fact answers. |
| 34 | // The question is asked against the compacted context, so a wrong answer means |
| 35 | // compaction lost the fact — not that the model is weak. |
| 36 | type probe struct { |
| 37 | class string |
| 38 | plantAt int |
| 39 | plant func(*agent.Session) |
| 40 | // later plants more of the same fact in later generations, so a decision |
| 41 | // that changes across folds is tested the way it actually happens: each |
| 42 | // revision lands on the far side of a compaction, not next to the last one. |
| 43 | later map[int]func(*agent.Session) |
| 44 | question string |
| 45 | want []string // answer must contain one of these, lowercased |
| 46 | reject []string // ...and none of these: the pre-correction answer |
| 47 | } |
| 48 | |
| 49 | func userTurn(text string) func(*agent.Session) { |
| 50 | return func(s *agent.Session) { s.Add(provider.Message{Role: provider.RoleUser, Content: text}) } |
| 51 | } |
| 52 | |
| 53 | func toolRound(id, name, args, result string) func(*agent.Session) { |
| 54 | return func(s *agent.Session) { |
| 55 | s.Add(provider.Message{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{{ID: id, Name: name, Arguments: args}}}) |
| 56 | s.Add(provider.Message{Role: provider.RoleTool, ToolCallID: id, Name: name, Content: result}) |
| 57 | } |
| 58 | } |
| 59 | |
| 60 | // failedToolRound is a tool round the host recorded as failed, the way a real |
| 61 | // non-zero bash run arrives. Without the execution record the keep policy sees |
| 62 | // only text, which is exactly the gap the buried-evidence probe measures. |
| 63 | func failedToolRound(id, name, args, result string, code int) func(*agent.Session) { |
| 64 | return func(s *agent.Session) { |
| 65 | exit := code |
| 66 | s.Add(provider.Message{Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{{ID: id, Name: name, Arguments: args}}}) |
| 67 | s.Add(provider.Message{ |
| 68 | Role: provider.RoleTool, ToolCallID: id, Name: name, Content: result, |
| 69 | ToolExecution: &provider.ToolExecution{ |
| 70 | Kind: "shell", |
| 71 | State: tool.ShellStateFailed, |
| 72 | ExitCode: &exit, |
| 73 | }, |
| 74 | }) |
| 75 | } |
| 76 | } |
| 77 | |
| 78 | func seq(fns ...func(*agent.Session)) func(*agent.Session) { |
| 79 | return func(s *agent.Session) { |
| 80 | for _, fn := range fns { |
| 81 | fn(s) |
| 82 | } |
| 83 | } |
| 84 | } |
| 85 | |
| 86 | // probeSuite covers what a coding agent must not lose across a fold. The |
| 87 | // freshness and correction probes are the ones summaries classically get wrong: |
| 88 | // both have a plausible stale answer that reads as correct. |
| 89 | func probeSuite() []probe { |
| 90 | suite := append(factProbes(), revisionProbes()...) |
| 91 | return append(suite, snipProbes()...) |
| 92 | } |
| 93 | |
| 94 | // buriedTestLog is a `go test -v` log whose only failure detail sits in the |
| 95 | // middle. Geometric snipping keeps the first 80 and last 12 lines, so this is |
| 96 | // precisely the shape it drops while reporting a plausible-looking remainder. |
| 97 | // It names a package no other probe touches: a run covering config/ would be a |
| 98 | // truthful "yes" for verification-freshness and score that probe as lost. |
| 99 | func buriedTestLog() string { |
| 100 | var b strings.Builder |
| 101 | pass := func(i int) { |
| 102 | fmt.Fprintf(&b, "=== RUN TestEmitterCase%03d\n--- PASS: TestEmitterCase%03d (0.00s)\n", i, i) |
| 103 | } |
| 104 | for i := range 120 { |
| 105 | pass(i) |
| 106 | } |
| 107 | b.WriteString("=== RUN TestEmitterQuoting\n") |
| 108 | b.WriteString(" emitter_test.go:412: assertion failed: expected beta-7d21, got gamma-4a88\n") |
| 109 | b.WriteString("--- FAIL: TestEmitterQuoting (0.01s)\n") |
| 110 | for i := 120; i < 200; i++ { |
| 111 | pass(i) |
| 112 | } |
| 113 | b.WriteString("FAIL\nexit status 1\nFAIL\treasonix/emitter\t0.234s\n") |
| 114 | return b.String() |
| 115 | } |
| 116 | |
| 117 | // snipProbes plant a fact in the middle of a long tool result, out of reach of |
| 118 | // the head/tail geometry. They score the maintenance pass rather than the fold: |
| 119 | // every other probe in the suite is short enough that snipping cannot touch it. |
| 120 | func snipProbes() []probe { |
| 121 | return []probe{ |
| 122 | { |
| 123 | class: "buried-evidence", |
| 124 | plantAt: 1, |
| 125 | plant: failedToolRound("bt1", "bash", `{"cmd":"go test ./emitter -v"}`, buriedTestLog(), 1), |
| 126 | question: "In the failed test, what value did the assertion expect? Reply with just the value.", |
| 127 | want: []string{"beta-7d21"}, |
| 128 | reject: []string{"gamma-4a88"}, |
| 129 | }, |
| 130 | } |
| 131 | } |
| 132 | |
| 133 | func factProbes() []probe { |
| 134 | return []probe{ |
| 135 | { |
| 136 | class: "user-constraint", |
| 137 | plantAt: 0, |
| 138 | plant: userTurn("Hard rule for this whole task: never modify config/schema.sql."), |
| 139 | question: "Are you allowed to modify config/schema.sql? Answer yes or no.", |
| 140 | want: []string{"no"}, |
| 141 | reject: []string{"yes"}, |
| 142 | }, |
| 143 | { |
| 144 | class: "correction", |
| 145 | plantAt: 0, |
| 146 | plant: seq( |
| 147 | userTurn("Indent the generated config with tabs."), |
| 148 | userTurn("Correction: indent the generated config with spaces, not tabs. This supersedes what I said before."), |
| 149 | ), |
| 150 | question: "Should the generated config be indented with tabs or with spaces? Answer with one word.", |
| 151 | want: []string{"spaces", "space"}, |
| 152 | reject: []string{"tabs", "tab"}, |
| 153 | }, |
| 154 | { |
| 155 | class: "exact-identifier", |
| 156 | plantAt: 0, |
| 157 | plant: userTurn("Track this work under ticket RX-4821; put that id in the commit message."), |
| 158 | question: "What is the ticket id for this work? Reply with just the id.", |
| 159 | want: []string{"rx-4821"}, |
| 160 | }, |
| 161 | { |
| 162 | class: "objective", |
| 163 | plantAt: 0, |
| 164 | plant: userTurn("To be clear, the objective is the config round-trip formatting bug, not performance."), |
| 165 | question: "In a few words, what is the current objective?", |
| 166 | want: []string{"round-trip", "round trip", "roundtrip", "formatting"}, |
| 167 | }, |
| 168 | { |
| 169 | class: "pending-requirement", |
| 170 | plantAt: 1, |
| 171 | plant: seq( |
| 172 | userTurn("Two requirements: R1 preserve unknown keys, R2 keep quoted values quoted."), |
| 173 | toolRound("r1", "bash", `{"cmd":"go test ./config -run TestUnknownKeys"}`, "ok\nPASS: TestUnknownKeys (R1 satisfied)"), |
| 174 | ), |
| 175 | question: "Of requirements R1 and R2, which one is still not satisfied? Reply with just R1 or R2.", |
| 176 | want: []string{"r2"}, |
| 177 | reject: []string{"r1"}, |
| 178 | }, |
| 179 | { |
| 180 | class: "verification-freshness", |
| 181 | plantAt: 1, |
| 182 | plant: seq( |
| 183 | toolRound("v1", "bash", `{"cmd":"go test ./config -run TestRoundTrip"}`, "ok config\tPASS: TestRoundTrip"), |
| 184 | toolRound("w1", "write_file", `{"path":"config/format.go"}`, "wrote config/format.go (42 lines changed)"), |
| 185 | userTurn("Note that config/format.go changed after that test run."), |
| 186 | ), |
| 187 | question: "Has TestRoundTrip been run again since config/format.go was last edited? Answer yes or no.", |
| 188 | want: []string{"no"}, |
| 189 | reject: []string{"yes"}, |
| 190 | }, |
| 191 | { |
| 192 | class: "negative-evidence", |
| 193 | plantAt: 1, |
| 194 | plant: seq( |
| 195 | userTurn("We suspected parser normalization was the root cause."), |
| 196 | toolRound("n1", "bash", `{"cmd":"go test ./config -run TestParserNormalization"}`, "PASS — parser normalization is NOT the root cause; ruled out."), |
| 197 | ), |
| 198 | question: "Is parser normalization the root cause of the bug? Answer yes or no.", |
| 199 | want: []string{"no"}, |
| 200 | reject: []string{"yes"}, |
| 201 | }, |
| 202 | { |
| 203 | class: "tool-outcome", |
| 204 | plantAt: 2, |
| 205 | plant: seq( |
| 206 | toolRound("b1", "bash", `{"cmd":"go vet ./..."}`, "config/format.go:88: printf: non-constant format string\nexit status 1"), |
| 207 | userTurn("Leave that vet warning for now; we will fix it at the end."), |
| 208 | ), |
| 209 | question: "Did `go vet ./...` pass the last time it ran? Answer yes or no.", |
| 210 | want: []string{"no"}, |
| 211 | reject: []string{"yes"}, |
| 212 | }, |
| 213 | { |
| 214 | class: "code-fact", |
| 215 | plantAt: 2, |
| 216 | plant: toolRound("c1", "read_file", `{"path":"config/save.go"}`, "// Config.Save intentionally preserves unknown keys so plugins round-trip through this path.\nfunc (c *Config) Save() error { /* ... */ }"), |
| 217 | question: "Does Config.Save preserve unknown keys? Answer yes or no.", |
| 218 | want: []string{"yes"}, |
| 219 | reject: []string{"no"}, |
| 220 | }, |
| 221 | { |
| 222 | class: "chronology", |
| 223 | plantAt: 2, |
| 224 | plant: seq( |
| 225 | toolRound("o1", "write_file", `{"path":"config/parser.go"}`, "wrote config/parser.go"), |
| 226 | toolRound("o2", "write_file", `{"path":"config/format.go"}`, "wrote config/format.go"), |
| 227 | ), |
| 228 | question: "Was config/parser.go edited before or after config/format.go? Reply with just: before, or after.", |
| 229 | want: []string{"before"}, |
| 230 | reject: []string{"after"}, |
| 231 | }, |
| 232 | } |
| 233 | } |
| 234 | |
| 235 | // revisionProbes state a fact and then change it in a later generation. The |
| 236 | // revision lands on the far side of a fold, so carrying it forward means |
| 237 | // superseding the digest's own earlier claim rather than a neighbouring turn. |
| 238 | func revisionProbes() []probe { |
| 239 | return []probe{ |
| 240 | { |
| 241 | class: "late-constraint", |
| 242 | plantAt: 3, |
| 243 | plant: userTurn("New hard rule from here on: never edit anything under internal/store/."), |
| 244 | question: "Are you allowed to edit files under internal/store/? Answer yes or no.", |
| 245 | want: []string{"no"}, |
| 246 | reject: []string{"yes"}, |
| 247 | }, |
| 248 | { |
| 249 | class: "reversal-chain", |
| 250 | plantAt: 0, |
| 251 | plant: userTurn("Use PostgreSQL for the datastore."), |
| 252 | later: map[int]func(*agent.Session){ |
| 253 | 1: userTurn("Change of plan: drop PostgreSQL, use SQLite instead."), |
| 254 | 2: userTurn("Final call: back to PostgreSQL after all."), |
| 255 | }, |
| 256 | question: "Which datastore is the current decision? Reply with one word.", |
| 257 | want: []string{"postgresql", "postgres"}, |
| 258 | reject: []string{"sqlite"}, |
| 259 | }, |
| 260 | { |
| 261 | class: "distractor-file", |
| 262 | plantAt: 1, |
| 263 | plant: userTurn("Draft note: the fix might belong in config/legacy_parser.go."), |
| 264 | later: map[int]func(*agent.Session){ |
| 265 | 3: userTurn("Confirmed: the fix belongs in config/emitter.go; the legacy parser is not involved."), |
| 266 | }, |
| 267 | question: "Which file does the fix belong in? Reply with just the filename, no path.", |
| 268 | want: []string{"emitter"}, |
| 269 | reject: []string{"legacy"}, |
| 270 | }, |
| 271 | } |
| 272 | } |
| 273 | |
| 274 | // score reports whether an answer keeps the planted fact. A rejected token |
| 275 | // anywhere in the answer counts as lost even when the wanted token also |
| 276 | // appears, so "yes, but it was not re-run" does not pass as "no". |
| 277 | func (p probe) score(answer string) bool { |
| 278 | a := strings.ToLower(answer) |
| 279 | for _, bad := range p.reject { |
| 280 | if matchesToken(a, bad) { |
| 281 | return false |
| 282 | } |
| 283 | } |
| 284 | for _, good := range p.want { |
| 285 | if matchesToken(a, good) { |
| 286 | return true |
| 287 | } |
| 288 | } |
| 289 | return false |
| 290 | } |
| 291 | |
| 292 | // matchesToken compares whole words, never substrings: "I am not sure" must not |
| 293 | // count as the answer "no". Multi-word wants are phrases and match literally. |
| 294 | func matchesToken(lowered, want string) bool { |
| 295 | if strings.Contains(want, " ") { |
| 296 | return strings.Contains(lowered, want) |
| 297 | } |
| 298 | return slices.Contains(strings.FieldsFunc(lowered, func(r rune) bool { |
| 299 | return !unicode.IsLetter(r) && !unicode.IsDigit(r) && r != '-' |
| 300 | }), want) |
| 301 | } |
| 302 | |
| 303 | // settledAt is the first generation whose answer is the one want/reject scores. |
| 304 | // A probe revised across folds has a different correct answer while the chain is |
| 305 | // still running, so asking before it settles would score a right answer as lost. |
| 306 | func (p probe) settledAt() int { |
| 307 | at := p.plantAt |
| 308 | for gen := range p.later { |
| 309 | at = max(at, gen) |
| 310 | } |
| 311 | return at |
| 312 | } |
| 313 | |
| 314 | func (p probe) String() string { return fmt.Sprintf("%s@gen%d", p.class, p.settledAt()) } |
| 315 |