| 1 | // MemoryBench support: per-task memory-store seeding, the memory-off |
| 2 | // counterfactual arm, recall/marker extraction from trajectories, and the |
| 3 | // paired memory-utility readout. The core KPI is Task Pass(memory on) minus |
| 4 | // Task Pass(memory off) on identical tasks — retrieval that looks relevant |
| 5 | // but does not move task outcomes counts as overhead, not as recall quality. |
| 6 | package main |
| 7 | |
| 8 | import ( |
| 9 | "bufio" |
| 10 | "encoding/json" |
| 11 | "fmt" |
| 12 | "os" |
| 13 | "path/filepath" |
| 14 | "strings" |
| 15 | |
| 16 | "reasonix/internal/config" |
| 17 | ) |
| 18 | |
| 19 | // seedTaskMemory populates an isolated memory state root from the task's |
| 20 | // memory/ directory (memory/project/*.md and memory/global/*.md) and returns |
| 21 | // the env entry pointing the child at it. Tasks without seeds get no env, so |
| 22 | // ordinary suites keep using the developer's real store root untouched. |
| 23 | func seedTaskMemory(taskDir, work string) ([]string, error) { |
| 24 | seeds := filepath.Join(taskDir, "memory") |
| 25 | if _, err := os.Stat(seeds); err != nil { |
| 26 | return nil, nil |
| 27 | } |
| 28 | stateHome, err := os.MkdirTemp("", "e2ebench-mem-") |
| 29 | if err != nil { |
| 30 | return nil, err |
| 31 | } |
| 32 | absWork, err := filepath.Abs(work) |
| 33 | if err != nil { |
| 34 | return nil, err |
| 35 | } |
| 36 | // The child derives its project slug from Getwd, which returns the |
| 37 | // symlink-resolved path (/private/var vs /var on macOS); seed under the |
| 38 | // same identity or the store lands in a directory nobody reads. |
| 39 | if resolved, rErr := filepath.EvalSymlinks(absWork); rErr == nil { |
| 40 | absWork = resolved |
| 41 | } |
| 42 | pairs := [][2]string{ |
| 43 | {filepath.Join(seeds, "project"), filepath.Join(stateHome, "projects", config.WorkspaceSlug(absWork), "memory")}, |
| 44 | {filepath.Join(seeds, "global"), filepath.Join(stateHome, "memory", "global")}, |
| 45 | } |
| 46 | for _, pair := range pairs { |
| 47 | if _, err := os.Stat(pair[0]); err != nil { |
| 48 | continue |
| 49 | } |
| 50 | if err := os.MkdirAll(pair[1], 0o755); err != nil { |
| 51 | return nil, err |
| 52 | } |
| 53 | if err := copyDir(pair[0], pair[1]); err != nil { |
| 54 | return nil, err |
| 55 | } |
| 56 | } |
| 57 | return []string{"REASONIX_STATE_HOME=" + stateHome}, nil |
| 58 | } |
| 59 | |
| 60 | // taskExperimentEnv assembles one run's experiment environment: the policy |
| 61 | // arm, fork capture, and the seeded memory state root. The note reports a |
| 62 | // seeding failure without aborting the run. |
| 63 | func taskExperimentEnv(cfg suiteConfig, t task, work string) (env []string, note string) { |
| 64 | switch cfg.policy { |
| 65 | case "ebm": |
| 66 | env = append(env, "REASONIX_EXPERIMENT_EBM=1") |
| 67 | case "governor": |
| 68 | env = append(env, "REASONIX_EXPERIMENT_GOVERNOR=1") |
| 69 | case "memory-off": |
| 70 | env = append(env, "REASONIX_EXPERIMENT_NO_MEMORY=1") |
| 71 | } |
| 72 | if cfg.forkCapture != "" { |
| 73 | env = append(env, "REASONIX_EXPERIMENT_FORK_CAPTURE_DIR="+filepath.Join(cfg.forkCapture, t.ID)) |
| 74 | } |
| 75 | seedEnv, err := seedTaskMemory(t.dir, work) |
| 76 | if err != nil { |
| 77 | return env, "memory seed: " + err.Error() |
| 78 | } |
| 79 | return append(env, seedEnv...), "" |
| 80 | } |
| 81 | |
| 82 | // applyMemoryStats folds one trajectory's recall behavior into the result row. |
| 83 | func applyMemoryStats(r *result, trajPath string, t task) { |
| 84 | stats := scanMemoryRecall(trajPath, t.MemoryMarkers, t.MemoryMarkersPrefix) |
| 85 | r.MemoryRecallEvents, r.MemoryRecallHits = stats.RecallEvents, stats.RecallHits |
| 86 | r.MemoryRecallChars, r.MemorySuppressed = stats.RecallChars, stats.Suppressed |
| 87 | r.MemoryMarkersUsed, r.MemoryShadowAgree = stats.MarkersUsed, stats.ShadowAgree |
| 88 | } |
| 89 | |
| 90 | // memoryRunStats is what one trajectory reveals about recall behavior. |
| 91 | type memoryRunStats struct { |
| 92 | RecallEvents int // user turns where automatic recall ran and injected facts |
| 93 | RecallHits int |
| 94 | RecallChars int |
| 95 | Suppressed int // recall decisions that stayed silent |
| 96 | MarkersUsed int // task markers seen in tool args or answer text after recall |
| 97 | ShadowAgree int // recall events where the V2 shadow's top hit matched production's |
| 98 | } |
| 99 | |
| 100 | // scanMemoryRecall extracts recall decisions and point-of-use evidence: a |
| 101 | // marker (a unique token planted in a seeded fact body) counts as used only |
| 102 | // when it appears in tool arguments or answer text AFTER a recall injected |
| 103 | // facts — the fact reached the decision path, not just the ranking. |
| 104 | func scanMemoryRecall(path string, markers []string, markersInPrefix bool) memoryRunStats { |
| 105 | var stats memoryRunStats |
| 106 | f, err := os.Open(path) |
| 107 | if err != nil { |
| 108 | return stats |
| 109 | } |
| 110 | defer f.Close() |
| 111 | type record struct { |
| 112 | MemoryRecall *struct { |
| 113 | Hits []struct{ ID string } `json:"hits"` |
| 114 | UsedChars int `json:"used_chars"` |
| 115 | Suppressed string `json:"suppressed"` |
| 116 | ShadowHits []struct{ ID string } `json:"shadow_hits"` |
| 117 | } `json:"memory_recall"` |
| 118 | Event *struct { |
| 119 | Kind string `json:"kind"` |
| 120 | Text string `json:"text"` |
| 121 | Tool *struct { |
| 122 | Args string `json:"args"` |
| 123 | } `json:"tool"` |
| 124 | } `json:"event"` |
| 125 | } |
| 126 | used := make(map[string]bool, len(markers)) |
| 127 | // Pinned facts arrive via the stable prefix, before any recall: their |
| 128 | // markers count from the first record. |
| 129 | recalled := markersInPrefix |
| 130 | scanner := bufio.NewScanner(f) |
| 131 | scanner.Buffer(make([]byte, 0, 1024*1024), 64*1024*1024) |
| 132 | for scanner.Scan() { |
| 133 | var rec record |
| 134 | if err := json.Unmarshal(scanner.Bytes(), &rec); err != nil { |
| 135 | continue |
| 136 | } |
| 137 | if mr := rec.MemoryRecall; mr != nil { |
| 138 | if len(mr.Hits) > 0 { |
| 139 | stats.RecallEvents++ |
| 140 | stats.RecallHits += len(mr.Hits) |
| 141 | stats.RecallChars += mr.UsedChars |
| 142 | recalled = true |
| 143 | if len(mr.ShadowHits) > 0 && mr.ShadowHits[0].ID == mr.Hits[0].ID { |
| 144 | stats.ShadowAgree++ |
| 145 | } |
| 146 | } else if mr.Suppressed != "" { |
| 147 | stats.Suppressed++ |
| 148 | } |
| 149 | } |
| 150 | if !recalled || rec.Event == nil { |
| 151 | continue |
| 152 | } |
| 153 | var haystack string |
| 154 | if rec.Event.Tool != nil { |
| 155 | haystack = rec.Event.Tool.Args |
| 156 | } else if rec.Event.Kind == "text" || rec.Event.Kind == "message" { |
| 157 | haystack = rec.Event.Text |
| 158 | } |
| 159 | if haystack == "" { |
| 160 | continue |
| 161 | } |
| 162 | for _, marker := range markers { |
| 163 | if !used[marker] && strings.Contains(haystack, marker) { |
| 164 | used[marker] = true |
| 165 | } |
| 166 | } |
| 167 | } |
| 168 | stats.MarkersUsed = len(used) |
| 169 | return stats |
| 170 | } |
| 171 | |
| 172 | // renderMemoryShadow aggregates recall behavior across a suite run; empty when |
| 173 | // no run recalled anything and no task planted markers. |
| 174 | func renderMemoryShadow(results []result) string { |
| 175 | runs, recallRuns, hits, chars, suppressed, markersUsed, markersTotal := 0, 0, 0, 0, 0, 0, 0 |
| 176 | for _, r := range results { |
| 177 | runs++ |
| 178 | if r.MemoryRecallEvents > 0 { |
| 179 | recallRuns++ |
| 180 | } |
| 181 | hits += r.MemoryRecallHits |
| 182 | chars += r.MemoryRecallChars |
| 183 | suppressed += r.MemorySuppressed |
| 184 | markersUsed += r.MemoryMarkersUsed |
| 185 | markersTotal += len(r.MemoryMarkers) |
| 186 | } |
| 187 | if hits == 0 && markersTotal == 0 { |
| 188 | return "" |
| 189 | } |
| 190 | shadowAgree := 0 |
| 191 | for _, r := range results { |
| 192 | shadowAgree += r.MemoryShadowAgree |
| 193 | } |
| 194 | line := fmt.Sprintf("**Memory shadow** (%d runs): **recall fired** in %d runs · **hits** %d · **injected chars** %d", |
| 195 | runs, recallRuns, hits, chars) |
| 196 | if recallRuns > 0 { |
| 197 | line += fmt.Sprintf(" · **V2 top1 agree** %d/%d", shadowAgree, recallRuns) |
| 198 | } |
| 199 | if markersTotal > 0 { |
| 200 | line += fmt.Sprintf(" · **point-of-use** %d/%d markers", markersUsed, markersTotal) |
| 201 | } |
| 202 | if suppressed > 0 { |
| 203 | line += fmt.Sprintf(" · suppressed %d", suppressed) |
| 204 | } |
| 205 | return line + "\n\n" |
| 206 | } |
| 207 | |
| 208 | // memoryUtilitySection is the paired counterfactual readout for two arms of |
| 209 | // the same suite. The on-arm is whichever side recalled; pairing by task ID |
| 210 | // cancels task difficulty, so the delta is memory's contribution. |
| 211 | func memoryUtilitySection(pathA, pathB string) string { |
| 212 | a, errA := loadResults(pathA) |
| 213 | b, errB := loadResults(pathB) |
| 214 | if errA != nil || errB != nil { |
| 215 | return "" |
| 216 | } |
| 217 | on, off := a, b |
| 218 | if recallTotal(b) > recallTotal(a) { |
| 219 | on, off = b, a |
| 220 | } |
| 221 | if recallTotal(on) == 0 { |
| 222 | return "" |
| 223 | } |
| 224 | offByID := make(map[string]result, len(off)) |
| 225 | for _, r := range off { |
| 226 | offByID[r.ID] = r |
| 227 | } |
| 228 | paired, onPass, offPass := 0, 0, 0 |
| 229 | var helpful, harmful []string |
| 230 | overheadChars := 0 |
| 231 | for _, r := range on { |
| 232 | counterpart, ok := offByID[r.ID] |
| 233 | if !ok || r.Skipped || counterpart.Skipped { |
| 234 | continue |
| 235 | } |
| 236 | paired++ |
| 237 | overheadChars += r.MemoryRecallChars |
| 238 | if r.Passed { |
| 239 | onPass++ |
| 240 | } |
| 241 | if counterpart.Passed { |
| 242 | offPass++ |
| 243 | } |
| 244 | switch { |
| 245 | case r.Passed && !counterpart.Passed: |
| 246 | helpful = append(helpful, r.ID) |
| 247 | case !r.Passed && counterpart.Passed && r.MemoryRecallEvents > 0: |
| 248 | harmful = append(harmful, r.ID) |
| 249 | } |
| 250 | } |
| 251 | if paired == 0 { |
| 252 | return "" |
| 253 | } |
| 254 | var s strings.Builder |
| 255 | s.WriteString("\n## Memory utility (paired counterfactual)\n\n") |
| 256 | fmt.Fprintf(&s, "**Utility delta** %+.1fpp (on %d/%d vs off %d/%d, %d paired tasks) · ", |
| 257 | 100*(float64(onPass)-float64(offPass))/float64(paired), onPass, paired, offPass, paired, paired) |
| 258 | fmt.Fprintf(&s, "**helpful** %d · **harmful** %d · **avg injected chars** %d\n", len(helpful), len(harmful), overheadChars/paired) |
| 259 | if len(helpful) > 0 { |
| 260 | fmt.Fprintf(&s, "\n- helpful (on-pass, off-fail): %s\n", strings.Join(helpful, ", ")) |
| 261 | } |
| 262 | if len(harmful) > 0 { |
| 263 | fmt.Fprintf(&s, "- harmful (on-fail, off-pass, recall fired): %s\n", strings.Join(harmful, ", ")) |
| 264 | } |
| 265 | s.WriteString("\n<sub>Harmful attribution is paired, not judged: the same task passed without memory and failed with it while recall fired. Point-of-use markers live in the per-arm Memory shadow line.</sub>\n") |
| 266 | return s.String() |
| 267 | } |
| 268 | |
| 269 | func loadResults(path string) ([]result, error) { |
| 270 | data, err := os.ReadFile(path) |
| 271 | if err != nil { |
| 272 | return nil, err |
| 273 | } |
| 274 | var results []result |
| 275 | if err := json.Unmarshal(data, &results); err != nil { |
| 276 | return nil, err |
| 277 | } |
| 278 | return results, nil |
| 279 | } |
| 280 | |
| 281 | func recallTotal(results []result) int { |
| 282 | total := 0 |
| 283 | for _, r := range results { |
| 284 | total += r.MemoryRecallEvents |
| 285 | } |
| 286 | return total |
| 287 | } |
| 288 |