| 1 | package agent |
| 2 | |
| 3 | // Crash-injection pins for each durable save boundary. A refactor that |
| 4 | // changes an outcome must update the matching test in the same commit. |
| 5 | |
| 6 | import ( |
| 7 | "os" |
| 8 | "path/filepath" |
| 9 | "slices" |
| 10 | "strings" |
| 11 | "testing" |
| 12 | |
| 13 | "reasonix/internal/fileutil" |
| 14 | "reasonix/internal/provider" |
| 15 | "reasonix/internal/store" |
| 16 | ) |
| 17 | |
| 18 | // crashAt installs a fileutil.CrashPoint that panics the first time op is |
| 19 | // about to touch path. The returned restore must be deferred; the returned |
| 20 | // channel receives the op-path pair when the crash fires. |
| 21 | func crashAt(t *testing.T, op, path string) (fired <-chan struct{}, restore func()) { |
| 22 | return crashAtOccurrence(t, op, path, 1) |
| 23 | } |
| 24 | |
| 25 | func crashAtOccurrence(t *testing.T, op, path string, occurrence int) (fired <-chan struct{}, restore func()) { |
| 26 | t.Helper() |
| 27 | firedCh := make(chan struct{}, 1) |
| 28 | prev := fileutil.CrashPoint |
| 29 | seen := 0 |
| 30 | fileutil.CrashPoint = func(firedOp, firedPath string) { |
| 31 | if firedOp != op || firedPath != path { |
| 32 | return |
| 33 | } |
| 34 | seen++ |
| 35 | if seen != occurrence { |
| 36 | return |
| 37 | } |
| 38 | select { |
| 39 | case firedCh <- struct{}{}: |
| 40 | default: |
| 41 | } |
| 42 | panic(crashInjected{op: firedOp, path: firedPath}) |
| 43 | } |
| 44 | return firedCh, func() { fileutil.CrashPoint = prev } |
| 45 | } |
| 46 | |
| 47 | type crashInjected struct { |
| 48 | op string |
| 49 | path string |
| 50 | } |
| 51 | |
| 52 | func (c crashInjected) Error() string { return "crash injected at " + c.op + " (" + c.path + ")" } |
| 53 | |
| 54 | // saveCrashing runs fn and converts an injected crash panic into a |
| 55 | // crashInjected error, so callers can assert the crash fired without the |
| 56 | // panic unwinding the test. |
| 57 | func saveCrashing(fn func()) (crash error) { |
| 58 | defer func() { |
| 59 | if r := recover(); r != nil { |
| 60 | if ci, ok := r.(crashInjected); ok { |
| 61 | crash = ci |
| 62 | return |
| 63 | } |
| 64 | panic(r) |
| 65 | } |
| 66 | }() |
| 67 | fn() |
| 68 | return nil |
| 69 | } |
| 70 | |
| 71 | func messageCount(t *testing.T, s *Session) int { |
| 72 | t.Helper() |
| 73 | return len(s.Snapshot()) |
| 74 | } |
| 75 | |
| 76 | // TestCrashAtWALAppendKeepsPreviousCheckpointUsable pins the first boundary: |
| 77 | // a crash before the WAL record lands leaves the previous checkpoint as the |
| 78 | // only forward progress. A reload observes the old transcript, and the next |
| 79 | // save replays the append without duplicating or losing messages. |
| 80 | // (Message counts include the leading system message.) |
| 81 | func TestCrashAtWALAppendKeepsPreviousCheckpointUsable(t *testing.T) { |
| 82 | path := schemaOneSessionPath(t, "session.jsonl") |
| 83 | s := NewSession("system") |
| 84 | s.Add(userMessage("first")) |
| 85 | if err := s.SaveSnapshot(path); err != nil { |
| 86 | t.Fatalf("initial save: %v", err) |
| 87 | } |
| 88 | |
| 89 | s.Add(userMessage("second")) |
| 90 | _, restore := crashAt(t, "wal-append", store.SessionEventLog(path)) |
| 91 | crash := saveCrashing(func() { _ = s.SaveSnapshot(path) }) |
| 92 | restore() |
| 93 | if crash == nil { |
| 94 | t.Fatal("save must crash at the wal-append boundary") |
| 95 | } |
| 96 | |
| 97 | reloaded, err := LoadSession(path) |
| 98 | if err != nil { |
| 99 | t.Fatalf("reload after wal-append crash: %v", err) |
| 100 | } |
| 101 | if got := messageCount(t, reloaded); got != 2 { |
| 102 | t.Fatalf("reload after wal-append crash = %d messages, want 2 (previous checkpoint)", got) |
| 103 | } |
| 104 | |
| 105 | // The retry save must heal forward without duplicating events. |
| 106 | reloaded.Add(userMessage("second")) |
| 107 | if err := reloaded.SaveSnapshot(path); err != nil { |
| 108 | t.Fatalf("retry save: %v", err) |
| 109 | } |
| 110 | final, err := LoadSession(path) |
| 111 | if err != nil { |
| 112 | t.Fatalf("final reload: %v", err) |
| 113 | } |
| 114 | if got := messageCount(t, final); got != 3 { |
| 115 | t.Fatalf("final reload = %d messages, want 3", got) |
| 116 | } |
| 117 | if got := countEventLogRecords(t, path); got != 2 { |
| 118 | t.Fatalf("event log = %d records after retry, want 2 (replace + append)", got) |
| 119 | } |
| 120 | } |
| 121 | |
| 122 | // TestCrashAtCheckpointWriteLeavesEventLogAuthoritative pins the WAL-first |
| 123 | // ordering on the full-rewrite path (first save, repairs, compactions): a |
| 124 | // crash after the WAL replace record landed but before the .jsonl checkpoint |
| 125 | // rename still exposes the full transcript on reload, because the event log |
| 126 | // is authoritative. The compatibility checkpoint never comes to exist. |
| 127 | func TestCrashAtCheckpointWriteLeavesEventLogAuthoritative(t *testing.T) { |
| 128 | path := schemaOneSessionPath(t, "session.jsonl") |
| 129 | s := NewSession("system") |
| 130 | s.Add(userMessage("first")) |
| 131 | s.Add(userMessage("second")) |
| 132 | _, restore := crashAt(t, "session-checkpoint", path) |
| 133 | crash := saveCrashing(func() { _ = s.SaveSnapshot(path) }) |
| 134 | restore() |
| 135 | if crash == nil { |
| 136 | t.Fatal("save must crash at the session-checkpoint boundary") |
| 137 | } |
| 138 | |
| 139 | if _, err := os.Lstat(path); !os.IsNotExist(err) { |
| 140 | t.Fatalf("checkpoint must not exist after crash before its rename (err=%v)", err) |
| 141 | } |
| 142 | reloaded, err := LoadSession(path) |
| 143 | if err != nil { |
| 144 | t.Fatalf("reload after checkpoint crash: %v", err) |
| 145 | } |
| 146 | if got := messageCount(t, reloaded); got != 3 { |
| 147 | t.Fatalf("reload = %d messages, want 3 (event log authoritative)", got) |
| 148 | } |
| 149 | // The follow-up save publishes the missing checkpoint without duplicating |
| 150 | // WAL history. |
| 151 | if err := reloaded.SaveSnapshot(path); err != nil { |
| 152 | t.Fatalf("follow-up save: %v", err) |
| 153 | } |
| 154 | if got := countEventLogRecords(t, path); got != 1 { |
| 155 | t.Fatalf("event log = %d records, want 1 (no duplicate replace)", got) |
| 156 | } |
| 157 | checkpointBytes, err := os.ReadFile(path) |
| 158 | if err != nil { |
| 159 | t.Fatalf("read checkpoint: %v", err) |
| 160 | } |
| 161 | if got := strings.Count(string(checkpointBytes), "\n"); got != 3 { |
| 162 | t.Fatalf("checkpoint = %d lines, want 3", got) |
| 163 | } |
| 164 | } |
| 165 | |
| 166 | // TestCrashAtRevisionLedgerHealsOnNextSave pins the ledger-lag window: a |
| 167 | // crash after the transcript landed but before the revision ledger recorded |
| 168 | // the new digest leaves a stale ledger. The next same-content save must heal |
| 169 | // the ledger via the ledgerStale path instead of appending new events. |
| 170 | func TestCrashAtRevisionLedgerHealsOnNextSave(t *testing.T) { |
| 171 | path := schemaOneSessionPath(t, "session.jsonl") |
| 172 | s := NewSession("system") |
| 173 | s.Add(userMessage("first")) |
| 174 | if err := s.SaveSnapshot(path); err != nil { |
| 175 | t.Fatalf("initial save: %v", err) |
| 176 | } |
| 177 | recordsBefore := countEventLogRecords(t, path) |
| 178 | |
| 179 | s.Add(userMessage("second")) |
| 180 | // The first branch-meta write invalidates the listing projection; the |
| 181 | // second is the revision-ledger commit we want to interrupt. |
| 182 | _, restore := crashAtOccurrence(t, "branch-meta", BranchMetaPath(path), 2) |
| 183 | crash := saveCrashing(func() { _ = s.SaveSnapshot(path) }) |
| 184 | restore() |
| 185 | if crash == nil { |
| 186 | t.Fatal("save must crash at the branch-meta boundary") |
| 187 | } |
| 188 | |
| 189 | // Transcript and ledger have diverged: the event log replay observes three |
| 190 | // messages while the ledger still stamps the two-message digest. |
| 191 | reloaded, err := LoadSession(path) |
| 192 | if err != nil { |
| 193 | t.Fatalf("reload after ledger crash: %v", err) |
| 194 | } |
| 195 | if got := messageCount(t, reloaded); got != 3 { |
| 196 | t.Fatalf("reload = %d messages, want 3", got) |
| 197 | } |
| 198 | if preview, turns, ok := SessionPreviewCached(path); ok { |
| 199 | t.Fatalf("stale projection survived interrupted ledger commit: preview=%q turns=%d", preview, turns) |
| 200 | } |
| 201 | |
| 202 | // A same-content save must heal the ledger without new WAL records. |
| 203 | if err := reloaded.SaveSnapshot(path); err != nil { |
| 204 | t.Fatalf("healing save: %v", err) |
| 205 | } |
| 206 | if got := countEventLogRecords(t, path); got != recordsBefore+1 { |
| 207 | t.Fatalf("event log = %d records after healing save, want %d (no extra records)", got, recordsBefore+1) |
| 208 | } |
| 209 | healed, err := LoadSession(path) |
| 210 | if err != nil { |
| 211 | t.Fatalf("reload after healing: %v", err) |
| 212 | } |
| 213 | if got := messageCount(t, healed); got != 3 { |
| 214 | t.Fatalf("healed reload = %d messages, want 3", got) |
| 215 | } |
| 216 | meta, ok, err := LoadBranchMeta(path) |
| 217 | if err != nil || !ok { |
| 218 | t.Fatalf("load healed branch meta: ok=%v err=%v", ok, err) |
| 219 | } |
| 220 | if meta.ContentDigest == "" { |
| 221 | t.Fatal("healed ledger must stamp the current content digest") |
| 222 | } |
| 223 | if preview, turns, ok := SessionPreviewCached(path); !ok || preview != "first" || turns != 2 { |
| 224 | t.Fatalf("healed listing projection = (%q,%d,%v), want (%q,2,true)", preview, turns, ok, "first") |
| 225 | } |
| 226 | } |
| 227 | |
| 228 | // TestCrashAtEventIndexKeepsSaveDurable pins the derived-index ordering: the |
| 229 | // event index is a pure accelerator, so a crash at its boundary loses nothing |
| 230 | // authoritative. A reload observes the new transcript and the next save |
| 231 | // succeeds without event-log duplication. |
| 232 | func TestCrashAtEventIndexKeepsSaveDurable(t *testing.T) { |
| 233 | path := schemaOneSessionPath(t, "session.jsonl") |
| 234 | s := NewSession("system") |
| 235 | s.Add(userMessage("first")) |
| 236 | if err := s.SaveSnapshot(path); err != nil { |
| 237 | t.Fatalf("initial save: %v", err) |
| 238 | } |
| 239 | |
| 240 | s.Add(userMessage("second")) |
| 241 | _, restore := crashAt(t, "event-index", store.SessionEventIndex(path)) |
| 242 | crash := saveCrashing(func() { _ = s.SaveSnapshot(path) }) |
| 243 | restore() |
| 244 | if crash == nil { |
| 245 | t.Fatal("save must crash at the event-index boundary") |
| 246 | } |
| 247 | |
| 248 | reloaded, err := LoadSession(path) |
| 249 | if err != nil { |
| 250 | t.Fatalf("reload after event-index crash: %v", err) |
| 251 | } |
| 252 | if got := messageCount(t, reloaded); got != 3 { |
| 253 | t.Fatalf("reload = %d messages, want 3 (index is derived)", got) |
| 254 | } |
| 255 | if err := reloaded.SaveSnapshot(path); err != nil { |
| 256 | t.Fatalf("follow-up save: %v", err) |
| 257 | } |
| 258 | if got := countEventLogRecords(t, path); got != 2 { |
| 259 | t.Fatalf("event log = %d records, want 2 (no duplication from index loss)", got) |
| 260 | } |
| 261 | } |
| 262 | |
| 263 | // TestSavedSessionSidecarSetBaseline pins the sidecar inventory a healthy |
| 264 | // first save produces. Every sidecar listed here is load-bearing for some |
| 265 | // reader; the refactor may shrink this set (with a migration story for each |
| 266 | // removed file) but must never grow it silently. |
| 267 | func TestSavedSessionSidecarSetBaseline(t *testing.T) { |
| 268 | path := filepath.Join(t.TempDir(), "session.jsonl") |
| 269 | s := NewSession("system") |
| 270 | s.Add(userMessage("first")) |
| 271 | if err := s.SaveSnapshot(path); err != nil { |
| 272 | t.Fatalf("save: %v", err) |
| 273 | } |
| 274 | |
| 275 | want := []string{ |
| 276 | filepath.Base(path), // compatibility checkpoint |
| 277 | filepath.Base(store.SessionEventLog(path)), // authoritative event log |
| 278 | filepath.Base(store.SessionEventIndex(path)), |
| 279 | filepath.Base(store.SessionDisplayIndex(path)), |
| 280 | filepath.Base(BranchMetaPath(path)), |
| 281 | } |
| 282 | found := map[string]bool{} |
| 283 | entries, err := os.ReadDir(filepath.Dir(path)) |
| 284 | if err != nil { |
| 285 | t.Fatalf("read session dir: %v", err) |
| 286 | } |
| 287 | for _, e := range entries { |
| 288 | found[e.Name()] = true |
| 289 | } |
| 290 | for _, name := range want { |
| 291 | if !found[name] { |
| 292 | t.Errorf("missing expected sidecar %q after first save", name) |
| 293 | } |
| 294 | } |
| 295 | for name := range found { |
| 296 | if !slices.Contains(want, name) { |
| 297 | // Legacy .lock sidecars still outlive a save; the refactor removes them. |
| 298 | if strings.HasSuffix(name, ".lock") { |
| 299 | continue |
| 300 | } |
| 301 | t.Errorf("unexpected extra sidecar %q after first save", name) |
| 302 | } |
| 303 | } |
| 304 | } |
| 305 | |
| 306 | func userMessage(content string) provider.Message { |
| 307 | return provider.Message{Role: provider.RoleUser, Content: content} |
| 308 | } |
| 309 | |
| 310 | func countEventLogRecords(t *testing.T, sessionPath string) int { |
| 311 | t.Helper() |
| 312 | b, err := os.ReadFile(store.SessionEventLog(sessionPath)) |
| 313 | if err != nil { |
| 314 | if os.IsNotExist(err) { |
| 315 | return 0 |
| 316 | } |
| 317 | t.Fatalf("read event log: %v", err) |
| 318 | } |
| 319 | count := 0 |
| 320 | for line := range strings.SplitSeq(string(b), "\n") { |
| 321 | if strings.TrimSpace(line) != "" { |
| 322 | count++ |
| 323 | } |
| 324 | } |
| 325 | return count |
| 326 | } |
| 327 |