返回 DeepSeek-Reasonix
recovery_gc_test.go
根目录 / internal / agent / recovery_gc_test.go
1 package agent
2
3 import (
4 "errors"
5 "os"
6 "path/filepath"
7 "testing"
8 "time"
9
10 "reasonix/internal/provider"
11 "reasonix/internal/store"
12 )
13
14 // forkRecoveryBranch builds a real conflict-recovery branch: a diverged disk
15 // parent plus a stale in-memory session, forked through SaveRecoveryBranch —
16 // the exact artifacts GC will meet in the field.
17 func forkRecoveryBranch(t *testing.T, dir, name string) (parentPath, branchPath string, branchMsgs []provider.Message) {
18 t.Helper()
19 parentPath = filepath.Join(dir, name+".jsonl")
20 disk := NewSession("sys")
21 disk.Add(provider.Message{Role: provider.RoleUser, Content: "first"})
22 disk.Add(provider.Message{Role: provider.RoleAssistant, Content: "one"})
23 disk.Add(provider.Message{Role: provider.RoleUser, Content: "disk " + name})
24 if err := disk.Save(parentPath); err != nil {
25 t.Fatalf("Save parent: %v", err)
26 }
27 stale := NewSession("sys")
28 stale.Add(provider.Message{Role: provider.RoleUser, Content: "first"})
29 stale.Add(provider.Message{Role: provider.RoleAssistant, Content: "one"})
30 stale.Add(provider.Message{Role: provider.RoleUser, Content: "local " + name})
31 info, err := stale.SaveRecoveryBranch(RecoveryBranchOptions{OriginalPath: parentPath})
32 if err != nil {
33 t.Fatalf("SaveRecoveryBranch: %v", err)
34 }
35 return parentPath, info.Path, stale.Snapshot()
36 }
37
38 // coverBranchInParent rewrites the parent so it contains the branch's content
39 // plus later turns — the "original session went on and kept everything the
40 // fork preserved" shape that makes the fork redundant.
41 func coverBranchInParent(t *testing.T, parentPath string, branchMsgs []provider.Message) {
42 t.Helper()
43 merged, err := LoadSession(parentPath)
44 if err != nil {
45 t.Fatalf("Load covering parent: %v", err)
46 }
47 merged.Replace(append([]provider.Message(nil), branchMsgs...))
48 merged.Add(provider.Message{Role: provider.RoleAssistant, Content: "answered after recovery"})
49 if err := merged.SaveRewrite(parentPath); err != nil {
50 t.Fatalf("Save covering parent: %v", err)
51 }
52 }
53
54 func TestReclaimableRecoveryBranchesCollectsOnlyCoveredIdleForks(t *testing.T) {
55 dir := t.TempDir()
56 later := time.Now().Add(48 * time.Hour)
57
58 // Covered + idle + unleased: reclaimable.
59 _, covered, coveredMsgs := forkRecoveryBranch(t, dir, "covered")
60 coverBranchInParent(t, filepath.Join(dir, "covered.jsonl"), coveredMsgs)
61
62 // Diverged parent: the fork holds turns that exist nowhere else — kept.
63 forkRecoveryBranch(t, dir, "diverged")
64
65 // Continued on: one follow-up turn on the branch disqualifies it forever.
66 continuedParent, continuedBranch, continuedMsgs := forkRecoveryBranch(t, dir, "continued")
67 coverBranchInParent(t, continuedParent, continuedMsgs)
68 cont, err := LoadSession(continuedBranch)
69 if err != nil {
70 t.Fatalf("LoadSession continued branch: %v", err)
71 }
72 cont.Add(provider.Message{Role: provider.RoleAssistant, Content: "user kept chatting here"})
73 if err := cont.Save(continuedBranch); err != nil {
74 t.Fatalf("Save continued branch: %v", err)
75 }
76
77 got, err := ReclaimableRecoveryBranches(dir, later, RecoveryGCGracePeriod)
78 if err != nil {
79 t.Fatalf("ReclaimableRecoveryBranches: %v", err)
80 }
81 if len(got) != 1 || got[0] != covered {
82 t.Fatalf("reclaimable = %v, want only %q", got, covered)
83 }
84 }
85
86 func TestReclaimableRecoveryBranchesRespectsGraceLeaseAndMissingParent(t *testing.T) {
87 dir := t.TempDir()
88 later := time.Now().Add(48 * time.Hour)
89
90 parentPath, branchPath, branchMsgs := forkRecoveryBranch(t, dir, "guarded")
91 coverBranchInParent(t, parentPath, branchMsgs)
92
93 // Fresh fork inside the grace window: kept.
94 if got, err := ReclaimableRecoveryBranches(dir, time.Now(), RecoveryGCGracePeriod); err != nil || len(got) != 0 {
95 t.Fatalf("within grace = %v err=%v, want none", got, err)
96 }
97
98 // Lease held (by this very process): kept.
99 lease, err := TryAcquireSessionLease(branchPath)
100 if err != nil {
101 t.Fatalf("TryAcquireSessionLease: %v", err)
102 }
103 if !SessionLeaseHeld(branchPath) {
104 t.Fatal("SessionLeaseHeld = false while this process holds the lease")
105 }
106 if got, err := ReclaimableRecoveryBranches(dir, later, RecoveryGCGracePeriod); err != nil || len(got) != 0 {
107 t.Fatalf("with lease held = %v err=%v, want none", got, err)
108 }
109 lease.Release()
110 if SessionLeaseHeld(branchPath) {
111 t.Fatal("SessionLeaseHeld = true after release")
112 }
113
114 // Released + idle: reclaimable now.
115 if got, err := ReclaimableRecoveryBranches(dir, later, RecoveryGCGracePeriod); err != nil || len(got) != 1 || got[0] != branchPath {
116 t.Fatalf("after release = %v err=%v, want %q", got, err, branchPath)
117 }
118
119 // Parent gone: content is no longer covered anywhere — kept.
120 for _, artifact := range append([]string{parentPath}, store.SessionSidecarFiles(parentPath)...) {
121 if err := os.Remove(artifact); err != nil && !os.IsNotExist(err) {
122 t.Fatalf("remove parent artifact %s: %v", artifact, err)
123 }
124 }
125 if got, err := ReclaimableRecoveryBranches(dir, later, RecoveryGCGracePeriod); err != nil || len(got) != 0 {
126 t.Fatalf("parent missing = %v err=%v, want none", got, err)
127 }
128 }
129
130 func TestRecoveryBranchCoveredByParentReadsActualContent(t *testing.T) {
131 dir := t.TempDir()
132
133 _, divergedBranch, _ := forkRecoveryBranch(t, dir, "diverged-proof")
134 if RecoveryBranchCoveredByParent(divergedBranch, dir) {
135 t.Fatal("diverged parent reported as covering its recovery branch")
136 }
137
138 coveredParent, coveredBranch, coveredMsgs := forkRecoveryBranch(t, dir, "covered-proof")
139 coverBranchInParent(t, coveredParent, coveredMsgs)
140 if !RecoveryBranchCoveredByParent(coveredBranch, dir) {
141 t.Fatal("parent containing the recovery transcript did not cover the branch")
142 }
143
144 // Restore the fork metadata after continuing the transcript. This models a
145 // stale sidecar that still claims content_digest == recovery_digest even
146 // though the actual branch has changed.
147 staleMeta, ok, err := LoadBranchMeta(coveredBranch)
148 if err != nil || !ok {
149 t.Fatalf("LoadBranchMeta stale branch: ok=%v err=%v", ok, err)
150 }
151 continued, err := LoadSession(coveredBranch)
152 if err != nil {
153 t.Fatalf("LoadSession stale branch: %v", err)
154 }
155 continued.Add(provider.Message{Role: provider.RoleAssistant, Content: "continued after sidecar snapshot"})
156 if err := continued.Save(coveredBranch); err != nil {
157 t.Fatalf("Save continued branch: %v", err)
158 }
159 if err := SaveBranchMetaPreserveUpdated(coveredBranch, staleMeta); err != nil {
160 t.Fatalf("restore stale branch meta: %v", err)
161 }
162 if RecoveryBranchCoveredByParent(coveredBranch, dir) {
163 t.Fatal("stale sidecar authorized cleanup of changed branch content")
164 }
165
166 missingParent, missingBranch, missingMsgs := forkRecoveryBranch(t, dir, "missing-proof")
167 coverBranchInParent(t, missingParent, missingMsgs)
168 if !RecoveryBranchCoveredByParent(missingBranch, dir) {
169 t.Fatal("missing-parent fixture was not covered before parent removal")
170 }
171 for _, artifact := range append([]string{missingParent}, store.SessionSidecarFiles(missingParent)...) {
172 if err := os.Remove(artifact); err != nil && !os.IsNotExist(err) {
173 t.Fatalf("remove parent artifact %s: %v", artifact, err)
174 }
175 }
176 if RecoveryBranchCoveredByParent(missingBranch, dir) {
177 t.Fatal("missing parent reported as covering its recovery branch")
178 }
179 }
180
181 func TestRecoveryParentGuardBlocksRewindAfterValidation(t *testing.T) {
182 dir := schemaOneTempDir(t)
183 parentPath, branchPath, branchMsgs := forkRecoveryBranch(t, dir, "rewind-race")
184 coverBranchInParent(t, parentPath, branchMsgs)
185
186 guard, err := TryAcquireRecoveryParentGuard(branchPath, dir)
187 if err != nil {
188 t.Fatalf("TryAcquireRecoveryParentGuard: %v", err)
189 }
190 // Force the dangerous ordering: coverage validation has completed, then a
191 // concurrent rewind tries to take the parent's save lock before purge. The
192 // guard must keep that lock unavailable until the caller finishes deleting
193 // the redundant branch.
194 if lock, err := tryTakeSessionLockFile(store.SessionLockFile(parentPath)); !errors.Is(err, ErrSessionFileLockHeld) {
195 if lock != nil {
196 lock.Unlock()
197 }
198 guard.Release()
199 t.Fatalf("parent save lock after coverage validation = %v, want ErrSessionFileLockHeld", err)
200 }
201 guard.Release()
202
203 parent, err := LoadSession(parentPath)
204 if err != nil {
205 t.Fatalf("LoadSession parent: %v", err)
206 }
207 parent.Messages = append([]provider.Message(nil), parent.Messages[:1]...)
208 if err := parent.SaveRewrite(parentPath); err != nil {
209 t.Fatalf("SaveRewrite after guard release: %v", err)
210 }
211 if RecoveryBranchCoveredByParent(branchPath, dir) {
212 t.Fatal("rewound parent still reported as covering the recovery branch")
213 }
214 }
215
216 func TestRecoveryParentGuardRefusesInFlightParentRewrite(t *testing.T) {
217 dir := t.TempDir()
218 parentPath, branchPath, branchMsgs := forkRecoveryBranch(t, dir, "rewrite-busy")
219 coverBranchInParent(t, parentPath, branchMsgs)
220
221 unlock, err := lockSessionFile(parentPath)
222 if err != nil {
223 t.Fatalf("lockSessionFile: %v", err)
224 }
225 defer unlock()
226 if guard, err := TryAcquireRecoveryParentGuard(branchPath, dir); !errors.Is(err, ErrSessionLeaseHeld) {
227 if guard != nil {
228 guard.Release()
229 }
230 t.Fatalf("guard during parent rewrite err = %v, want ErrSessionLeaseHeld", err)
231 }
232 }
233
234 func ageRecoveryBranchForGC(t *testing.T, path string) {
235 t.Helper()
236 meta, ok, err := LoadBranchMeta(path)
237 if err != nil || !ok {
238 t.Fatalf("LoadBranchMeta: ok=%v err=%v", ok, err)
239 }
240 meta.UpdatedAt = time.Now().Add(-2 * RecoveryGCGracePeriod)
241 if err := SaveBranchMetaPreserveUpdated(path, meta); err != nil {
242 t.Fatalf("age recovery meta: %v", err)
243 }
244 }
245
246 func TestTrashReclaimableRecoveryBranchUsesRecoverableDesktopLayout(t *testing.T) {
247 dir := t.TempDir()
248 parentPath, branchPath, branchMsgs := forkRecoveryBranch(t, dir, "trash-covered")
249 coverBranchInParent(t, parentPath, branchMsgs)
250 ageRecoveryBranchForGC(t, branchPath)
251
252 if err := TrashReclaimableRecoveryBranch(branchPath, dir); err != nil {
253 t.Fatalf("TrashReclaimableRecoveryBranch: %v", err)
254 }
255 if _, err := os.Stat(branchPath); !os.IsNotExist(err) {
256 t.Fatalf("live recovery transcript still exists: %v", err)
257 }
258 if IsCleanupPending(branchPath) {
259 t.Fatal("cleanup-pending marker remained after completed trash move")
260 }
261 itemDir := filepath.Join(dir, recoveryTrashDir, filepath.Base(branchPath))
262 for _, path := range []string{
263 filepath.Join(itemDir, filepath.Base(branchPath)),
264 filepath.Join(itemDir, filepath.Base(BranchMetaPath(branchPath))),
265 filepath.Join(itemDir, filepath.Base(store.SessionEventLog(branchPath))),
266 filepath.Join(itemDir, recoveryTrashMetaFile),
267 } {
268 if _, err := os.Stat(path); err != nil {
269 t.Fatalf("recoverable trash artifact %s: %v", path, err)
270 }
271 }
272 if _, err := os.Stat(parentPath); err != nil {
273 t.Fatalf("parent session changed by recovery trash: %v", err)
274 }
275 if _, err := os.Stat(filepath.Join(itemDir, recoveryTrashPendingFile)); !os.IsNotExist(err) {
276 t.Fatalf("completed trash entry retained pending marker: %v", err)
277 }
278 }
279
280 func TestTrashRecoveryBranchCoveredByCanonicalCompactsLegacyChain(t *testing.T) {
281 dir := t.TempDir()
282 rootPath := filepath.Join(dir, "root.jsonl")
283 ancestorPath := filepath.Join(dir, "ancestor.jsonl")
284 canonicalPath := filepath.Join(dir, "canonical.jsonl")
285 root := NewSession("sys")
286 root.Add(provider.Message{Role: provider.RoleUser, Content: "root"})
287 if err := root.Save(rootPath); err != nil {
288 t.Fatal(err)
289 }
290 ancestor := NewSession("")
291 ancestor.Replace(root.Snapshot())
292 ancestor.Add(provider.Message{Role: provider.RoleAssistant, Content: "ancestor"})
293 if err := ancestor.Save(ancestorPath); err != nil {
294 t.Fatal(err)
295 }
296 if err := SaveBranchMetaPreserveUpdated(ancestorPath, BranchMeta{ID: "ancestor", Recovered: true, ParentID: "root", RecoveryDepth: 1}); err != nil {
297 t.Fatal(err)
298 }
299 canonical := NewSession("")
300 canonical.Replace(ancestor.Snapshot())
301 canonical.Add(provider.Message{Role: provider.RoleUser, Content: "continued"})
302 if err := canonical.Save(canonicalPath); err != nil {
303 t.Fatal(err)
304 }
305 if err := SaveBranchMetaPreserveUpdated(canonicalPath, BranchMeta{ID: "canonical", Recovered: true, ParentID: "ancestor", RecoveryDepth: 2}); err != nil {
306 t.Fatal(err)
307 }
308 if err := ReparentRecoveryCanonical(canonicalPath, "root", dir); err != nil {
309 t.Fatalf("ReparentRecoveryCanonical: %v", err)
310 }
311 if err := TrashRecoveryBranchCoveredBy(ancestorPath, canonicalPath, dir); err != nil {
312 t.Fatalf("TrashRecoveryBranchCoveredBy: %v", err)
313 }
314 meta, ok, err := LoadBranchMeta(canonicalPath)
315 if err != nil || !ok || meta.ParentID != "root" || meta.RecoveryDepth != 1 {
316 t.Fatalf("canonical meta = %+v ok=%v err=%v", meta, ok, err)
317 }
318 if _, err := os.Stat(rootPath); err != nil {
319 t.Fatalf("root was removed: %v", err)
320 }
321 if _, err := os.Stat(canonicalPath); err != nil {
322 t.Fatalf("canonical was removed: %v", err)
323 }
324 if _, err := os.Stat(ancestorPath); !os.IsNotExist(err) {
325 t.Fatalf("ancestor remained live: %v", err)
326 }
327 }
328
329 func TestSetRecoveryPreferredKeepsExactlyOneChoice(t *testing.T) {
330 dir := t.TempDir()
331 paths := []string{filepath.Join(dir, "left.jsonl"), filepath.Join(dir, "right.jsonl")}
332 for _, path := range paths {
333 session := NewSession("sys")
334 session.Add(provider.Message{Role: provider.RoleUser, Content: path})
335 if err := session.Save(path); err != nil {
336 t.Fatal(err)
337 }
338 if err := SaveBranchMeta(path, BranchMeta{ID: BranchID(path), Recovered: true, ParentID: "root"}); err != nil {
339 t.Fatal(err)
340 }
341 }
342 if err := SetRecoveryPreferred(paths, paths[0]); err != nil {
343 t.Fatal(err)
344 }
345 if err := SetRecoveryPreferred(paths, paths[1]); err != nil {
346 t.Fatal(err)
347 }
348 for index, path := range paths {
349 meta, ok, err := LoadBranchMeta(path)
350 if err != nil || !ok {
351 t.Fatalf("meta %q ok=%v err=%v", path, ok, err)
352 }
353 if meta.RecoveryPreferred != (index == 1) {
354 t.Fatalf("preferred[%d] = %v", index, meta.RecoveryPreferred)
355 }
356 if index == 1 && !RecoveryPreferenceCurrent(path, meta) {
357 t.Fatal("saved preference fingerprint was not current")
358 }
359 }
360 chosen, err := LoadSession(paths[1])
361 if err != nil {
362 t.Fatal(err)
363 }
364 chosen.Add(provider.Message{Role: provider.RoleAssistant, Content: "changed"})
365 if err := chosen.SaveSnapshot(paths[1]); err != nil {
366 t.Fatal(err)
367 }
368 meta, _, _ := LoadBranchMeta(paths[1])
369 if RecoveryPreferenceCurrent(paths[1], meta) {
370 t.Fatal("content change must invalidate the explicit preference")
371 }
372 }
373
374 func TestSetRecoveryPreferredAllowsOriginalLineageMember(t *testing.T) {
375 dir := t.TempDir()
376 root := filepath.Join(dir, "root.jsonl")
377 branch := filepath.Join(dir, "branch.jsonl")
378 for _, path := range []string{root, branch} {
379 session := NewSession("sys")
380 session.Add(provider.Message{Role: provider.RoleUser, Content: filepath.Base(path)})
381 if err := session.Save(path); err != nil {
382 t.Fatal(err)
383 }
384 }
385 if err := SaveBranchMeta(root, BranchMeta{ID: BranchID(root)}); err != nil {
386 t.Fatal(err)
387 }
388 if err := SaveBranchMeta(branch, BranchMeta{ID: BranchID(branch), Recovered: true, ParentID: BranchID(root)}); err != nil {
389 t.Fatal(err)
390 }
391 if err := SetRecoveryPreferred([]string{root, branch}, root); err != nil {
392 t.Fatal(err)
393 }
394 rootMeta, ok, err := LoadBranchMeta(root)
395 if err != nil || !ok || !rootMeta.RecoveryPreferred || !RecoveryPreferenceCurrent(root, rootMeta) {
396 t.Fatalf("original preference ok=%v err=%v meta=%+v", ok, err, rootMeta)
397 }
398 branchMeta, _, _ := LoadBranchMeta(branch)
399 if branchMeta.RecoveryPreferred {
400 t.Fatal("choosing the original left a recovery leaf preferred")
401 }
402 }
403
404 func TestTrashReclaimableRecoveryBranchEnforcesGraceAtFinalGuard(t *testing.T) {
405 dir := t.TempDir()
406 parentPath, branchPath, branchMsgs := forkRecoveryBranch(t, dir, "trash-fresh")
407 coverBranchInParent(t, parentPath, branchMsgs)
408
409 if err := TrashReclaimableRecoveryBranch(branchPath, dir); !errors.Is(err, ErrRecoveryBranchNotIdle) {
410 t.Fatalf("fresh recovery trash err = %v, want ErrRecoveryBranchNotIdle", err)
411 }
412 if _, err := os.Stat(branchPath); err != nil {
413 t.Fatalf("fresh recovery branch was not preserved: %v", err)
414 }
415 }
416
417 func TestInterruptedRecoveryTrashStageReconcilesBeforePublication(t *testing.T) {
418 dir := t.TempDir()
419 _, branchPath, _ := forkRecoveryBranch(t, dir, "staged-crash")
420 key := filepath.Base(branchPath)
421 stageDir, err := reserveRecoveryTrashStage(dir)
422 if err != nil {
423 t.Fatalf("reserveRecoveryTrashStage: %v", err)
424 }
425 if err := prepareRecoveryTrashStage(branchPath, key, stageDir); err != nil {
426 t.Fatalf("prepareRecoveryTrashStage: %v", err)
427 }
428 if IsCleanupPending(branchPath) {
429 t.Fatal("live cleanup marker should not be used by the staging protocol")
430 }
431 if _, err := os.Stat(branchPath); !os.IsNotExist(err) {
432 t.Fatalf("live transcript still exists after staging: %v", err)
433 }
434 if store.IsSessionTranscriptName(filepath.Base(stageDir)) {
435 t.Fatalf("staging directory is Desktop-visible by name: %s", stageDir)
436 }
437 for _, target := range []string{
438 filepath.Join(stageDir, key),
439 filepath.Join(stageDir, recoveryTrashPendingFile),
440 } {
441 if _, err := os.Stat(target); err != nil {
442 t.Fatalf("staged recovery artifact %s: %v", target, err)
443 }
444 }
445 if _, err := os.Stat(filepath.Join(stageDir, recoveryTrashMetaFile)); !os.IsNotExist(err) {
446 t.Fatalf("incomplete stage became Desktop-visible through trash metadata: %v", err)
447 }
448
449 // Simulate a process crash after the transcript rename but before any
450 // sidecars moved. Startup reconciliation must finish the hidden stage and
451 // publish one complete Desktop trash item without the hard-delete callback.
452 calledFallback := false
453 if err := ReconcileCleanupPending(dir, func(CleanupPendingInfo) error {
454 calledFallback = true
455 return errors.New("hard-delete fallback must not run")
456 }); err != nil {
457 t.Fatalf("ReconcileCleanupPending: %v", err)
458 }
459 if calledFallback {
460 t.Fatal("recovery trash stage reached hard-delete fallback")
461 }
462 if _, err := os.Stat(stageDir); !os.IsNotExist(err) {
463 t.Fatalf("staging directory remained after publication: %v", err)
464 }
465 itemDir := filepath.Join(dir, recoveryTrashDir, key)
466 for _, target := range []string{
467 filepath.Join(itemDir, key),
468 filepath.Join(itemDir, filepath.Base(BranchMetaPath(branchPath))),
469 filepath.Join(itemDir, filepath.Base(store.SessionEventLog(branchPath))),
470 filepath.Join(itemDir, recoveryTrashMetaFile),
471 } {
472 if _, err := os.Stat(target); err != nil {
473 t.Fatalf("reconciled recovery artifact %s: %v", target, err)
474 }
475 }
476 if _, err := os.Stat(filepath.Join(itemDir, recoveryTrashPendingFile)); !os.IsNotExist(err) {
477 t.Fatalf("published trash entry retained pending marker: %v", err)
478 }
479 }
480
481 func TestReconcileCleanupPendingFinishesRecoveryTrashWithoutHardDeleteCallback(t *testing.T) {
482 dir := t.TempDir()
483 _, branchPath, _ := forkRecoveryBranch(t, dir, "trash-interrupted")
484 key := filepath.Base(branchPath)
485 itemName, itemDir, err := reserveRecoveryTrashItemDir(dir, key)
486 if err != nil {
487 t.Fatalf("reserveRecoveryTrashItemDir: %v", err)
488 }
489 if err := prepareRecoveryTrashEntry(branchPath, key, itemDir); err != nil {
490 t.Fatalf("prepare trash entry before simulated crash: %v", err)
491 }
492 if err := MarkCleanupPending(branchPath, recoveryTrashOperationPrefix+itemName); err != nil {
493 t.Fatalf("MarkCleanupPending: %v", err)
494 }
495
496 calledFallback := false
497 if err := ReconcileCleanupPending(dir, func(CleanupPendingInfo) error {
498 calledFallback = true
499 return errors.New("hard-delete fallback must not run")
500 }); err != nil {
501 t.Fatalf("ReconcileCleanupPending: %v", err)
502 }
503 if calledFallback {
504 t.Fatal("recovery-trash marker reached hard-delete fallback")
505 }
506 if IsCleanupPending(branchPath) {
507 t.Fatal("cleanup-pending marker remained after reconciliation")
508 }
509 if _, err := os.Stat(filepath.Join(itemDir, recoveryTrashMetaFile)); err != nil {
510 t.Fatalf("reconciled trash metadata: %v", err)
511 }
512 }
513
513 lines GO