| 1 | package historycatalog |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "database/sql" |
| 6 | "os" |
| 7 | "strings" |
| 8 | |
| 9 | "reasonix/internal/config" |
| 10 | ) |
| 11 | |
| 12 | const ( |
| 13 | // DefaultMaxBytes caps the disposable index; session files stay authoritative. |
| 14 | DefaultMaxBytes = 256 << 20 |
| 15 | // rebuildOversizeFactor: past this multiple of the cap a background |
| 16 | // wipe+rebuild beats evicting nearly every session, and applies tool-text |
| 17 | // truncation to legacy rows (#8717). |
| 18 | rebuildOversizeFactor = 2 |
| 19 | // evictTargetPercent: reclaim down to this share of the cap so the next |
| 20 | // persist batch does not immediately re-trigger eviction. |
| 21 | evictTargetPercent = 80 |
| 22 | maxEvictRounds = 8 |
| 23 | ) |
| 24 | |
| 25 | func resolveMaxBytes(option int64, configuredMB int) int64 { |
| 26 | if option > 0 { |
| 27 | return option |
| 28 | } |
| 29 | if configuredMB > 0 { |
| 30 | return int64(configuredMB) << 20 |
| 31 | } |
| 32 | return DefaultMaxBytes |
| 33 | } |
| 34 | |
| 35 | func configuredMaxMB() int { |
| 36 | return config.HistorySearchMaxMB() |
| 37 | } |
| 38 | |
| 39 | func historyDBFileSize(path string) int64 { |
| 40 | var total int64 |
| 41 | for _, candidate := range []string{path, path + "-wal"} { |
| 42 | if info, err := os.Stat(candidate); err == nil { |
| 43 | total += info.Size() |
| 44 | } |
| 45 | } |
| 46 | return total |
| 47 | } |
| 48 | |
| 49 | // governSize enforces the on-disk size cap. Best-effort: failures surface via |
| 50 | // status.LastError and the next reconcile tick retries. |
| 51 | func (c *Catalog) governSize(ctx context.Context) { |
| 52 | if c.opts.MaxBytes <= 0 || c.opts.InMemory || strings.TrimSpace(c.opts.Path) == "" { |
| 53 | return |
| 54 | } |
| 55 | size := historyDBFileSize(c.opts.Path) |
| 56 | if size <= c.opts.MaxBytes { |
| 57 | return |
| 58 | } |
| 59 | // Live WAL bytes are not reclaimable; fold them before measuring again. |
| 60 | _, _ = c.db.ExecContext(ctx, `PRAGMA wal_checkpoint(TRUNCATE)`) |
| 61 | size = historyDBFileSize(c.opts.Path) |
| 62 | if size <= c.opts.MaxBytes { |
| 63 | return |
| 64 | } |
| 65 | if size > rebuildOversizeFactor*c.opts.MaxBytes { |
| 66 | c.wipeForRebuild(ctx) |
| 67 | return |
| 68 | } |
| 69 | c.evictToTarget(ctx, size) |
| 70 | } |
| 71 | |
| 72 | func wipeProjectionRows(ctx context.Context, tx *sql.Tx) error { |
| 73 | for _, statement := range []string{`DELETE FROM history_fts`, `DELETE FROM history_documents`, |
| 74 | `DELETE FROM history_sources`, `DELETE FROM history_roots`} { |
| 75 | if _, err := tx.ExecContext(ctx, statement); err != nil { |
| 76 | return err |
| 77 | } |
| 78 | } |
| 79 | return nil |
| 80 | } |
| 81 | |
| 82 | // reclaimDiskSpace collapses FTS delete tombstones and returns freed pages to |
| 83 | // the OS. auto_vacuum never stuck on these pooled handles, so |
| 84 | // incremental_vacuum is a no-op here; VACUUM is the only working reclaim, and |
| 85 | // in WAL mode its pages land in the WAL first, so the checkpoint follows it. |
| 86 | func (c *Catalog) reclaimDiskSpace(ctx context.Context) { |
| 87 | _, _ = c.db.ExecContext(ctx, `INSERT INTO history_fts(history_fts) VALUES('optimize')`) |
| 88 | _, _ = c.db.ExecContext(ctx, `VACUUM`) |
| 89 | _, _ = c.db.ExecContext(ctx, `PRAGMA wal_checkpoint(TRUNCATE)`) |
| 90 | } |
| 91 | |
| 92 | // wipeForRebuild drops the whole projection so roots rescan under current |
| 93 | // indexing rules; source session files are never touched. |
| 94 | func (c *Catalog) wipeForRebuild(ctx context.Context) { |
| 95 | tx, err := c.db.BeginTx(ctx, nil) |
| 96 | if err != nil { |
| 97 | c.setError(err) |
| 98 | return |
| 99 | } |
| 100 | if err := wipeProjectionRows(ctx, tx); err != nil { |
| 101 | _ = tx.Rollback() |
| 102 | c.setError(err) |
| 103 | return |
| 104 | } |
| 105 | revision, err := bump(ctx, tx) |
| 106 | if err != nil { |
| 107 | _ = tx.Rollback() |
| 108 | c.setError(err) |
| 109 | return |
| 110 | } |
| 111 | if err := tx.Commit(); err != nil { |
| 112 | c.setError(err) |
| 113 | return |
| 114 | } |
| 115 | c.reclaimDiskSpace(ctx) |
| 116 | c.markAllRootsDirty() |
| 117 | c.publish(revision, nil, "rebuild-oversize") |
| 118 | } |
| 119 | |
| 120 | func (c *Catalog) evictToTarget(ctx context.Context, size int64) { |
| 121 | target := c.opts.MaxBytes * evictTargetPercent / 100 |
| 122 | for range maxEvictRounds { |
| 123 | if size <= target { |
| 124 | return |
| 125 | } |
| 126 | evicted, err := c.evictOldestBatch(ctx, size, size-target) |
| 127 | if err != nil { |
| 128 | c.setError(err) |
| 129 | return |
| 130 | } |
| 131 | if evicted == 0 { |
| 132 | return |
| 133 | } |
| 134 | c.reclaimDiskSpace(ctx) |
| 135 | size = historyDBFileSize(c.opts.Path) |
| 136 | } |
| 137 | } |
| 138 | |
| 139 | // evictOldestBatch drops index rows for the least-recently-active sessions |
| 140 | // whose estimated footprint covers overage (always at least one). The |
| 141 | // history_sources row survives as health='evicted' so an unchanged file is not |
| 142 | // indexed back in on the next rescan; it fully re-indexes on its next content |
| 143 | // change. Source session files are never touched. |
| 144 | func (c *Catalog) evictOldestBatch(ctx context.Context, size, overage int64) (int, error) { |
| 145 | rows, err := c.db.QueryContext(ctx, `SELECT s.path,COALESCE(SUM(d.token_count),0) |
| 146 | FROM history_sources s JOIN history_documents d ON d.source_path=s.path |
| 147 | GROUP BY s.path ORDER BY s.last_activity_at ASC,s.path ASC`) |
| 148 | if err != nil { |
| 149 | return 0, err |
| 150 | } |
| 151 | type candidate struct { |
| 152 | path string |
| 153 | tokens int64 |
| 154 | } |
| 155 | candidates := []candidate{} |
| 156 | var totalTokens int64 |
| 157 | for rows.Next() { |
| 158 | var cand candidate |
| 159 | if err := rows.Scan(&cand.path, &cand.tokens); err != nil { |
| 160 | _ = rows.Close() |
| 161 | return 0, err |
| 162 | } |
| 163 | candidates = append(candidates, cand) |
| 164 | totalTokens += cand.tokens |
| 165 | } |
| 166 | if err := rows.Err(); err != nil { |
| 167 | _ = rows.Close() |
| 168 | return 0, err |
| 169 | } |
| 170 | _ = rows.Close() |
| 171 | if len(candidates) == 0 || totalTokens <= 0 { |
| 172 | return 0, nil |
| 173 | } |
| 174 | // Self-calibrating estimate of file bytes per indexed token (fixed overhead |
| 175 | // included), so the prefix errs towards evicting too few per round rather |
| 176 | // than too many; the round loop re-measures and converges. |
| 177 | bytesPerToken := float64(size) / float64(totalTokens) |
| 178 | batch := []string{} |
| 179 | covered := 0.0 |
| 180 | for _, cand := range candidates { |
| 181 | batch = append(batch, cand.path) |
| 182 | covered += float64(cand.tokens) * bytesPerToken |
| 183 | if covered >= float64(overage) { |
| 184 | break |
| 185 | } |
| 186 | } |
| 187 | tx, err := c.db.BeginTx(ctx, nil) |
| 188 | if err != nil { |
| 189 | return 0, err |
| 190 | } |
| 191 | for _, path := range batch { |
| 192 | for _, statement := range []string{ |
| 193 | `DELETE FROM history_fts WHERE rowid IN (SELECT id FROM history_documents WHERE source_path=?)`, |
| 194 | `DELETE FROM history_documents WHERE source_path=?`, |
| 195 | } { |
| 196 | if _, err := tx.ExecContext(ctx, statement, path); err != nil { |
| 197 | _ = tx.Rollback() |
| 198 | return 0, err |
| 199 | } |
| 200 | } |
| 201 | if _, err := tx.ExecContext(ctx, `UPDATE history_sources SET health='evicted' WHERE path=?`, path); err != nil { |
| 202 | _ = tx.Rollback() |
| 203 | return 0, err |
| 204 | } |
| 205 | } |
| 206 | revision, err := bump(ctx, tx) |
| 207 | if err != nil { |
| 208 | _ = tx.Rollback() |
| 209 | return 0, err |
| 210 | } |
| 211 | if err := tx.Commit(); err != nil { |
| 212 | return 0, err |
| 213 | } |
| 214 | c.publish(revision, nil, "evict") |
| 215 | return len(batch), nil |
| 216 | } |
| 217 |