| 1 | package historycatalog |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/sha256" |
| 6 | "database/sql" |
| 7 | "encoding/hex" |
| 8 | "errors" |
| 9 | "fmt" |
| 10 | "os" |
| 11 | "path/filepath" |
| 12 | "runtime" |
| 13 | "sort" |
| 14 | "strings" |
| 15 | "sync" |
| 16 | "sync/atomic" |
| 17 | "time" |
| 18 | |
| 19 | "reasonix/internal/agent" |
| 20 | "reasonix/internal/projectiondb" |
| 21 | "reasonix/internal/retrieval" |
| 22 | "reasonix/internal/store" |
| 23 | ) |
| 24 | |
| 25 | const defaultMissingGrace = 30 * time.Second |
| 26 | |
| 27 | type Catalog struct { |
| 28 | db *sql.DB |
| 29 | opts Options |
| 30 | revision atomic.Uint64 |
| 31 | statusMu sync.RWMutex |
| 32 | status Status |
| 33 | ctx context.Context |
| 34 | cancel context.CancelFunc |
| 35 | queue chan string |
| 36 | rootCh chan string |
| 37 | flushCh chan chan struct{} |
| 38 | mu sync.Mutex |
| 39 | paths map[string]queuedPath |
| 40 | roots map[string]Root |
| 41 | dirtyRoots map[string]bool |
| 42 | wg sync.WaitGroup |
| 43 | closeOnce sync.Once |
| 44 | closeDone chan struct{} |
| 45 | closeErr error |
| 46 | } |
| 47 | |
| 48 | type queuedPath struct { |
| 49 | root Root |
| 50 | appendFrom int |
| 51 | } |
| 52 | |
| 53 | func Open(ctx context.Context, opts Options) (*Catalog, error) { |
| 54 | if opts.Path == "" { |
| 55 | opts.Path = DefaultPath() |
| 56 | } |
| 57 | if strings.TrimSpace(opts.Path) == "" { |
| 58 | opts.Path = "" |
| 59 | opts.InMemory = true |
| 60 | } |
| 61 | if opts.Now == nil { |
| 62 | opts.Now = time.Now |
| 63 | } |
| 64 | if opts.QueueCapacity <= 0 { |
| 65 | opts.QueueCapacity = 1024 |
| 66 | } |
| 67 | if opts.MissingGrace <= 0 { |
| 68 | opts.MissingGrace = defaultMissingGrace |
| 69 | } |
| 70 | if opts.ReconcileInterval <= 0 { |
| 71 | // Periodic root rescans are fingerprint-cheap when nothing changed; keep |
| 72 | // the interval longer so large installs are not re-walked every minute. |
| 73 | opts.ReconcileInterval = 5 * time.Minute |
| 74 | } |
| 75 | opts.MaxBytes = resolveMaxBytes(opts.MaxBytes, configuredMaxMB()) |
| 76 | handle, err := projectiondb.Open(ctx, projectiondb.OpenOptions{ |
| 77 | Path: opts.Path, MemoryName: "history-search", Migrations: migrations(), InMemory: opts.InMemory, |
| 78 | MaxOpenConns: 4, Now: opts.Now, SecureDelete: true, AutoVacuum: true, |
| 79 | }) |
| 80 | if err != nil { |
| 81 | return nil, err |
| 82 | } |
| 83 | workerCtx, cancel := context.WithCancel(context.Background()) |
| 84 | c := &Catalog{db: handle.DB, opts: opts, ctx: workerCtx, cancel: cancel, |
| 85 | queue: make(chan string, opts.QueueCapacity), rootCh: make(chan string, 64), |
| 86 | flushCh: make(chan chan struct{}, 1), |
| 87 | paths: map[string]queuedPath{}, roots: map[string]Root{}, dirtyRoots: map[string]bool{}, |
| 88 | closeDone: make(chan struct{}), |
| 89 | status: Status{State: string(handle.Status.State), Mode: handle.Status.Mode, Path: handle.Status.Path, |
| 90 | LastError: handle.Status.LastError, QuarantinedPath: handle.Status.QuarantinedPath}} |
| 91 | if err := c.db.QueryRowContext(ctx, `SELECT revision FROM history_state WHERE id=1`).Scan(new(uint64)); err != nil { |
| 92 | _ = c.db.Close() |
| 93 | cancel() |
| 94 | return nil, err |
| 95 | } |
| 96 | if err := c.ensureTokenizerVersion(ctx); err != nil { |
| 97 | _ = c.db.Close() |
| 98 | cancel() |
| 99 | return nil, err |
| 100 | } |
| 101 | var revision uint64 |
| 102 | _ = c.db.QueryRowContext(ctx, `SELECT revision FROM history_state WHERE id=1`).Scan(&revision) |
| 103 | c.revision.Store(revision) |
| 104 | c.refreshStatus(ctx) |
| 105 | c.wg.Add(1) |
| 106 | go c.worker() |
| 107 | // A far-over-cap index from before the cap existed (#8717) is cheaper to |
| 108 | // rebuild than to evict session-by-session; wipe async so startup never |
| 109 | // blocks. Registered roots rescan afterwards and re-index truncated. |
| 110 | if !opts.InMemory && strings.TrimSpace(opts.Path) != "" && |
| 111 | historyDBFileSize(opts.Path) > rebuildOversizeFactor*opts.MaxBytes { |
| 112 | c.wg.Go(func() { |
| 113 | c.wipeForRebuild(c.ctx) |
| 114 | }) |
| 115 | } |
| 116 | return c, nil |
| 117 | } |
| 118 | |
| 119 | func (c *Catalog) ensureTokenizerVersion(ctx context.Context) error { |
| 120 | var version int |
| 121 | if err := c.db.QueryRowContext(ctx, `SELECT tokenizer_version FROM history_state WHERE id=1`).Scan(&version); err != nil { |
| 122 | return err |
| 123 | } |
| 124 | if version == TokenizerVersion { |
| 125 | return nil |
| 126 | } |
| 127 | tx, err := c.db.BeginTx(ctx, nil) |
| 128 | if err != nil { |
| 129 | return err |
| 130 | } |
| 131 | if err := wipeProjectionRows(ctx, tx); err != nil { |
| 132 | _ = tx.Rollback() |
| 133 | return err |
| 134 | } |
| 135 | if _, err := tx.ExecContext(ctx, `UPDATE history_state SET tokenizer_version=?,revision=revision+1 WHERE id=1`, TokenizerVersion); err != nil { |
| 136 | _ = tx.Rollback() |
| 137 | return err |
| 138 | } |
| 139 | return tx.Commit() |
| 140 | } |
| 141 | |
| 142 | func (c *Catalog) RegisterRoot(root Root) bool { |
| 143 | if c == nil || strings.TrimSpace(root.Path) == "" { |
| 144 | return false |
| 145 | } |
| 146 | root.Path = filepath.Clean(root.Path) |
| 147 | if root.Scope != "project" { |
| 148 | root.Scope = "global" |
| 149 | root.WorkspaceRoot = "" |
| 150 | } |
| 151 | c.mu.Lock() |
| 152 | _, alreadyRegistered := c.roots[root.Path] |
| 153 | c.roots[root.Path] = root |
| 154 | c.mu.Unlock() |
| 155 | if !alreadyRegistered { |
| 156 | c.statusMu.Lock() |
| 157 | c.status.Pending++ |
| 158 | c.statusMu.Unlock() |
| 159 | } |
| 160 | select { |
| 161 | case c.rootCh <- root.Path: |
| 162 | return true |
| 163 | default: |
| 164 | c.markRootDirty(root.Path) |
| 165 | return false |
| 166 | } |
| 167 | } |
| 168 | |
| 169 | // ReconcileRoot performs one deterministic scan. Production callers normally |
| 170 | // use RegisterRoot; the synchronous form exists for doctor/reindex and tests. |
| 171 | func (c *Catalog) ReconcileRoot(ctx context.Context, root Root) error { |
| 172 | root.Path = filepath.Clean(root.Path) |
| 173 | if root.Scope != "project" { |
| 174 | root.Scope = "global" |
| 175 | root.WorkspaceRoot = "" |
| 176 | } |
| 177 | return c.reconcileRoot(ctx, root) |
| 178 | } |
| 179 | |
| 180 | func (c *Catalog) EnqueuePath(root Root, path string) bool { |
| 181 | return c.enqueuePath(root, path, -1) |
| 182 | } |
| 183 | |
| 184 | func (c *Catalog) EnqueuePersist(root Root, event agent.SessionPersistEvent) bool { |
| 185 | appendFrom := event.AppendFrom |
| 186 | if event.Rewrite { |
| 187 | appendFrom = -1 |
| 188 | } |
| 189 | return c.enqueuePath(root, event.Path, appendFrom) |
| 190 | } |
| 191 | |
| 192 | func (c *Catalog) enqueuePath(root Root, path string, appendFrom int) bool { |
| 193 | if c == nil || strings.TrimSpace(path) == "" { |
| 194 | return false |
| 195 | } |
| 196 | path = filepath.Clean(path) |
| 197 | c.mu.Lock() |
| 198 | if queued, exists := c.paths[path]; exists { |
| 199 | queued.root = root |
| 200 | if queued.appendFrom < 0 || appendFrom < 0 { |
| 201 | queued.appendFrom = -1 |
| 202 | } else if appendFrom < queued.appendFrom { |
| 203 | queued.appendFrom = appendFrom |
| 204 | } |
| 205 | c.paths[path] = queued |
| 206 | c.mu.Unlock() |
| 207 | return true |
| 208 | } |
| 209 | c.paths[path] = queuedPath{root: root, appendFrom: appendFrom} |
| 210 | c.mu.Unlock() |
| 211 | select { |
| 212 | case c.queue <- path: |
| 213 | return true |
| 214 | default: |
| 215 | c.mu.Lock() |
| 216 | delete(c.paths, path) |
| 217 | c.dirtyRoots[root.Path] = true |
| 218 | c.mu.Unlock() |
| 219 | return false |
| 220 | } |
| 221 | } |
| 222 | |
| 223 | // EnqueueExisting prioritizes a source already known to the catalog without |
| 224 | // requiring the caller to retain its root metadata. |
| 225 | func (c *Catalog) EnqueueExisting(ctx context.Context, path string) bool { |
| 226 | var root Root |
| 227 | err := c.db.QueryRowContext(ctx, `SELECT root,source,scope,workspace_root FROM history_sources WHERE path=?`, filepath.Clean(path)).Scan( |
| 228 | &root.Path, &root.Source, &root.Scope, &root.WorkspaceRoot) |
| 229 | if err != nil { |
| 230 | return false |
| 231 | } |
| 232 | return c.EnqueuePath(root, path) |
| 233 | } |
| 234 | |
| 235 | func (c *Catalog) worker() { |
| 236 | defer c.wg.Done() |
| 237 | ticker := time.NewTicker(c.opts.ReconcileInterval) |
| 238 | defer ticker.Stop() |
| 239 | for { |
| 240 | if root, ok := c.takeDirtyRoot(); ok { |
| 241 | _ = c.reconcileRoot(c.ctx, root) |
| 242 | continue |
| 243 | } |
| 244 | select { |
| 245 | case <-c.ctx.Done(): |
| 246 | return |
| 247 | case <-ticker.C: |
| 248 | c.markAllRootsDirty() |
| 249 | c.governSize(c.ctx) |
| 250 | case done := <-c.flushCh: |
| 251 | c.drainPending(c.ctx) |
| 252 | close(done) |
| 253 | case path := <-c.queue: |
| 254 | c.mu.Lock() |
| 255 | queued := c.paths[path] |
| 256 | delete(c.paths, path) |
| 257 | c.mu.Unlock() |
| 258 | _ = c.indexPath(c.ctx, queued.root, path, 0, queued.appendFrom) |
| 259 | case path := <-c.rootCh: |
| 260 | c.mu.Lock() |
| 261 | root, ok := c.roots[path] |
| 262 | c.mu.Unlock() |
| 263 | if ok { |
| 264 | _ = c.reconcileRoot(c.ctx, root) |
| 265 | } |
| 266 | } |
| 267 | } |
| 268 | } |
| 269 | |
| 270 | // Flush drains dirty roots and the path queue until empty or ctx cancels, then |
| 271 | // waits for the worker to acknowledge. Callers use this on shutdown so pending |
| 272 | // index work is not silently abandoned. |
| 273 | func (c *Catalog) Flush(ctx context.Context) error { |
| 274 | if c == nil { |
| 275 | return nil |
| 276 | } |
| 277 | done := make(chan struct{}) |
| 278 | select { |
| 279 | case c.flushCh <- done: |
| 280 | case <-ctx.Done(): |
| 281 | return ctx.Err() |
| 282 | } |
| 283 | select { |
| 284 | case <-done: |
| 285 | return nil |
| 286 | case <-ctx.Done(): |
| 287 | return ctx.Err() |
| 288 | } |
| 289 | } |
| 290 | |
| 291 | func (c *Catalog) drainPending(ctx context.Context) { |
| 292 | for { |
| 293 | if err := ctx.Err(); err != nil { |
| 294 | return |
| 295 | } |
| 296 | if root, ok := c.takeDirtyRoot(); ok { |
| 297 | _ = c.reconcileRoot(ctx, root) |
| 298 | continue |
| 299 | } |
| 300 | select { |
| 301 | case path := <-c.queue: |
| 302 | c.mu.Lock() |
| 303 | queued := c.paths[path] |
| 304 | delete(c.paths, path) |
| 305 | c.mu.Unlock() |
| 306 | _ = c.indexPath(ctx, queued.root, path, 0, queued.appendFrom) |
| 307 | case path := <-c.rootCh: |
| 308 | c.mu.Lock() |
| 309 | root, ok := c.roots[path] |
| 310 | c.mu.Unlock() |
| 311 | if ok { |
| 312 | _ = c.reconcileRoot(ctx, root) |
| 313 | } |
| 314 | default: |
| 315 | c.mu.Lock() |
| 316 | empty := len(c.paths) == 0 && len(c.dirtyRoots) == 0 |
| 317 | c.mu.Unlock() |
| 318 | if empty && len(c.queue) == 0 && len(c.rootCh) == 0 { |
| 319 | return |
| 320 | } |
| 321 | // Another goroutine may have enqueued between checks; yield once. |
| 322 | runtime.Gosched() |
| 323 | c.mu.Lock() |
| 324 | empty = len(c.paths) == 0 && len(c.dirtyRoots) == 0 |
| 325 | c.mu.Unlock() |
| 326 | if empty && len(c.queue) == 0 && len(c.rootCh) == 0 { |
| 327 | return |
| 328 | } |
| 329 | } |
| 330 | } |
| 331 | } |
| 332 | |
| 333 | func (c *Catalog) markRootDirty(path string) { |
| 334 | c.mu.Lock() |
| 335 | c.dirtyRoots[path] = true |
| 336 | c.mu.Unlock() |
| 337 | } |
| 338 | |
| 339 | func (c *Catalog) markAllRootsDirty() { |
| 340 | c.mu.Lock() |
| 341 | for path := range c.roots { |
| 342 | c.dirtyRoots[path] = true |
| 343 | } |
| 344 | c.mu.Unlock() |
| 345 | } |
| 346 | |
| 347 | func (c *Catalog) takeDirtyRoot() (Root, bool) { |
| 348 | c.mu.Lock() |
| 349 | defer c.mu.Unlock() |
| 350 | for path := range c.dirtyRoots { |
| 351 | delete(c.dirtyRoots, path) |
| 352 | root, ok := c.roots[path] |
| 353 | return root, ok |
| 354 | } |
| 355 | return Root{}, false |
| 356 | } |
| 357 | |
| 358 | func historyRootSignature(paths []string) string { |
| 359 | hash := sha256.New() |
| 360 | for _, path := range paths { |
| 361 | for _, candidate := range []string{path, agent.BranchMetaPath(path)} { |
| 362 | info, err := os.Stat(candidate) |
| 363 | if err != nil { |
| 364 | _, _ = fmt.Fprintf(hash, "%s\x00missing\n", candidate) |
| 365 | continue |
| 366 | } |
| 367 | _, _ = fmt.Fprintf(hash, "%s\x00%d\x00%d\n", candidate, info.Size(), info.ModTime().UnixNano()) |
| 368 | } |
| 369 | } |
| 370 | return hex.EncodeToString(hash.Sum(nil)) |
| 371 | } |
| 372 | |
| 373 | func (c *Catalog) reconcileRoot(ctx context.Context, root Root) error { |
| 374 | entries, err := os.ReadDir(root.Path) |
| 375 | if err != nil && !errors.Is(err, os.ErrNotExist) { |
| 376 | c.setError(err) |
| 377 | return err |
| 378 | } |
| 379 | paths := make([]string, 0, len(entries)) |
| 380 | for _, entry := range entries { |
| 381 | if entry.IsDir() || !store.IsSessionTranscriptName(entry.Name()) { |
| 382 | continue |
| 383 | } |
| 384 | path := filepath.Join(root.Path, entry.Name()) |
| 385 | if !root.Archive && !agent.IsVisibleSession(path) { |
| 386 | continue |
| 387 | } |
| 388 | paths = append(paths, path) |
| 389 | } |
| 390 | sort.Strings(paths) |
| 391 | signature := historyRootSignature(paths) |
| 392 | var previousSig, previousState string |
| 393 | _ = c.db.QueryRowContext(ctx, `SELECT signature,state FROM history_roots WHERE path=?`, root.Path).Scan(&previousSig, &previousState) |
| 394 | if previousState == "ready" && previousSig == signature && signature != "" { |
| 395 | return nil |
| 396 | } |
| 397 | now := c.opts.Now().UnixMilli() |
| 398 | tx, err := c.db.BeginTx(ctx, nil) |
| 399 | if err != nil { |
| 400 | return err |
| 401 | } |
| 402 | var generation int64 |
| 403 | if err := tx.QueryRowContext(ctx, `INSERT INTO history_roots(path,source,scope,workspace_root,signature,scan_generation,state,total) |
| 404 | VALUES(?,?,?,?,?,1,'scanning',?) ON CONFLICT(path) DO UPDATE SET source=excluded.source,scope=excluded.scope, |
| 405 | workspace_root=excluded.workspace_root,signature=excluded.signature,scan_generation=history_roots.scan_generation+1,state='scanning',error='',total=excluded.total |
| 406 | RETURNING scan_generation`, root.Path, root.Source, root.Scope, root.WorkspaceRoot, signature, len(paths)).Scan(&generation); err != nil { |
| 407 | _ = tx.Rollback() |
| 408 | return err |
| 409 | } |
| 410 | if err := tx.Commit(); err != nil { |
| 411 | return err |
| 412 | } |
| 413 | indexed := 0 |
| 414 | for i, path := range paths { |
| 415 | if err := ctx.Err(); err != nil { |
| 416 | return err |
| 417 | } |
| 418 | if err := c.indexPath(ctx, root, path, generation, -1); err == nil { |
| 419 | indexed++ |
| 420 | } |
| 421 | if (i+1)%32 == 0 { |
| 422 | runtime.Gosched() |
| 423 | } |
| 424 | } |
| 425 | tx, err = c.db.BeginTx(ctx, nil) |
| 426 | if err != nil { |
| 427 | return err |
| 428 | } |
| 429 | if _, err := tx.ExecContext(ctx, `UPDATE history_sources SET missing_since=CASE WHEN missing_since=0 THEN ? ELSE missing_since END, |
| 430 | health='missing' WHERE root=? AND seen_generation<>?`, now, root.Path, generation); err != nil { |
| 431 | _ = tx.Rollback() |
| 432 | return err |
| 433 | } |
| 434 | cutoff := now - c.opts.MissingGrace.Milliseconds() |
| 435 | if _, err := tx.ExecContext(ctx, `DELETE FROM history_fts WHERE rowid IN ( |
| 436 | SELECT d.id FROM history_documents d JOIN history_sources s ON s.path=d.source_path |
| 437 | WHERE s.root=? AND s.seen_generation<>? AND s.missing_since>0 AND s.missing_since<=? |
| 438 | )`, root.Path, generation, cutoff); err != nil { |
| 439 | _ = tx.Rollback() |
| 440 | return err |
| 441 | } |
| 442 | if _, err := tx.ExecContext(ctx, `DELETE FROM history_documents WHERE source_path IN ( |
| 443 | SELECT path FROM history_sources WHERE root=? AND seen_generation<>? AND missing_since>0 AND missing_since<=? |
| 444 | )`, root.Path, generation, cutoff); err != nil { |
| 445 | _ = tx.Rollback() |
| 446 | return err |
| 447 | } |
| 448 | if _, err := tx.ExecContext(ctx, `DELETE FROM history_sources |
| 449 | WHERE root=? AND seen_generation<>? AND missing_since>0 AND missing_since<=?`, root.Path, generation, cutoff); err != nil { |
| 450 | _ = tx.Rollback() |
| 451 | return err |
| 452 | } |
| 453 | if _, err := tx.ExecContext(ctx, `UPDATE history_roots SET state='ready',signature=?,indexed=?,completed_at=?,scan_cursor='' WHERE path=?`, signature, indexed, now, root.Path); err != nil { |
| 454 | _ = tx.Rollback() |
| 455 | return err |
| 456 | } |
| 457 | revision, err := bump(ctx, tx) |
| 458 | if err != nil { |
| 459 | _ = tx.Rollback() |
| 460 | return err |
| 461 | } |
| 462 | if err := tx.Commit(); err != nil { |
| 463 | return err |
| 464 | } |
| 465 | c.publish(revision, []string{root.Path}, "reconcile") |
| 466 | return nil |
| 467 | } |
| 468 | |
| 469 | func fileFingerprint(path string) string { |
| 470 | info, err := os.Stat(path) |
| 471 | if err != nil { |
| 472 | return "" |
| 473 | } |
| 474 | return fmt.Sprintf("%d:%d", info.Size(), info.ModTime().UnixNano()) |
| 475 | } |
| 476 | |
| 477 | func (c *Catalog) indexPath(ctx context.Context, root Root, path string, generation int64, appendFrom int) error { |
| 478 | if !root.Archive && !agent.IsVisibleSession(path) { |
| 479 | return c.Purge(ctx, path) |
| 480 | } |
| 481 | contentFingerprint := fileFingerprint(path) |
| 482 | metaFingerprint := fileFingerprint(agent.BranchMetaPath(path)) |
| 483 | state, known, identityErr := agent.SessionContentIdentity(path) |
| 484 | digest := "" |
| 485 | revision := int64(0) |
| 486 | if identityErr == nil && known { |
| 487 | digest, revision = state.DigestHex, state.Revision |
| 488 | } |
| 489 | var oldFingerprint, oldMetaFingerprint, oldDigest, oldHealth string |
| 490 | var oldGeneration, oldRevision int64 |
| 491 | var oldMessageCount int |
| 492 | err := c.db.QueryRowContext(ctx, `SELECT content_fingerprint,meta_fingerprint,content_digest,seen_generation, |
| 493 | content_revision,indexed_message_count,health FROM history_sources WHERE path=?`, path).Scan( |
| 494 | &oldFingerprint, &oldMetaFingerprint, &oldDigest, &oldGeneration, &oldRevision, &oldMessageCount, &oldHealth) |
| 495 | if sourceProjectionUnchanged(err, oldFingerprint, contentFingerprint, oldMetaFingerprint, metaFingerprint, oldDigest, digest) { |
| 496 | if generation != 0 && oldGeneration != generation { |
| 497 | // Keep evicted rows evicted: an unchanged file must not re-enter the index. |
| 498 | _, _ = c.db.ExecContext(ctx, `UPDATE history_sources SET seen_generation=?,missing_since=0, |
| 499 | health=CASE WHEN health='evicted' THEN 'evicted' ELSE 'ok' END WHERE path=?`, generation, path) |
| 500 | } |
| 501 | return nil |
| 502 | } |
| 503 | if err != nil && !errors.Is(err, sql.ErrNoRows) { |
| 504 | return err |
| 505 | } |
| 506 | // An evicted projection has no prefix to append onto; fall through to a full reload. |
| 507 | if err == nil && known && appendFrom >= 0 && oldHealth != "evicted" { |
| 508 | handled, appendErr := c.tryAppendPath(ctx, root, path, generation, appendFrom, oldMessageCount, oldRevision, |
| 509 | revision, digest, contentFingerprint, metaFingerprint) |
| 510 | if appendErr != nil { |
| 511 | return appendErr |
| 512 | } |
| 513 | if handled { |
| 514 | return nil |
| 515 | } |
| 516 | } |
| 517 | session, err := agent.LoadSession(path) |
| 518 | if err != nil { |
| 519 | _, _ = c.db.ExecContext(ctx, `INSERT INTO history_sources(path,root,source,scope,workspace_root,content_fingerprint,meta_fingerprint,health,last_error,seen_generation) |
| 520 | VALUES(?,?,?,?,?,?,?,'corrupt',?,?) ON CONFLICT(path) DO UPDATE SET health='corrupt',last_error=excluded.last_error, |
| 521 | content_fingerprint=excluded.content_fingerprint,meta_fingerprint=excluded.meta_fingerprint,seen_generation=excluded.seen_generation`, |
| 522 | path, root.Path, root.Source, root.Scope, root.WorkspaceRoot, contentFingerprint, metaFingerprint, err.Error(), generation) |
| 523 | c.setError(err) |
| 524 | return err |
| 525 | } |
| 526 | messages := session.Snapshot() |
| 527 | if digest == "" { |
| 528 | h := sha256.New() |
| 529 | for _, doc := range documents(messages) { |
| 530 | _, _ = h.Write([]byte(doc.terms)) |
| 531 | _, _ = h.Write([]byte{0}) |
| 532 | } |
| 533 | digest = hex.EncodeToString(h.Sum(nil)) |
| 534 | } |
| 535 | meta, _, _ := agent.LoadBranchMeta(path) |
| 536 | lastActivity := max(int64(0), agent.SessionContentModTime(path).UnixMilli()) |
| 537 | // Hide stale terms as soon as the authoritative fingerprint changes. Rows |
| 538 | // remain available for retry and are atomically replaced below. |
| 539 | if _, err := c.db.ExecContext(ctx, `UPDATE history_sources SET health='stale',last_error='' WHERE path=?`, path); err != nil { |
| 540 | return err |
| 541 | } |
| 542 | tx, err := c.db.BeginTx(ctx, nil) |
| 543 | if err != nil { |
| 544 | return err |
| 545 | } |
| 546 | if _, err := tx.ExecContext(ctx, `DELETE FROM history_fts WHERE rowid IN (SELECT id FROM history_documents WHERE source_path=?)`, path); err != nil { |
| 547 | _ = tx.Rollback() |
| 548 | return err |
| 549 | } |
| 550 | if _, err := tx.ExecContext(ctx, `DELETE FROM history_documents WHERE source_path=?`, path); err != nil { |
| 551 | _ = tx.Rollback() |
| 552 | return err |
| 553 | } |
| 554 | _, err = tx.ExecContext(ctx, `INSERT INTO history_sources(path,root,source,scope,workspace_root,content_revision,content_digest, |
| 555 | content_fingerprint,meta_fingerprint,message_count,indexed_message_count,custom_title,topic_id,topic_title,preview,created_at, |
| 556 | last_activity_at,health,missing_since,seen_generation,last_error) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,'ok',0,?,'') |
| 557 | ON CONFLICT(path) DO UPDATE SET root=excluded.root,source=excluded.source,scope=excluded.scope,workspace_root=excluded.workspace_root, |
| 558 | content_revision=excluded.content_revision,content_digest=excluded.content_digest,content_fingerprint=excluded.content_fingerprint, |
| 559 | meta_fingerprint=excluded.meta_fingerprint,message_count=excluded.message_count,indexed_message_count=excluded.indexed_message_count, |
| 560 | custom_title=excluded.custom_title,topic_id=excluded.topic_id,topic_title=excluded.topic_title,preview=excluded.preview, |
| 561 | created_at=excluded.created_at,last_activity_at=excluded.last_activity_at,health='ok',missing_since=0, |
| 562 | seen_generation=excluded.seen_generation,last_error=''`, path, root.Path, root.Source, root.Scope, root.WorkspaceRoot, revision, digest, |
| 563 | contentFingerprint, metaFingerprint, len(messages), len(messages), meta.CustomTitle, meta.TopicID, meta.TopicTitle, meta.Preview, |
| 564 | meta.CreatedAt.UnixMilli(), lastActivity, generation) |
| 565 | if err != nil { |
| 566 | _ = tx.Rollback() |
| 567 | return err |
| 568 | } |
| 569 | for _, doc := range documents(messages) { |
| 570 | result, err := tx.ExecContext(ctx, `INSERT INTO history_documents(source_path,message_index,part_index,role,kind,tool_name,token_count) |
| 571 | VALUES(?,?,?,?,?,?,?)`, path, doc.message, doc.part, doc.role, doc.kind, doc.tool, doc.count) |
| 572 | if err != nil { |
| 573 | _ = tx.Rollback() |
| 574 | return err |
| 575 | } |
| 576 | rowID, err := result.LastInsertId() |
| 577 | if err != nil { |
| 578 | _ = tx.Rollback() |
| 579 | return err |
| 580 | } |
| 581 | if _, err := tx.ExecContext(ctx, `INSERT INTO history_fts(rowid,terms) VALUES(?,?)`, rowID, doc.terms); err != nil { |
| 582 | _ = tx.Rollback() |
| 583 | return err |
| 584 | } |
| 585 | } |
| 586 | newRevision, err := bump(ctx, tx) |
| 587 | if err != nil { |
| 588 | _ = tx.Rollback() |
| 589 | return err |
| 590 | } |
| 591 | if err := tx.Commit(); err != nil { |
| 592 | return err |
| 593 | } |
| 594 | c.publish(newRevision, []string{root.Path}, "source-indexed") |
| 595 | return nil |
| 596 | } |
| 597 | |
| 598 | func bump(ctx context.Context, tx *sql.Tx) (uint64, error) { |
| 599 | if _, err := tx.ExecContext(ctx, `UPDATE history_state SET revision=revision+1 WHERE id=1`); err != nil { |
| 600 | return 0, err |
| 601 | } |
| 602 | var revision uint64 |
| 603 | err := tx.QueryRowContext(ctx, `SELECT revision FROM history_state WHERE id=1`).Scan(&revision) |
| 604 | return revision, err |
| 605 | } |
| 606 | |
| 607 | func (c *Catalog) Search(ctx context.Context, req SearchRequest) (SearchResult, error) { |
| 608 | out := SearchResult{Items: []Candidate{}, Revision: c.revision.Load(), Partial: c.Status().Pending > 0} |
| 609 | terms, err := retrieval.QueryTerms(req.Query) |
| 610 | if err != nil { |
| 611 | return out, err |
| 612 | } |
| 613 | limit := req.Limit |
| 614 | if limit <= 0 { |
| 615 | limit = DefaultLimit |
| 616 | } |
| 617 | if limit > MaxLimit { |
| 618 | limit = MaxLimit |
| 619 | } |
| 620 | match := make([]string, 0, len(terms)) |
| 621 | for _, term := range terms { |
| 622 | match = append(match, `"`+strings.ReplaceAll(term, `"`, `""`)+`"`) |
| 623 | } |
| 624 | where := []string{`history_fts MATCH ?`, `s.health='ok'`, `s.missing_since=0`} |
| 625 | args := []any{strings.Join(match, " OR ")} |
| 626 | if req.Scope == "project" { |
| 627 | where = append(where, `s.scope='project'`, `s.workspace_root=?`) |
| 628 | args = append(args, strings.TrimSpace(req.WorkspaceRoot)) |
| 629 | } |
| 630 | if path := strings.TrimSpace(req.SessionPath); path != "" { |
| 631 | where = append(where, `d.source_path=?`) |
| 632 | args = append(args, filepath.Clean(path)) |
| 633 | } |
| 634 | if len(req.Kinds) > 0 { |
| 635 | placeholders := make([]string, len(req.Kinds)) |
| 636 | for i, kind := range req.Kinds { |
| 637 | placeholders[i] = "?" |
| 638 | args = append(args, kind) |
| 639 | } |
| 640 | where = append(where, `d.kind IN (`+strings.Join(placeholders, ",")+`)`) |
| 641 | } |
| 642 | if tool := strings.TrimSpace(req.ToolName); tool != "" { |
| 643 | where = append(where, `d.tool_name=?`) |
| 644 | args = append(args, tool) |
| 645 | } |
| 646 | if len(req.Roots) > 0 { |
| 647 | placeholders := make([]string, 0, len(req.Roots)) |
| 648 | for _, root := range req.Roots { |
| 649 | if strings.TrimSpace(root) == "" { |
| 650 | continue |
| 651 | } |
| 652 | placeholders = append(placeholders, "?") |
| 653 | args = append(args, filepath.Clean(root)) |
| 654 | } |
| 655 | if len(placeholders) > 0 { |
| 656 | where = append(where, `s.root IN (`+strings.Join(placeholders, ",")+`)`) |
| 657 | } |
| 658 | } |
| 659 | baseQuery := `SELECT d.id AS id,d.source_path AS source_path,s.root AS root,s.source AS source,s.scope AS scope, |
| 660 | s.workspace_root AS workspace_root,s.content_digest AS content_digest,d.message_index AS message_index, |
| 661 | d.part_index AS part_index,d.role AS role,d.kind AS kind,d.tool_name AS tool_name,bm25(history_fts) AS rank, |
| 662 | s.custom_title AS custom_title,s.topic_title AS topic_title,s.last_activity_at AS last_activity_at |
| 663 | FROM history_fts JOIN history_documents d ON d.id=history_fts.rowid JOIN history_sources s ON s.path=d.source_path |
| 664 | WHERE ` + strings.Join(where, ` AND `) |
| 665 | query := baseQuery + ` ORDER BY bm25(history_fts),d.source_path,d.message_index,d.part_index,d.id LIMIT ?` |
| 666 | if after := req.After; after != nil { |
| 667 | query = `WITH ranked AS MATERIALIZED (` + baseQuery + `) |
| 668 | SELECT * FROM ranked WHERE rank>? OR (rank=? AND source_path>?) OR |
| 669 | (rank=? AND source_path=? AND message_index>?) OR |
| 670 | (rank=? AND source_path=? AND message_index=? AND part_index>?) OR |
| 671 | (rank=? AND source_path=? AND message_index=? AND part_index=? AND id>?) |
| 672 | ORDER BY rank,source_path,message_index,part_index,id LIMIT ?` |
| 673 | args = append(args, after.Rank, after.Rank, after.SessionPath, |
| 674 | after.Rank, after.SessionPath, after.MessageIndex, |
| 675 | after.Rank, after.SessionPath, after.MessageIndex, after.PartIndex, |
| 676 | after.Rank, after.SessionPath, after.MessageIndex, after.PartIndex, after.RowID) |
| 677 | } |
| 678 | args = append(args, limit) |
| 679 | rows, err := c.db.QueryContext(ctx, query, args...) |
| 680 | if err != nil { |
| 681 | return out, err |
| 682 | } |
| 683 | defer rows.Close() |
| 684 | for rows.Next() { |
| 685 | var item Candidate |
| 686 | if err := rows.Scan(&item.RowID, &item.SessionPath, &item.Root, &item.Source, &item.Scope, &item.WorkspaceRoot, &item.ContentDigest, |
| 687 | &item.MessageIndex, &item.PartIndex, &item.Role, &item.Kind, &item.ToolName, &item.Rank, |
| 688 | &item.SessionTitle, &item.TopicTitle, &item.LastActivityAt); err != nil { |
| 689 | return out, err |
| 690 | } |
| 691 | if !catalogPathWithin(item.SessionPath, item.Root) { |
| 692 | continue |
| 693 | } |
| 694 | if item.Rank < 0 { |
| 695 | item.Score = -item.Rank |
| 696 | } else { |
| 697 | item.Score = 1 / (1 + item.Rank) |
| 698 | } |
| 699 | out.Items = append(out.Items, item) |
| 700 | } |
| 701 | return out, rows.Err() |
| 702 | } |
| 703 | |
| 704 | func catalogPathWithin(path, root string) bool { |
| 705 | absPath, err := filepath.Abs(filepath.Clean(strings.TrimSpace(path))) |
| 706 | if err != nil { |
| 707 | return false |
| 708 | } |
| 709 | absRoot, err := filepath.Abs(filepath.Clean(strings.TrimSpace(root))) |
| 710 | if err != nil { |
| 711 | return false |
| 712 | } |
| 713 | rel, err := filepath.Rel(absRoot, absPath) |
| 714 | return err == nil && (rel == "." || (rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)))) |
| 715 | } |
| 716 | |
| 717 | func (c *Catalog) Purge(ctx context.Context, path string) error { |
| 718 | if c == nil { |
| 719 | return nil |
| 720 | } |
| 721 | tx, err := c.db.BeginTx(ctx, nil) |
| 722 | if err != nil { |
| 723 | return err |
| 724 | } |
| 725 | if _, err := tx.ExecContext(ctx, `DELETE FROM history_fts WHERE rowid IN (SELECT id FROM history_documents WHERE source_path=?)`, path); err != nil { |
| 726 | _ = tx.Rollback() |
| 727 | return err |
| 728 | } |
| 729 | if _, err := tx.ExecContext(ctx, `DELETE FROM history_sources WHERE path=?`, path); err != nil { |
| 730 | _ = tx.Rollback() |
| 731 | return err |
| 732 | } |
| 733 | revision, err := bump(ctx, tx) |
| 734 | if err != nil { |
| 735 | _ = tx.Rollback() |
| 736 | return err |
| 737 | } |
| 738 | if err := tx.Commit(); err != nil { |
| 739 | return err |
| 740 | } |
| 741 | _, _ = c.db.ExecContext(ctx, `PRAGMA wal_checkpoint(TRUNCATE)`) |
| 742 | _, _ = c.db.ExecContext(ctx, `PRAGMA incremental_vacuum(64)`) |
| 743 | c.publish(revision, nil, "purge") |
| 744 | return nil |
| 745 | } |
| 746 | |
| 747 | func (c *Catalog) refreshStatus(ctx context.Context) { |
| 748 | var indexed, total, pending, failed int64 |
| 749 | _ = c.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM history_sources WHERE health='ok'`).Scan(&indexed) |
| 750 | _ = c.db.QueryRowContext(ctx, `SELECT COALESCE(SUM(total),0) FROM history_roots`).Scan(&total) |
| 751 | _ = c.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM history_roots WHERE state<>'ready'`).Scan(&pending) |
| 752 | _ = c.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM history_sources WHERE health='corrupt'`).Scan(&failed) |
| 753 | c.statusMu.Lock() |
| 754 | c.status.Indexed, c.status.Total, c.status.Pending, c.status.Failed = indexed, total, pending, failed |
| 755 | c.status.Revision = c.revision.Load() |
| 756 | c.statusMu.Unlock() |
| 757 | } |
| 758 | func (c *Catalog) publish(revision uint64, roots []string, reason string) { |
| 759 | c.revision.Store(revision) |
| 760 | statusCtx := c.ctx |
| 761 | if statusCtx == nil { |
| 762 | statusCtx = context.Background() |
| 763 | } |
| 764 | c.refreshStatus(statusCtx) |
| 765 | if c.opts.OnRevision != nil { |
| 766 | c.opts.OnRevision(c.Status(), roots, reason) |
| 767 | } |
| 768 | } |
| 769 | |
| 770 | func (c *Catalog) setError(err error) { |
| 771 | c.statusMu.Lock() |
| 772 | c.status.LastError = err.Error() |
| 773 | c.status.Failed++ |
| 774 | c.statusMu.Unlock() |
| 775 | } |
| 776 | |
| 777 | func (c *Catalog) Status() Status { |
| 778 | if c == nil { |
| 779 | return Status{State: "degraded", Mode: projectiondb.ModeMemory, LastError: "history catalog unavailable"} |
| 780 | } |
| 781 | c.statusMu.RLock() |
| 782 | defer c.statusMu.RUnlock() |
| 783 | return c.status |
| 784 | } |
| 785 | |
| 786 | func (c *Catalog) Close(ctx context.Context) error { |
| 787 | if c == nil { |
| 788 | return nil |
| 789 | } |
| 790 | c.closeOnce.Do(func() { |
| 791 | c.cancel() |
| 792 | go func() { |
| 793 | c.wg.Wait() |
| 794 | c.closeErr = c.db.Close() |
| 795 | close(c.closeDone) |
| 796 | }() |
| 797 | }) |
| 798 | select { |
| 799 | case <-c.closeDone: |
| 800 | return c.closeErr |
| 801 | case <-ctx.Done(): |
| 802 | return ctx.Err() |
| 803 | } |
| 804 | } |
| 805 |