| 1 | // Package skillwatch shares physical skill-directory watches across stores. |
| 2 | // Healthy roots use coalesced native events. Failed roots use bounded backoff |
| 3 | // scans until registration recovers. Windows isolates blocking filesystem APIs |
| 4 | // in a helper process. Subscribe completes registration before the caller's |
| 5 | // first catalog scan, subject to the helper timeout. |
| 6 | package skillwatch |
| 7 | |
| 8 | import ( |
| 9 | "context" |
| 10 | "crypto/sha256" |
| 11 | "fmt" |
| 12 | "io" |
| 13 | "path/filepath" |
| 14 | "sync" |
| 15 | "time" |
| 16 | ) |
| 17 | |
| 18 | // scanBackoffs is the degraded-mode cadence replacing the retired 250ms |
| 19 | // polling generation. |
| 20 | var scanBackoffs = []time.Duration{2 * time.Second, 5 * time.Second, 15 * time.Second, 30 * time.Second} |
| 21 | |
| 22 | const ( |
| 23 | coalesceWindow = 100 * time.Millisecond |
| 24 | coalesceMaxDelay = 500 * time.Millisecond |
| 25 | // helperControlTimeout bounds one helper control round trip; a slower |
| 26 | // helper is torn down and rebuilt. helperRestartWindow / helperRestartLimit |
| 27 | // cap automatic rebuilds before the service degrades to scanning. |
| 28 | helperControlTimeout = 5 * time.Second |
| 29 | helperRestartWindow = 30 * time.Second |
| 30 | helperRestartLimit = 2 |
| 31 | // helperSubscribeWait bounds how long Subscribe waits for the helper to |
| 32 | // confirm a registration before letting the caller's first scan proceed. |
| 33 | // Expiry is non-fatal: the registration stays in flight under the control |
| 34 | // timeout and late confirmation still arms the watches. |
| 35 | helperSubscribeWait = 2 * time.Second |
| 36 | ) |
| 37 | |
| 38 | // ScopeFunc returns the directories discovery can visit under root for the |
| 39 | // given depth. It includes the nearest existing ancestor of a missing root so |
| 40 | // later creation invalidates the snapshot, and excludes subtrees discovery |
| 41 | // skips (scripts/assets/references bodies) so their content churn cannot |
| 42 | // trigger catalog rebuilds. |
| 43 | type ScopeFunc func(ctx context.Context, root string, maxDepth int) (dirs []string, complete bool) |
| 44 | |
| 45 | // HashFunc summarizes one root's watched tree for degraded-mode scanning and |
| 46 | // reports how many entries the scan visited. ok is false when the scan could |
| 47 | // not complete (cancelled, IO error). |
| 48 | type HashFunc func(ctx context.Context, root string, maxDepth int) (sum [sha256.Size]byte, entries int, ok bool) |
| 49 | |
| 50 | // Options configure the service. |
| 51 | type Options struct { |
| 52 | // Stderr receives diagnostic warnings; nil defaults to os.Stderr. |
| 53 | Stderr io.Writer |
| 54 | // HelperCommand overrides the helper process used by Windows and tests. |
| 55 | HelperCommand func(ctx context.Context) (helperProcess, error) |
| 56 | // ForceHelper routes every platform through the helper backend. Test-only. |
| 57 | ForceHelper bool |
| 58 | } |
| 59 | |
| 60 | // Diagnostics snapshots the resource counters. Healthy idle state keeps scans |
| 61 | // at zero after initialization and never grows physical watches while the |
| 62 | // subscription set is constant. |
| 63 | type Diagnostics struct { |
| 64 | PhysicalWatches uint64 `json:"physicalWatches"` |
| 65 | LogicalSubscriptions uint64 `json:"logicalSubscriptions"` |
| 66 | Scans uint64 `json:"scans"` |
| 67 | ScannedEntries uint64 `json:"scannedEntries"` |
| 68 | EventsReceived uint64 `json:"eventsReceived"` |
| 69 | Notifications uint64 `json:"notifications"` |
| 70 | DegradedRoots uint64 `json:"degradedRoots"` |
| 71 | HelperRestarts uint64 `json:"helperRestarts"` |
| 72 | } |
| 73 | |
| 74 | // Subscription is one store's handle on a physical root. Release is safe to |
| 75 | // call twice and never blocks on backend IO: the logical subscription dies |
| 76 | // immediately and late events for it are dropped. |
| 77 | type Subscription struct { |
| 78 | svc *Service |
| 79 | root string // canonical root path (resolved) |
| 80 | maxDepth int |
| 81 | onChange func(reason string) |
| 82 | |
| 83 | mu sync.Mutex |
| 84 | released bool |
| 85 | } |
| 86 | |
| 87 | // Release detaches the subscription. The last subscription on a root tears the |
| 88 | // physical watches down; no historical project watches are retained. |
| 89 | func (sub *Subscription) Release() { |
| 90 | if sub == nil { |
| 91 | return |
| 92 | } |
| 93 | sub.mu.Lock() |
| 94 | if sub.released { |
| 95 | sub.mu.Unlock() |
| 96 | return |
| 97 | } |
| 98 | sub.released = true |
| 99 | sub.mu.Unlock() |
| 100 | sub.svc.dropSubscription(sub) |
| 101 | } |
| 102 | |
| 103 | type rootState struct { |
| 104 | canonical string |
| 105 | gen uint64 // bumped on every (re)registration; late events dropped |
| 106 | dirs []string |
| 107 | watchID uint64 // backend registration carrying the current gen |
| 108 | maxDepth int |
| 109 | subs map[*Subscription]struct{} |
| 110 | scope ScopeFunc |
| 111 | hash HashFunc |
| 112 | |
| 113 | scopeDirty bool |
| 114 | |
| 115 | // regDone closes when the current registration attempt settles, bounding |
| 116 | // the Subscribe-side wait for the register-before-scan ordering. |
| 117 | regDone chan struct{} |
| 118 | |
| 119 | // coalescing |
| 120 | pending bool |
| 121 | firstSeen time.Time |
| 122 | timer *time.Timer |
| 123 | |
| 124 | // degraded scanning |
| 125 | degraded bool |
| 126 | scanStop chan struct{} |
| 127 | scanDone chan struct{} |
| 128 | lastHash [sha256.Size]byte |
| 129 | hasHash bool |
| 130 | backoffIx int |
| 131 | } |
| 132 | |
| 133 | // Service owns physical watches for any number of subscription roots. |
| 134 | type Service struct { |
| 135 | stderr io.Writer |
| 136 | |
| 137 | mu sync.Mutex |
| 138 | roots map[string]*rootState |
| 139 | nextID uint64 |
| 140 | closed bool |
| 141 | stats Diagnostics |
| 142 | backend backend |
| 143 | backendKind string |
| 144 | helper *helperClient |
| 145 | } |
| 146 | |
| 147 | // NewService builds the host watch service. The backend is chosen once: the |
| 148 | // helper process where available (Windows by default), otherwise in-process |
| 149 | // native watches. |
| 150 | func NewService(opts Options) *Service { |
| 151 | stderr := opts.Stderr |
| 152 | if stderr == nil { |
| 153 | stderr = io.Discard |
| 154 | } |
| 155 | svc := &Service{ |
| 156 | stderr: stderr, |
| 157 | roots: map[string]*rootState{}, |
| 158 | // Registration ids start at 1: zero means "no previous registration" |
| 159 | // in the cancel-on-reregister bookkeeping. |
| 160 | nextID: 1, |
| 161 | } |
| 162 | svc.backend, svc.backendKind, svc.helper = newPlatformBackend(svc, opts) |
| 163 | return svc |
| 164 | } |
| 165 | |
| 166 | // Subscribe adds one logical subscription for root. It registers the physical |
| 167 | // watches before returning (bounded on the helper path) so the caller's first |
| 168 | // scan cannot race the watch into missing changes. A registration failure |
| 169 | // degrades the root to backoff scanning instead of failing the store. |
| 170 | func (s *Service) Subscribe(root string, maxDepth int, scope ScopeFunc, hash HashFunc, onChange func(reason string)) *Subscription { |
| 171 | if abs, err := filepath.Abs(root); err == nil { |
| 172 | root = abs |
| 173 | } |
| 174 | if resolved, err := filepath.EvalSymlinks(root); err == nil { |
| 175 | root = resolved |
| 176 | } |
| 177 | sub := &Subscription{ |
| 178 | svc: s, |
| 179 | root: root, |
| 180 | maxDepth: maxDepth, |
| 181 | onChange: onChange, |
| 182 | } |
| 183 | s.mu.Lock() |
| 184 | if s.closed { |
| 185 | s.mu.Unlock() |
| 186 | // A closed service hands out dead subscriptions; stores built during |
| 187 | // teardown keep working from their initial scan. |
| 188 | return sub |
| 189 | } |
| 190 | state := s.roots[root] |
| 191 | if state == nil { |
| 192 | state = &rootState{ |
| 193 | canonical: root, subs: map[*Subscription]struct{}{}, |
| 194 | maxDepth: maxDepth, scope: scope, hash: hash, |
| 195 | } |
| 196 | s.roots[root] = state |
| 197 | } |
| 198 | state.subs[sub] = struct{}{} |
| 199 | raised := maxDepth > state.maxDepth |
| 200 | if raised { |
| 201 | state.maxDepth = maxDepth |
| 202 | } |
| 203 | s.stats.LogicalSubscriptions++ |
| 204 | if len(state.dirs) == 0 || raised || state.scopeDirty { |
| 205 | s.registerRootLocked(state) |
| 206 | } |
| 207 | regDone := state.regDone |
| 208 | kind := s.backendKind |
| 209 | s.mu.Unlock() |
| 210 | // Preserve register-before-scan ordering. Native registration is local and |
| 211 | // must settle before Subscribe returns. Helper confirmation travels over a |
| 212 | // pipe, so only that path needs a timeout to keep a wedged child bounded. |
| 213 | if regDone != nil && kind == "native" { |
| 214 | <-regDone |
| 215 | } |
| 216 | if regDone != nil && kind == "helper" { |
| 217 | timer := time.NewTimer(helperSubscribeWait) |
| 218 | defer timer.Stop() |
| 219 | select { |
| 220 | case <-regDone: |
| 221 | case <-timer.C: |
| 222 | } |
| 223 | } |
| 224 | return sub |
| 225 | } |
| 226 | |
| 227 | // registerRootLocked rebuilds the watch scope for one root and hands it to the |
| 228 | // backend. Caller holds s.mu. Registration is generation fenced: any result |
| 229 | // for an older generation is discarded. The superseded registration is |
| 230 | // cancelled once the new one settles, so a root keeps at most one live |
| 231 | // backend registration and physical watches never accumulate. |
| 232 | func (s *Service) registerRootLocked(state *rootState) { |
| 233 | state.gen++ |
| 234 | gen := state.gen |
| 235 | dirs, _ := state.scope(context.Background(), state.canonical, state.maxDepth) |
| 236 | state.dirs = dirs |
| 237 | state.scopeDirty = false |
| 238 | id := s.nextID |
| 239 | s.nextID++ |
| 240 | oldID := state.watchID |
| 241 | state.watchID = id |
| 242 | done := make(chan struct{}) |
| 243 | state.regDone = done |
| 244 | go func() { |
| 245 | defer close(done) |
| 246 | err := s.backend.register(id, gen, state.canonical, dirs) |
| 247 | if oldID != 0 { |
| 248 | s.backend.cancel(oldID) |
| 249 | } |
| 250 | s.mu.Lock() |
| 251 | if state.gen != gen { |
| 252 | // Superseded by a newer registration, which owns cancelling id. |
| 253 | s.mu.Unlock() |
| 254 | return |
| 255 | } |
| 256 | if len(state.subs) == 0 { |
| 257 | // Fully released while registering: reclaim our own registration. |
| 258 | s.mu.Unlock() |
| 259 | s.backend.cancel(id) |
| 260 | return |
| 261 | } |
| 262 | if err != nil { |
| 263 | s.degradeRootLocked(state, err) |
| 264 | s.mu.Unlock() |
| 265 | return |
| 266 | } |
| 267 | if state.degraded { |
| 268 | s.stopScanLocked(state) |
| 269 | } |
| 270 | s.mu.Unlock() |
| 271 | }() |
| 272 | } |
| 273 | |
| 274 | func (s *Service) dropSubscription(sub *Subscription) { |
| 275 | s.mu.Lock() |
| 276 | state := s.roots[sub.root] |
| 277 | if state == nil { |
| 278 | s.mu.Unlock() |
| 279 | return |
| 280 | } |
| 281 | if _, ok := state.subs[sub]; !ok { |
| 282 | s.mu.Unlock() |
| 283 | return |
| 284 | } |
| 285 | delete(state.subs, sub) |
| 286 | if s.stats.LogicalSubscriptions > 0 { |
| 287 | s.stats.LogicalSubscriptions-- |
| 288 | } |
| 289 | if len(state.subs) > 0 { |
| 290 | s.mu.Unlock() |
| 291 | return |
| 292 | } |
| 293 | // Last subscription gone: release the physical watches. Reference counting |
| 294 | // to zero is what keeps historical project roots from accumulating. |
| 295 | id := state.watchID |
| 296 | var scanDone chan struct{} |
| 297 | if state.degraded { |
| 298 | scanDone = state.scanDone |
| 299 | s.stopScanLocked(state) // closes scanStop |
| 300 | } |
| 301 | s.cancelTimerLocked(state) |
| 302 | delete(s.roots, sub.root) |
| 303 | s.mu.Unlock() |
| 304 | |
| 305 | // Backend IO happens outside the lock: cancel never waits on registration. |
| 306 | if id != 0 { |
| 307 | s.backend.cancel(id) |
| 308 | } |
| 309 | if scanDone != nil { |
| 310 | <-scanDone |
| 311 | } |
| 312 | } |
| 313 | |
| 314 | // eventArrived receives one raw backend event. Events for unknown, released or |
| 315 | // superseded registrations are dropped without producing a notification. |
| 316 | func (s *Service) eventArrived(id, rootGen uint64, op Op) { |
| 317 | s.mu.Lock() |
| 318 | defer s.mu.Unlock() |
| 319 | if s.closed { |
| 320 | return |
| 321 | } |
| 322 | var state *rootState |
| 323 | for _, r := range s.roots { |
| 324 | if r.watchID == id { |
| 325 | state = r |
| 326 | break |
| 327 | } |
| 328 | } |
| 329 | if state == nil || state.gen != rootGen || len(state.subs) == 0 { |
| 330 | return |
| 331 | } |
| 332 | s.stats.EventsReceived++ |
| 333 | if state.pending { |
| 334 | // Extend the window unless that would exceed the maximum delay. |
| 335 | if time.Since(state.firstSeen)+coalesceWindow <= coalesceMaxDelay { |
| 336 | state.timer.Reset(coalesceWindow) |
| 337 | if op&(OpCreate|OpRename) != 0 { |
| 338 | state.scopeDirty = true |
| 339 | } |
| 340 | return |
| 341 | } |
| 342 | s.fireRootLocked(state) |
| 343 | } |
| 344 | state.pending = true |
| 345 | state.firstSeen = time.Now() |
| 346 | state.timer = time.AfterFunc(coalesceWindow, func() { |
| 347 | s.mu.Lock() |
| 348 | defer s.mu.Unlock() |
| 349 | if state.pending { |
| 350 | s.fireRootLocked(state) |
| 351 | } |
| 352 | }) |
| 353 | if op&(OpCreate|OpRename) != 0 { |
| 354 | // A create/rename can introduce a nested directory, a symlink target or |
| 355 | // a previously missing root; rebuild subscriptions after notification. |
| 356 | state.scopeDirty = true |
| 357 | } |
| 358 | } |
| 359 | |
| 360 | // fireRootLocked delivers the coalesced notification and applies a deferred |
| 361 | // scope rebuild. Caller holds s.mu. |
| 362 | func (s *Service) fireRootLocked(state *rootState) { |
| 363 | s.cancelTimerLocked(state) |
| 364 | state.pending = false |
| 365 | s.stats.Notifications++ |
| 366 | if state.scopeDirty { |
| 367 | s.registerRootLocked(state) |
| 368 | } |
| 369 | for sub := range state.subs { |
| 370 | sub.onChange("filesystem changed") |
| 371 | } |
| 372 | } |
| 373 | |
| 374 | func (s *Service) cancelTimerLocked(state *rootState) { |
| 375 | if state.timer != nil { |
| 376 | state.timer.Stop() |
| 377 | state.timer = nil |
| 378 | } |
| 379 | } |
| 380 | |
| 381 | // degradeRootLocked switches a root to bounded signature scanning. The helper |
| 382 | // restart budget is spent by the helper client before this point; degraded |
| 383 | // roots recover automatically when a later registration succeeds. |
| 384 | // Caller holds s.mu. |
| 385 | func (s *Service) degradeRootLocked(state *rootState, cause error) { |
| 386 | if state.degraded { |
| 387 | return |
| 388 | } |
| 389 | state.degraded = true |
| 390 | s.stats.DegradedRoots++ |
| 391 | s.warnf("skillwatch: root %s degraded to scan fallback: %v", state.canonical, cause) |
| 392 | state.scanStop = make(chan struct{}) |
| 393 | state.scanDone = make(chan struct{}) |
| 394 | stop, done := state.scanStop, state.scanDone |
| 395 | go func() { |
| 396 | s.scanLoop(state, stop, done) |
| 397 | }() |
| 398 | } |
| 399 | |
| 400 | func (s *Service) stopScanLocked(state *rootState) { |
| 401 | if !state.degraded { |
| 402 | return |
| 403 | } |
| 404 | state.degraded = false |
| 405 | if s.stats.DegradedRoots > 0 { |
| 406 | s.stats.DegradedRoots-- |
| 407 | } |
| 408 | if state.scanStop != nil { |
| 409 | close(state.scanStop) |
| 410 | state.scanStop = nil |
| 411 | } |
| 412 | state.backoffIx = 0 |
| 413 | state.hasHash = false |
| 414 | } |
| 415 | |
| 416 | // scanLoop is the degraded-mode replacement for the retired 250ms polling |
| 417 | // generation: one scan at a time per root, growing 2/5/15/30s backoff while |
| 418 | // the watch backend stays unavailable. A signature diff notifies subscribers |
| 419 | // exactly like a native event would, and every pass retries the backend so a |
| 420 | // recovered helper or filesystem returns the root to native watching. |
| 421 | func (s *Service) scanLoop(state *rootState, stop, done chan struct{}) { |
| 422 | defer close(done) |
| 423 | for { |
| 424 | interval := s.nextScanInterval(state) |
| 425 | select { |
| 426 | case <-stop: |
| 427 | return |
| 428 | case <-time.After(interval): |
| 429 | } |
| 430 | s.mu.Lock() |
| 431 | if s.closed || !state.degraded || len(state.subs) == 0 { |
| 432 | s.mu.Unlock() |
| 433 | return |
| 434 | } |
| 435 | gen, hash, depth := state.gen, state.hash, state.maxDepth |
| 436 | s.mu.Unlock() |
| 437 | |
| 438 | sum, entries, ok := hash(context.Background(), state.canonical, depth) |
| 439 | |
| 440 | s.mu.Lock() |
| 441 | s.stats.Scans++ |
| 442 | s.stats.ScannedEntries += uint64(entries) |
| 443 | if state.gen != gen || !state.degraded { |
| 444 | s.mu.Unlock() |
| 445 | return |
| 446 | } |
| 447 | if ok && (!state.hasHash || sum != state.lastHash) { |
| 448 | state.lastHash, state.hasHash = sum, true |
| 449 | for sub := range state.subs { |
| 450 | sub.onChange("filesystem changed (scan fallback)") |
| 451 | } |
| 452 | } |
| 453 | // Every pass retries the watch backend: a recovered helper or |
| 454 | // filesystem flips the root back to native watching, and |
| 455 | // registerRootLocked itself stops the scan on success. |
| 456 | s.registerRootLocked(state) |
| 457 | s.mu.Unlock() |
| 458 | } |
| 459 | } |
| 460 | |
| 461 | func (s *Service) nextScanInterval(state *rootState) time.Duration { |
| 462 | s.mu.Lock() |
| 463 | defer s.mu.Unlock() |
| 464 | ix := state.backoffIx |
| 465 | if ix >= len(scanBackoffs) { |
| 466 | ix = len(scanBackoffs) - 1 |
| 467 | } |
| 468 | state.backoffIx++ |
| 469 | return scanBackoffs[ix] |
| 470 | } |
| 471 | |
| 472 | // helperDied tells the service the helper exhausted its restart budget (or |
| 473 | // never started): every live root re-registers, fails, and degrades to its |
| 474 | // scan fallback. Recovery from degraded state needs a working backend, which |
| 475 | // a degraded host regains only through the per-pass re-registration in |
| 476 | // scanLoop once the helper serves again. |
| 477 | func (s *Service) helperDied() { |
| 478 | s.mu.Lock() |
| 479 | defer s.mu.Unlock() |
| 480 | if s.closed { |
| 481 | return |
| 482 | } |
| 483 | states := make([]*rootState, 0, len(s.roots)) |
| 484 | for _, state := range s.roots { |
| 485 | states = append(states, state) |
| 486 | } |
| 487 | for _, state := range states { |
| 488 | if len(state.subs) > 0 { |
| 489 | s.registerRootLocked(state) |
| 490 | } |
| 491 | } |
| 492 | } |
| 493 | |
| 494 | func (s *Service) helperRestarted() { |
| 495 | s.mu.Lock() |
| 496 | s.stats.HelperRestarts++ |
| 497 | s.mu.Unlock() |
| 498 | } |
| 499 | |
| 500 | func (s *Service) warnf(format string, args ...any) { |
| 501 | // Diagnostics only: counts, root paths and failure causes, never contents. |
| 502 | _, _ = fmt.Fprintf(s.stderr, format+"\n", args...) |
| 503 | } |
| 504 | |
| 505 | // Diagnostics returns a snapshot of the resource counters. |
| 506 | func (s *Service) Diagnostics() Diagnostics { |
| 507 | s.mu.Lock() |
| 508 | defer s.mu.Unlock() |
| 509 | out := s.stats |
| 510 | out.PhysicalWatches = s.backend.physicalWatches() |
| 511 | out.DegradedRoots = 0 |
| 512 | for _, state := range s.roots { |
| 513 | if state.degraded { |
| 514 | out.DegradedRoots++ |
| 515 | } |
| 516 | } |
| 517 | return out |
| 518 | } |
| 519 | |
| 520 | // Close tears the service down: every subscription dies, physical watches and |
| 521 | // the helper process are reclaimed. It is idempotent and safe to call during |
| 522 | // application exit; it never waits on uninterruptible backend registration. |
| 523 | func (s *Service) Close() error { |
| 524 | s.mu.Lock() |
| 525 | if s.closed { |
| 526 | s.mu.Unlock() |
| 527 | return nil |
| 528 | } |
| 529 | s.closed = true |
| 530 | type teardown struct { |
| 531 | id uint64 |
| 532 | done chan struct{} |
| 533 | } |
| 534 | steps := make([]teardown, 0, len(s.roots)*2) |
| 535 | for _, state := range s.roots { |
| 536 | s.cancelTimerLocked(state) |
| 537 | if state.degraded && state.scanStop != nil { |
| 538 | close(state.scanStop) |
| 539 | state.scanStop = nil |
| 540 | steps = append(steps, teardown{done: state.scanDone}) |
| 541 | } |
| 542 | if state.watchID != 0 { |
| 543 | steps = append(steps, teardown{id: state.watchID}) |
| 544 | } |
| 545 | } |
| 546 | s.roots = map[string]*rootState{} |
| 547 | s.stats.LogicalSubscriptions = 0 |
| 548 | s.mu.Unlock() |
| 549 | for _, st := range steps { |
| 550 | if st.id != 0 { |
| 551 | s.backend.cancel(st.id) |
| 552 | } |
| 553 | if st.done != nil { |
| 554 | <-st.done |
| 555 | } |
| 556 | } |
| 557 | return s.backend.close() |
| 558 | } |
| 559 |