| 1 | package taskmonitor |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | crand "crypto/rand" |
| 6 | "crypto/sha256" |
| 7 | "encoding/hex" |
| 8 | "errors" |
| 9 | "fmt" |
| 10 | "sync" |
| 11 | "sync/atomic" |
| 12 | "time" |
| 13 | |
| 14 | "reasonix/internal/jobs" |
| 15 | ) |
| 16 | |
| 17 | // TaskRecorder bridges jobs.Manager lifecycle events into the Task Store. It |
| 18 | // is the write side of task monitoring: RecordStart persists a running |
| 19 | // snapshot, RecordDone advances it to its terminal state. All failures are |
| 20 | // swallowed — monitoring is best-effort and must never break the job pipeline. |
| 21 | // The store's per-task lock keeps concurrent recorders (CLI + Desktop) safe. |
| 22 | type TaskRecorder struct { |
| 23 | store WriteStore |
| 24 | projectDir string |
| 25 | sessionIDFn func() string |
| 26 | mu sync.Mutex |
| 27 | monitorIDs map[string]string |
| 28 | heartbeats map[string]context.CancelFunc |
| 29 | runtimeOwnerID string |
| 30 | } |
| 31 | |
| 32 | // NewTaskRecorder returns a TaskRecorder writing to store under projectDir. |
| 33 | // sessionIDFn is called per record so the snapshot reflects the session id at |
| 34 | // creation time (controllers resolve their session path lazily); it may return |
| 35 | // "" when no session is bound yet. |
| 36 | func NewTaskRecorder(store WriteStore, projectDir string, sessionIDFn func() string) *TaskRecorder { |
| 37 | return &TaskRecorder{store: store, projectDir: projectDir, sessionIDFn: sessionIDFn, monitorIDs: make(map[string]string), heartbeats: make(map[string]context.CancelFunc), runtimeOwnerID: newRuntimeOwnerID()} |
| 38 | } |
| 39 | |
| 40 | func (r *TaskRecorder) RuntimeOwnerID() string { return r.runtimeOwnerID } |
| 41 | |
| 42 | const ( |
| 43 | runtimeLeaseTTL = 30 * time.Second |
| 44 | runtimeHeartbeatEvery = 5 * time.Second |
| 45 | ) |
| 46 | |
| 47 | var runtimeOwnerSequence atomic.Uint64 |
| 48 | |
| 49 | func newRuntimeOwnerID() string { |
| 50 | var nonce [16]byte |
| 51 | if _, err := crand.Read(nonce[:]); err == nil { |
| 52 | return hex.EncodeToString(nonce[:]) |
| 53 | } |
| 54 | h := sha256.Sum256(fmt.Appendf(nil, "%d:%d", timeNow().UnixNano(), runtimeOwnerSequence.Add(1))) |
| 55 | return hex.EncodeToString(h[:16]) |
| 56 | } |
| 57 | |
| 58 | // monitorTaskID creates a globally unique monitor identity for a job within |
| 59 | // a session. jobs.Manager IDs are local to a manager and restart from task-1 |
| 60 | // for every session, so persisting the raw job ID would cause cross-session |
| 61 | // overwrites in the shared project store. |
| 62 | func monitorTaskID(sessionID, jobID string) string { |
| 63 | if sessionID == "" { |
| 64 | return jobID |
| 65 | } |
| 66 | id := fmt.Sprintf("%s--%s", sessionID, jobID) |
| 67 | if len(id) <= maxFieldLen { |
| 68 | return id |
| 69 | } |
| 70 | h := sha256.Sum256([]byte(sessionID)) |
| 71 | return hex.EncodeToString(h[:8]) + "--" + jobID |
| 72 | } |
| 73 | |
| 74 | func sessionlessMonitorTaskID(jobID string) string { |
| 75 | var nonce [8]byte |
| 76 | if _, err := crand.Read(nonce[:]); err != nil { |
| 77 | // crypto/rand failure is exceptional; retain a bounded, non-path-like |
| 78 | // identity rather than falling back to the colliding raw job ID. |
| 79 | h := sha256.Sum256(fmt.Appendf(nil, "%s:%d", jobID, timeNow().UnixNano())) |
| 80 | return hex.EncodeToString(h[:8]) + "--" + jobID |
| 81 | } |
| 82 | return hex.EncodeToString(nonce[:]) + "--" + jobID |
| 83 | } |
| 84 | |
| 85 | func (r *TaskRecorder) rememberMonitorID(jobID, monitorID string) { |
| 86 | r.mu.Lock() |
| 87 | r.monitorIDs[jobID] = monitorID |
| 88 | r.mu.Unlock() |
| 89 | } |
| 90 | |
| 91 | func (r *TaskRecorder) lookupMonitorID(jobID string) (string, bool) { |
| 92 | r.mu.Lock() |
| 93 | monitorID, ok := r.monitorIDs[jobID] |
| 94 | r.mu.Unlock() |
| 95 | return monitorID, ok |
| 96 | } |
| 97 | |
| 98 | func (r *TaskRecorder) startHeartbeat(monitorID string) { |
| 99 | ctx, cancel := context.WithCancel(context.Background()) |
| 100 | r.mu.Lock() |
| 101 | if old := r.heartbeats[monitorID]; old != nil { |
| 102 | old() |
| 103 | } |
| 104 | r.heartbeats[monitorID] = cancel |
| 105 | r.mu.Unlock() |
| 106 | go func() { |
| 107 | ticker := time.NewTicker(runtimeHeartbeatEvery) |
| 108 | defer ticker.Stop() |
| 109 | for { |
| 110 | select { |
| 111 | case <-ctx.Done(): |
| 112 | return |
| 113 | case <-ticker.C: |
| 114 | if !r.renewHeartbeat(ctx, monitorID) { |
| 115 | return |
| 116 | } |
| 117 | } |
| 118 | } |
| 119 | }() |
| 120 | } |
| 121 | |
| 122 | func (r *TaskRecorder) renewHeartbeat(ctx context.Context, monitorID string) bool { |
| 123 | renewed, err := r.store.RenewRuntimeLease(ctx, r.projectDir, monitorID, r.runtimeOwnerID, timeNow().Add(runtimeLeaseTTL)) |
| 124 | return err == nil && renewed |
| 125 | } |
| 126 | |
| 127 | func (r *TaskRecorder) stopHeartbeat(monitorID string) { |
| 128 | r.mu.Lock() |
| 129 | if cancel := r.heartbeats[monitorID]; cancel != nil { |
| 130 | cancel() |
| 131 | delete(r.heartbeats, monitorID) |
| 132 | } |
| 133 | r.mu.Unlock() |
| 134 | } |
| 135 | |
| 136 | // RecordStart implements jobs.TaskRecorder. |
| 137 | func (r *TaskRecorder) RecordStart(id, kind, label string) { |
| 138 | ctx := context.Background() |
| 139 | now := timeNow() |
| 140 | sessionID := "" |
| 141 | if r.sessionIDFn != nil { |
| 142 | sessionID = r.sessionIDFn() |
| 143 | } |
| 144 | monitorID := monitorTaskID(sessionID, id) |
| 145 | if sessionID == "" { |
| 146 | monitorID = sessionlessMonitorTaskID(id) |
| 147 | } |
| 148 | r.rememberMonitorID(id, monitorID) |
| 149 | snap := TaskSnapshot{ |
| 150 | SchemaVersion: 1, |
| 151 | TaskID: monitorID, |
| 152 | JobID: id, |
| 153 | SessionID: sessionID, |
| 154 | State: TaskStateRunning, |
| 155 | RuntimeState: RuntimeStateAlive, |
| 156 | RuntimeLeaseUntil: now.Add(runtimeLeaseTTL), |
| 157 | RuntimeOwnerID: r.runtimeOwnerID, |
| 158 | Version: 1, |
| 159 | CreatedAt: now, |
| 160 | UpdatedAt: now, |
| 161 | } |
| 162 | if err := r.store.SaveTask(ctx, r.projectDir, snap); err != nil { |
| 163 | return |
| 164 | } |
| 165 | _ = r.store.AppendAuditEvent(ctx, r.projectDir, TaskEvent{ |
| 166 | Timestamp: now, EventType: "state_change", |
| 167 | TaskID: monitorID, SessionID: sessionID, State: TaskStateRunning, |
| 168 | RuntimeState: RuntimeStateAlive, |
| 169 | }) |
| 170 | r.startHeartbeat(monitorID) |
| 171 | } |
| 172 | |
| 173 | // RecordDone implements jobs.TaskRecorder. |
| 174 | func (r *TaskRecorder) RecordDone(id string, st jobs.Status, jobErr error) { |
| 175 | ctx := context.Background() |
| 176 | target := terminalState(st) |
| 177 | if target == "" { |
| 178 | return // non-terminal/unknown status: leave the snapshot untouched |
| 179 | } |
| 180 | monitorID, ok := r.lookupMonitorID(id) |
| 181 | if !ok { |
| 182 | return // no matching lifecycle was recorded by this recorder |
| 183 | } |
| 184 | r.stopHeartbeat(monitorID) |
| 185 | const maxSaveAttempts = 4 |
| 186 | for range maxSaveAttempts { |
| 187 | cur, gerr := r.store.GetTask(ctx, r.projectDir, monitorID) |
| 188 | if gerr != nil || cur == nil { |
| 189 | return // never recorded (recorder attached after the job started) |
| 190 | } |
| 191 | if cur.RuntimeOwnerID != "" && cur.RuntimeOwnerID != r.runtimeOwnerID { |
| 192 | return // a newer recorder generation owns this reused task identity |
| 193 | } |
| 194 | now := timeNow() |
| 195 | cur.State = target |
| 196 | cur.RuntimeState = RuntimeStateExited |
| 197 | cur.RuntimeLeaseUntil = time.Time{} |
| 198 | cur.RuntimeOwnerID = "" |
| 199 | cur.Version++ |
| 200 | cur.UpdatedAt = now |
| 201 | cur.ErrorSummary = "" |
| 202 | if jobErr != nil { |
| 203 | cur.ErrorCode = "job_failed" |
| 204 | } |
| 205 | if serr := r.store.SaveTask(ctx, r.projectDir, *cur); serr != nil { |
| 206 | if errors.Is(serr, ErrStoreVersionConflict) { |
| 207 | continue |
| 208 | } |
| 209 | return |
| 210 | } |
| 211 | _ = r.store.AppendAuditEvent(ctx, r.projectDir, TaskEvent{ |
| 212 | Timestamp: now, EventType: "state_change", |
| 213 | TaskID: monitorID, SessionID: cur.SessionID, State: target, |
| 214 | RuntimeState: RuntimeStateExited, |
| 215 | ErrorCode: cur.ErrorCode, ErrorSummary: cur.ErrorSummary, |
| 216 | }) |
| 217 | return |
| 218 | } |
| 219 | } |
| 220 | |
| 221 | // terminalState maps a job status to the task state it reports. Unknown or |
| 222 | // non-terminal statuses map to "" (no update). |
| 223 | func terminalState(st jobs.Status) TaskState { |
| 224 | switch st { |
| 225 | case jobs.Done: |
| 226 | return TaskStateSucceeded |
| 227 | case jobs.Failed: |
| 228 | return TaskStateFailed |
| 229 | case jobs.Killed, jobs.Interrupted: |
| 230 | return TaskStateCancelled |
| 231 | default: |
| 232 | return "" |
| 233 | } |
| 234 | } |
| 235 |