| 1 | import { spawn as nodeSpawn, type ChildProcess, type SpawnOptions } from "node:child_process"; |
| 2 | import { randomUUID } from "node:crypto"; |
| 3 | import type { EventFrame, ServiceState } from "../shared/ipc.js"; |
| 4 | import { eventFrame } from "../shared/eventStream.js"; |
| 5 | import type { HelloResult } from "./handshake.js"; |
| 6 | import { errorText, type Logger } from "./log.js"; |
| 7 | import { RestartBudget } from "./restartBudget.js"; |
| 8 | import { RpcClient } from "./rpc.js"; |
| 9 | |
| 10 | export const LIFECYCLE_TIMEOUT_MS = 10_000; |
| 11 | const SHUTDOWN_STATUS_POLL_MS = 250; |
| 12 | export const EXIT_GRACE_MS = 5_000; |
| 13 | |
| 14 | export interface ServiceHandlers { |
| 15 | hello(client: RpcClient): Promise<HelloResult>; |
| 16 | onRequest(method: string, params: unknown): Promise<unknown>; |
| 17 | onEvent(frame: EventFrame): void; |
| 18 | onState(state: ServiceState): void; |
| 19 | onReady(hello: HelloResult, restarted: boolean): void | Promise<void>; |
| 20 | onFailed(error: unknown): void; |
| 21 | } |
| 22 | |
| 23 | export type SpawnFn = (command: string, args: string[], options: SpawnOptions) => ChildProcess; |
| 24 | |
| 25 | export interface ServiceOptions { |
| 26 | binary: string; |
| 27 | args: string[]; |
| 28 | env: NodeJS.ProcessEnv; |
| 29 | onStderr(chunk: Buffer): void; |
| 30 | log: Logger; |
| 31 | spawn?: SpawnFn; |
| 32 | budget?: RestartBudget; |
| 33 | now?: () => number; |
| 34 | exitGraceMs?: number; |
| 35 | } |
| 36 | |
| 37 | interface Session { |
| 38 | child: ChildProcess; |
| 39 | client: RpcClient; |
| 40 | generation: string; |
| 41 | alive: boolean; |
| 42 | ready: boolean; |
| 43 | eventSeq: number; |
| 44 | expectExit: boolean; |
| 45 | exited: Promise<void>; |
| 46 | } |
| 47 | |
| 48 | type ShutdownResult = { |
| 49 | requestId: string; |
| 50 | reason: string; |
| 51 | phase: "idle" | "preparing" | "saving" | "closing" | "completed"; |
| 52 | outcome: "not_started" | "in_progress" | "success" | "failed"; |
| 53 | completed: boolean; |
| 54 | retryable: boolean; |
| 55 | errorCode?: string; |
| 56 | error?: string; |
| 57 | }; |
| 58 | |
| 59 | export type ShutdownPhase = ShutdownResult["phase"]; |
| 60 | |
| 61 | function shutdownResult(value: unknown): ShutdownResult { |
| 62 | if (!value || typeof value !== "object") throw new Error("desktop shutdown returned an invalid result"); |
| 63 | const result = value as Partial<ShutdownResult>; |
| 64 | if ( |
| 65 | typeof result.requestId !== "string" || |
| 66 | typeof result.phase !== "string" || |
| 67 | typeof result.outcome !== "string" || |
| 68 | typeof result.completed !== "boolean" || |
| 69 | typeof result.retryable !== "boolean" |
| 70 | ) { |
| 71 | throw new Error("desktop shutdown returned an invalid result"); |
| 72 | } |
| 73 | return result as ShutdownResult; |
| 74 | } |
| 75 | |
| 76 | function describeExit(code: number | null, signal: NodeJS.Signals | null): string { |
| 77 | return signal ? `signal ${signal}` : `code ${code ?? "unknown"}`; |
| 78 | } |
| 79 | |
| 80 | function delay(ms: number): Promise<void> { |
| 81 | return new Promise((resolve) => setTimeout(resolve, ms)); |
| 82 | } |
| 83 | |
| 84 | export class ServiceSupervisor { |
| 85 | private session: Session | null = null; |
| 86 | private state: ServiceState = { phase: "starting", generation: "" }; |
| 87 | private hello: HelloResult | null = null; |
| 88 | private launching: Promise<HelloResult> | null = null; |
| 89 | private stopping = false; |
| 90 | private shutdownPending: Promise<void> | null = null; |
| 91 | private shutdownRequestId = ""; |
| 92 | private revision = 0; |
| 93 | private restarting: Promise<HelloResult> | null = null; |
| 94 | private readonly budget: RestartBudget; |
| 95 | private readonly spawnFn: SpawnFn; |
| 96 | private readonly now: () => number; |
| 97 | private readonly exitGraceMs: number; |
| 98 | |
| 99 | constructor( |
| 100 | private readonly options: ServiceOptions, |
| 101 | private readonly handlers: ServiceHandlers, |
| 102 | ) { |
| 103 | this.budget = options.budget ?? new RestartBudget(); |
| 104 | this.spawnFn = options.spawn ?? ((command, args, spawnOptions) => nodeSpawn(command, args, spawnOptions)); |
| 105 | this.now = options.now ?? (() => Date.now()); |
| 106 | this.exitGraceMs = options.exitGraceMs ?? EXIT_GRACE_MS; |
| 107 | } |
| 108 | |
| 109 | get current(): ServiceState { |
| 110 | return this.state; |
| 111 | } |
| 112 | |
| 113 | get helloResult(): HelloResult | null { |
| 114 | return this.hello; |
| 115 | } |
| 116 | |
| 117 | get generation(): string { |
| 118 | return this.session?.alive && this.session.ready ? this.session.generation : ""; |
| 119 | } |
| 120 | |
| 121 | get ready(): boolean { |
| 122 | return !this.stopping && this.state.phase === "ready" && this.session?.alive === true; |
| 123 | } |
| 124 | |
| 125 | get shutdownRequestIdentity(): string { |
| 126 | return this.shutdownRequestId; |
| 127 | } |
| 128 | |
| 129 | start(): Promise<HelloResult> { |
| 130 | return this.begin(false); |
| 131 | } |
| 132 | |
| 133 | async restart(): Promise<HelloResult> { |
| 134 | if (this.stopping) throw new Error("desktop service is shutting down"); |
| 135 | if (this.launching) return this.launching; |
| 136 | if (!this.restarting) { |
| 137 | this.restarting = (async () => { |
| 138 | const old = this.session; |
| 139 | if (old?.alive) await this.terminate(old); |
| 140 | return this.begin(true); |
| 141 | })().finally(() => { |
| 142 | this.restarting = null; |
| 143 | }); |
| 144 | } |
| 145 | return this.restarting; |
| 146 | } |
| 147 | |
| 148 | async invoke(method: string, args: unknown[]): Promise<unknown> { |
| 149 | return this.live().client.request("desktop/invoke", { method, args }); |
| 150 | } |
| 151 | |
| 152 | async request(method: string, params: unknown, timeoutMs = LIFECYCLE_TIMEOUT_MS): Promise<unknown> { |
| 153 | return this.live().client.request(method, params, timeoutMs); |
| 154 | } |
| 155 | |
| 156 | async hostEvent(name: string, payload: unknown): Promise<void> { |
| 157 | try { |
| 158 | await this.request("desktop/hostEvent", { name, payload }); |
| 159 | } catch (error) { |
| 160 | this.options.log.warn(`hostEvent ${name} not delivered: ${errorText(error)}`); |
| 161 | } |
| 162 | } |
| 163 | |
| 164 | shutdown( |
| 165 | reason: "user_quit" | "update_restart" | "system_signal" = "user_quit", |
| 166 | onProgress?: (phase: ShutdownPhase) => void, |
| 167 | ): Promise<void> { |
| 168 | if (!this.stopping) { |
| 169 | this.stopping = true; |
| 170 | this.revision++; |
| 171 | this.setState({ phase: "stopping", generation: this.session?.generation ?? "" }); |
| 172 | } |
| 173 | if (!this.shutdownPending) { |
| 174 | this.shutdownPending = this.finishShutdown(reason, onProgress) |
| 175 | .catch((error) => { |
| 176 | if (this.state.phase !== "exited") { |
| 177 | this.setState({ phase: "stopping", generation: this.session?.generation ?? "", error: errorText(error) }); |
| 178 | } |
| 179 | throw error; |
| 180 | }) |
| 181 | .finally(() => { |
| 182 | this.shutdownPending = null; |
| 183 | }); |
| 184 | } |
| 185 | return this.shutdownPending; |
| 186 | } |
| 187 | |
| 188 | private async finishShutdown( |
| 189 | reason: "user_quit" | "update_restart" | "system_signal", |
| 190 | onProgress?: (phase: ShutdownPhase) => void, |
| 191 | ): Promise<void> { |
| 192 | // A restart may already own termination of the current generation. Let that |
| 193 | // operation settle before selecting the session to shut down, otherwise the |
| 194 | // shutdown RPC races a closing stdin and turns an intentional quit into an |
| 195 | // indeterminate transport error. |
| 196 | if (this.restarting) await this.restarting.catch(() => undefined); |
| 197 | const session = this.session; |
| 198 | if (!session?.alive) { |
| 199 | this.setState({ phase: "exited", generation: "" }); |
| 200 | return; |
| 201 | } |
| 202 | session.expectExit = true; |
| 203 | if (!this.shutdownRequestId) this.shutdownRequestId = randomUUID(); |
| 204 | let result: ShutdownResult | null = null; |
| 205 | try { |
| 206 | result = shutdownResult( |
| 207 | await session.client.request( |
| 208 | "desktop/shutdown", |
| 209 | { |
| 210 | requestId: this.shutdownRequestId, |
| 211 | reason, |
| 212 | }, |
| 213 | LIFECYCLE_TIMEOUT_MS, |
| 214 | ), |
| 215 | ); |
| 216 | } catch (error) { |
| 217 | this.options.log.warn(`desktop/shutdown result unknown: ${errorText(error)}; querying status`); |
| 218 | result = shutdownResult( |
| 219 | await session.client.request( |
| 220 | "desktop/shutdownStatus", |
| 221 | { |
| 222 | requestId: this.shutdownRequestId, |
| 223 | }, |
| 224 | LIFECYCLE_TIMEOUT_MS, |
| 225 | ), |
| 226 | ); |
| 227 | } |
| 228 | const deadline = Date.now() + LIFECYCLE_TIMEOUT_MS; |
| 229 | onProgress?.(result.phase); |
| 230 | while (!result.completed && result.outcome === "in_progress" && Date.now() < deadline) { |
| 231 | await new Promise((resolve) => setTimeout(resolve, SHUTDOWN_STATUS_POLL_MS)); |
| 232 | result = shutdownResult( |
| 233 | await session.client.request( |
| 234 | "desktop/shutdownStatus", |
| 235 | { |
| 236 | requestId: this.shutdownRequestId, |
| 237 | }, |
| 238 | LIFECYCLE_TIMEOUT_MS, |
| 239 | ), |
| 240 | ); |
| 241 | onProgress?.(result.phase); |
| 242 | } |
| 243 | if (result.outcome === "failed") { |
| 244 | throw new Error(`${result.errorCode ?? "shutdown_failed"}: ${result.error ?? "desktop shutdown failed"}`); |
| 245 | } |
| 246 | if (!result.completed || result.outcome !== "success") { |
| 247 | throw new Error( |
| 248 | `${result.errorCode ?? "shutdown_incomplete"}: ${result.error ?? `desktop shutdown stopped in ${result.phase}`}`, |
| 249 | ); |
| 250 | } |
| 251 | // The service closes itself only after the completed result has reached the |
| 252 | // shell. stdin/kill are now a post-completion process-exit fallback. |
| 253 | await this.terminate(session); |
| 254 | this.setState({ phase: "exited", generation: "" }); |
| 255 | } |
| 256 | |
| 257 | private live(): Session { |
| 258 | const session = this.session; |
| 259 | if (this.stopping) throw new Error("desktop service is shutting down"); |
| 260 | if (!session?.alive || !session.ready) throw new Error(`desktop service is not running (${this.state.phase})`); |
| 261 | return session; |
| 262 | } |
| 263 | |
| 264 | private begin(restarted: boolean): Promise<HelloResult> { |
| 265 | if (this.stopping) return Promise.reject(new Error("desktop service is shutting down")); |
| 266 | if (this.launching) return this.launching; |
| 267 | this.launching = this.launch(restarted).finally(() => { |
| 268 | this.launching = null; |
| 269 | }); |
| 270 | return this.launching; |
| 271 | } |
| 272 | |
| 273 | private async launch(restarted: boolean): Promise<HelloResult> { |
| 274 | const revision = ++this.revision; |
| 275 | this.setState({ phase: restarted ? "restarting" : "starting", generation: "" }); |
| 276 | let session: Session | null = null; |
| 277 | try { |
| 278 | session = this.spawnSession(); |
| 279 | const hello = await this.handlers.hello(session.client); |
| 280 | if (this.stopping || revision !== this.revision) throw new Error("desktop startup cancelled"); |
| 281 | session.generation = hello.runtimeGeneration; |
| 282 | await session.client.request("desktop/start", {}, LIFECYCLE_TIMEOUT_MS); |
| 283 | if (!session.alive || this.stopping || revision !== this.revision) throw new Error("desktop service exited during startup"); |
| 284 | session.ready = true; |
| 285 | this.hello = hello; |
| 286 | this.setState({ phase: "ready", generation: hello.runtimeGeneration }); |
| 287 | await this.handlers.onReady(hello, restarted); |
| 288 | return hello; |
| 289 | } catch (error) { |
| 290 | if ((!session || this.session === session) && !this.stopping && revision === this.revision) { |
| 291 | this.setState({ phase: "failed", generation: "", error: errorText(error) }); |
| 292 | this.handlers.onFailed(error); |
| 293 | if (session) void this.terminate(session).catch((error) => this.options.log.error(`service termination failed: ${errorText(error)}`)); |
| 294 | } |
| 295 | throw error; |
| 296 | } |
| 297 | } |
| 298 | |
| 299 | private spawnSession(): Session { |
| 300 | const { binary, args, env, log } = this.options; |
| 301 | const child = this.spawnFn(binary, args, { |
| 302 | stdio: ["pipe", "pipe", "pipe"], |
| 303 | windowsHide: true, |
| 304 | env, |
| 305 | }); |
| 306 | const session: Session = { |
| 307 | child, |
| 308 | client: null as unknown as RpcClient, |
| 309 | generation: "", |
| 310 | alive: true, |
| 311 | ready: false, |
| 312 | eventSeq: 0, |
| 313 | expectExit: false, |
| 314 | exited: Promise.resolve(), |
| 315 | }; |
| 316 | session.client = new RpcClient( |
| 317 | { |
| 318 | write: (line) => { |
| 319 | if (!child.stdin || child.stdin.destroyed) throw new Error("desktop service stdin is closed"); |
| 320 | child.stdin.write(line); |
| 321 | }, |
| 322 | }, |
| 323 | { |
| 324 | onRequest: (method, params) => this.handlers.onRequest(method, params), |
| 325 | onNotification: (method, params) => this.onNotification(session, method, params), |
| 326 | onProtocolError: (kind, line) => log.warn(`service stdout ${kind}: ${line.slice(0, 200)}`), |
| 327 | }, |
| 328 | ); |
| 329 | session.exited = new Promise<void>((resolve) => { |
| 330 | const finish = (error: Error, code: number | null, signal: NodeJS.Signals | null) => { |
| 331 | if (!session.alive) return; |
| 332 | session.alive = false; |
| 333 | session.client.close(error); |
| 334 | resolve(); |
| 335 | this.onExit(session, code, signal); |
| 336 | }; |
| 337 | child.once("exit", (code, signal) => finish(new Error(`desktop service exited (${describeExit(code, signal)})`), code, signal)); |
| 338 | child.once("error", (error) => finish(new Error(`desktop service failed to start: ${errorText(error)}`), null, null)); |
| 339 | }); |
| 340 | child.stdout?.on("data", (chunk: Buffer) => { |
| 341 | try { |
| 342 | session.client.feed(chunk); |
| 343 | } catch (error) { |
| 344 | log.error(`service stream unusable: ${errorText(error)}`); |
| 345 | child.kill(); |
| 346 | } |
| 347 | }); |
| 348 | child.stderr?.on("data", (chunk: Buffer) => this.options.onStderr(chunk)); |
| 349 | child.stdin?.on("error", (error) => log.warn(`service stdin: ${errorText(error)}`)); |
| 350 | this.session = session; |
| 351 | return session; |
| 352 | } |
| 353 | |
| 354 | private onNotification(session: Session, method: string, params: unknown): void { |
| 355 | if (method !== "desktop/event") { |
| 356 | this.options.log.warn(`unknown service notification ${method}`); |
| 357 | return; |
| 358 | } |
| 359 | const frame = eventFrame(params); |
| 360 | if (!frame) { |
| 361 | this.options.log.warn("malformed desktop/event frame dropped"); |
| 362 | return; |
| 363 | } |
| 364 | if (this.session !== session || !session.alive || frame.generation !== session.generation) { |
| 365 | this.options.log.warn(`event ${frame.name} from dead generation ${frame.generation} dropped`); |
| 366 | return; |
| 367 | } |
| 368 | if (frame.seq <= session.eventSeq) return; |
| 369 | if (frame.seq > session.eventSeq + 1) this.options.log.warn(`desktop event gap: ${session.eventSeq} -> ${frame.seq}`); |
| 370 | session.eventSeq = frame.seq; |
| 371 | this.handlers.onEvent(frame); |
| 372 | } |
| 373 | |
| 374 | private onExit(session: Session, code: number | null, signal: NodeJS.Signals | null): void { |
| 375 | if (this.session !== session || !session.ready) return; |
| 376 | const reason = describeExit(code, signal); |
| 377 | if (session.expectExit || this.stopping) { |
| 378 | this.setState({ phase: "exited", generation: "" }); |
| 379 | return; |
| 380 | } |
| 381 | this.options.log.error(`desktop service exited unexpectedly (${reason})`); |
| 382 | if (this.budget.allow(this.now())) { |
| 383 | void this.begin(true).catch(() => undefined); |
| 384 | return; |
| 385 | } |
| 386 | const error = new Error(`desktop service exited (${reason}); automatic restarts exhausted`); |
| 387 | this.setState({ phase: "failed", generation: "", error: error.message }); |
| 388 | this.handlers.onFailed(error); |
| 389 | } |
| 390 | |
| 391 | private async terminate(session: Session): Promise<void> { |
| 392 | session.expectExit = true; |
| 393 | try { |
| 394 | session.child.stdin?.end(); |
| 395 | } catch { |
| 396 | // Already closed. |
| 397 | } |
| 398 | if (!session.alive) return; |
| 399 | await Promise.race([session.exited, delay(this.exitGraceMs)]); |
| 400 | if (!session.alive) return; |
| 401 | this.options.log.warn("desktop service did not exit after stdin close; killing it"); |
| 402 | session.child.kill("SIGKILL"); |
| 403 | await Promise.race([session.exited, delay(1000)]); |
| 404 | if (session.alive) throw new Error("desktop service did not terminate; shell exit withheld"); |
| 405 | } |
| 406 | |
| 407 | private setState(state: ServiceState): void { |
| 408 | this.state = state; |
| 409 | // A destroyed renderer or failing observer must not turn confirmed service |
| 410 | // exit into a failed shutdown, nor skip the shell's remaining cleanup. |
| 411 | try { |
| 412 | this.handlers.onState(state); |
| 413 | } catch (error) { |
| 414 | this.options.log.warn(`service state observer failed: ${errorText(error)}`); |
| 415 | } |
| 416 | } |
| 417 | } |
| 418 |