| 1 | import assert from "node:assert/strict"; |
| 2 | import type { ChildProcess } from "node:child_process"; |
| 3 | import { EventEmitter } from "node:events"; |
| 4 | import { PassThrough } from "node:stream"; |
| 5 | import { test } from "node:test"; |
| 6 | import type { ServiceState } from "../shared/ipc.js"; |
| 7 | import { validateHelloResult, type HelloResult } from "./handshake.js"; |
| 8 | import { RestartBudget } from "./restartBudget.js"; |
| 9 | import { ServiceSupervisor } from "./service.js"; |
| 10 | |
| 11 | const silent = { info() {}, warn() {}, error() {} }; |
| 12 | const tick = async (times = 4) => { |
| 13 | for (let i = 0; i < times; i++) await new Promise((resolve) => setImmediate(resolve)); |
| 14 | }; |
| 15 | |
| 16 | class FakeChild extends EventEmitter { |
| 17 | stdin = new PassThrough(); |
| 18 | stdout = new PassThrough(); |
| 19 | stderr = new PassThrough(); |
| 20 | alive = true; |
| 21 | requests: Array<{ id: number; method: string; params: unknown }> = []; |
| 22 | private buffered = ""; |
| 23 | |
| 24 | constructor(readonly generation: string, readonly behaviour: { helloError?: { code: number; message: string }; exitOnStdinEnd?: boolean; |
| 25 | shutdownResults?: Array<Record<string, unknown>>; },) { |
| 26 | super(); |
| 27 | this.stdin.on("data", (chunk: Buffer) => { |
| 28 | this.buffered += chunk.toString("utf8"); |
| 29 | let index: number; |
| 30 | while ((index = this.buffered.indexOf("\n")) >= 0) { |
| 31 | const line = this.buffered.slice(0, index); |
| 32 | this.buffered = this.buffered.slice(index + 1); |
| 33 | this.handle(JSON.parse(line) as { id?: number; method?: string; params?: unknown; },); |
| 34 | } |
| 35 | }); |
| 36 | this.stdin.on("end", () => { |
| 37 | if (this.behaviour.exitOnStdinEnd !== false) this.exit(0, null); |
| 38 | }); |
| 39 | } |
| 40 | |
| 41 | kill(): boolean { |
| 42 | this.exit(null, "SIGKILL"); |
| 43 | return true; |
| 44 | } |
| 45 | |
| 46 | exit(code: number | null, signal: NodeJS.Signals | null): void { |
| 47 | if (!this.alive) return; |
| 48 | this.alive = false; |
| 49 | this.emit("exit", code, signal); |
| 50 | } |
| 51 | |
| 52 | send(frame: Record<string, unknown>): void { |
| 53 | this.stdout.write(JSON.stringify({ jsonrpc: "2.0", ...frame }) + "\n"); |
| 54 | } |
| 55 | |
| 56 | event(name: string, generation = this.generation): void { |
| 57 | this.send({ method: "desktop/event", params: { seq: 1, generation, name, args: [{ ok: true }] }, }); |
| 58 | } |
| 59 | |
| 60 | private handle(frame: { id?: number; method?: string; params?: unknown }): void { |
| 61 | if (typeof frame.id !== "number" || typeof frame.method !== "string") return; |
| 62 | this.requests.push({ id: frame.id, method: frame.method, params: frame.params, }); |
| 63 | if (frame.method === "desktop/hello") { |
| 64 | if (this.behaviour.helloError) { |
| 65 | this.send({ id: frame.id, error: this.behaviour.helloError }); |
| 66 | return; |
| 67 | } |
| 68 | this.send({ |
| 69 | id: frame.id, |
| 70 | result: { |
| 71 | protocolVersion: 11, |
| 72 | contractDigest: "sha256:abc", |
| 73 | service: { version: "dev", channel: "dev", commit: "dev", pid: 1 }, |
| 74 | runtimeGeneration: this.generation, |
| 75 | runId: `run-${this.generation}`, |
| 76 | incidentId: `incident-${this.generation}`, |
| 77 | diagnosticsEnabled: true, |
| 78 | resources: { origin: "http://127.0.0.1:1", token: "t" }, |
| 79 | window: { width: 1000, height: 700, minWidth: 760, minHeight: 480, frameless: false, zoomFactor: 1, }, |
| 80 | }, |
| 81 | }); |
| 82 | return; |
| 83 | } |
| 84 | if (frame.method === "desktop/invoke") { |
| 85 | const params = frame.params as { method: string; args: unknown[] }; |
| 86 | if (params.method === "Fail") this.send({ id: frame.id, error: { code: -32000, message: "workspace not found", data: { method: "Fail" }, }, }); |
| 87 | else this.send({ id: frame.id, result: { method: params.method, args: params.args }, }); |
| 88 | return; |
| 89 | } |
| 90 | if (frame.method === "desktop/shutdown" || frame.method === "desktop/shutdownStatus") { |
| 91 | const params = frame.params as { requestId?: string; reason?: string }; |
| 92 | const configured = |
| 93 | this.behaviour.shutdownResults?.shift(); |
| 94 | this.send({ id: frame.id, result: { |
| 95 | requestId: params.requestId ?? "", |
| 96 | reason: params.reason ?? "user_quit", |
| 97 | phase: "completed", |
| 98 | outcome: "success", |
| 99 | completed: true, |
| 100 | retryable: false, |
| 101 | updatedAt: new Date().toISOString(), |
| 102 | ...configured,}, }); |
| 103 | return; |
| 104 | } |
| 105 | this.send({ id: frame.id, result: { |
| 106 | } }); |
| 107 | } |
| 108 | } |
| 109 | |
| 110 | function harness(options: { children?: FakeChild[]; budget?: RestartBudget; onState?(state: ServiceState): void; } = {},) { |
| 111 | const spawned: FakeChild[] = []; |
| 112 | const states: ServiceState[] = []; |
| 113 | const events: string[] = []; |
| 114 | const ready: Array<{ generation: string; restarted: boolean }> = []; |
| 115 | const failures: string[] = []; |
| 116 | let index = 0; |
| 117 | const supervisor = new ServiceSupervisor( |
| 118 | { |
| 119 | binary: "fake", |
| 120 | args: ["--host-rpc"], |
| 121 | env: {}, |
| 122 | onStderr: () => undefined, |
| 123 | log: silent, |
| 124 | budget: options.budget ?? new RestartBudget(), |
| 125 | exitGraceMs: 10, |
| 126 | spawn: () => { |
| 127 | const child = options.children?.[index++] ?? new FakeChild(`g-${spawned.length + 1}`, {}); |
| 128 | spawned.push(child); |
| 129 | return child as unknown as ChildProcess; |
| 130 | }, |
| 131 | }, |
| 132 | { |
| 133 | hello: async (client) => validateHelloResult(await client.request("desktop/hello", {}, 1000)), |
| 134 | onRequest: async () => ({}), |
| 135 | onEvent: (frame) => events.push(`${frame.generation}:${frame.name}`), |
| 136 | onState: (state) => { states.push(state); options.onState?.(state); }, |
| 137 | onReady: (hello: HelloResult, restarted) => { ready.push({ generation: hello.runtimeGeneration, restarted }); }, |
| 138 | onFailed: (error) => failures.push(error instanceof Error ? error.message : String(error)), |
| 139 | }, |
| 140 | ); |
| 141 | return { supervisor, spawned, states, events, ready, failures }; |
| 142 | } |
| 143 | |
| 144 | test("start runs hello then desktop/start and exposes the generation", async () => { |
| 145 | const h = harness(); |
| 146 | const hello = await h.supervisor.start(); |
| 147 | assert.equal(hello.runtimeGeneration, "g-1"); |
| 148 | assert.deepEqual(h.spawned[0]?.requests.map((r) => r.method), ["desktop/hello", "desktop/start"],); |
| 149 | assert.equal(h.supervisor.ready, true); |
| 150 | assert.equal(h.supervisor.generation, "g-1"); |
| 151 | assert.deepEqual(h.states.map((s) => s.phase), ["starting", "ready"],); |
| 152 | assert.deepEqual(h.ready, [{ generation: "g-1", restarted: false }]); |
| 153 | assert.deepEqual(await h.supervisor.invoke("OpenProjectTab", ["/p", true]), { method: "OpenProjectTab", args: ["/p", true], }); |
| 154 | await assert.rejects(h.supervisor.invoke("Fail", []), /workspace not found/); |
| 155 | }); |
| 156 | |
| 157 | test("events from the live generation are forwarded and stale ones dropped", async () => { |
| 158 | const h = harness(); |
| 159 | await h.supervisor.start(); |
| 160 | h.spawned[0]?.event("agent:event"); |
| 161 | h.spawned[0]?.event("agent:event", "g-old"); |
| 162 | h.spawned[0]?.event("duplicate"); |
| 163 | h.spawned[0]?.send({ method: "desktop/event", params: { seq: 3, generation: "g-1", name: "after-gap", args: [] }, }); |
| 164 | h.spawned[0]?.send({ method: "desktop/event", params: { seq: 2, generation: "g-1", name: "late", args: [] }, }); |
| 165 | await tick(); |
| 166 | assert.deepEqual(h.events, ["g-1:agent:event", "g-1:after-gap"]); |
| 167 | }); |
| 168 | |
| 169 | test("a handshake error fails the service without a restart and terminates the process", async () => { |
| 170 | const child = new FakeChild("g-1", { helloError: { code: -32003, message: "digest differs" }, }); |
| 171 | const h = harness({ children: [child] }); |
| 172 | await assert.rejects(h.supervisor.start(), /digest differs/); |
| 173 | await tick(); |
| 174 | assert.equal(h.supervisor.current.phase, "failed"); |
| 175 | assert.deepEqual(h.failures, ["digest differs"]); |
| 176 | assert.equal(child.alive, false, "stdin close makes the fake exit"); |
| 177 | assert.equal(h.spawned.length, 1); |
| 178 | }); |
| 179 | |
| 180 | test("an unexpected exit restarts automatically until the budget is exhausted", async () => { |
| 181 | const budget = new RestartBudget(2, 60_000); |
| 182 | const h = harness({ budget }); |
| 183 | await h.supervisor.start(); |
| 184 | h.spawned[0]?.exit(1, null); |
| 185 | await tick(8); |
| 186 | assert.equal(h.spawned.length, 2); |
| 187 | assert.equal(h.supervisor.generation, "g-2"); |
| 188 | assert.deepEqual(h.ready.map((r) => r.restarted), [false, true],); |
| 189 | h.spawned[1]?.exit(1, null); |
| 190 | await tick(8); |
| 191 | assert.equal(h.spawned.length, 3); |
| 192 | h.spawned[2]?.exit(1, null); |
| 193 | await tick(8); |
| 194 | assert.equal(h.spawned.length, 3, "no fourth spawn once the budget is spent"); |
| 195 | assert.equal(h.supervisor.current.phase, "failed"); |
| 196 | assert.match(h.supervisor.current.error ?? "", /automatic restarts exhausted/); |
| 197 | await assert.rejects(h.supervisor.invoke("X", []), /not running/); |
| 198 | await h.supervisor.restart(); |
| 199 | assert.equal(h.spawned.length, 4, "a manual restart is always allowed"); |
| 200 | assert.equal(h.supervisor.generation, "g-4"); |
| 201 | assert.deepEqual(h.states.map((s) => s.phase), ["starting", "ready", "restarting", "ready", "restarting", "ready", "failed", "restarting", "ready"],); |
| 202 | }); |
| 203 | |
| 204 | test("shutdown sends desktop/shutdown, closes stdin and waits for the exit", async () => { |
| 205 | const h = harness(); |
| 206 | await h.supervisor.start(); |
| 207 | const shutdown = h.supervisor.shutdown(); |
| 208 | assert.equal(h.supervisor.current.phase, "stopping"); |
| 209 | assert.equal(h.supervisor.ready, false); |
| 210 | await assert.rejects(h.supervisor.invoke("CloseMainWindow", []), /shutting down/); |
| 211 | await shutdown; |
| 212 | const child = h.spawned[0] as FakeChild; |
| 213 | assert.deepEqual(child.requests.map((r) => r.method), ["desktop/hello", "desktop/start", "desktop/shutdown"],); |
| 214 | assert.equal(child.alive, false); |
| 215 | assert.equal(h.supervisor.current.phase, "exited"); |
| 216 | assert.equal(h.spawned.length, 1, "a deliberate exit never restarts"); |
| 217 | assert.deepEqual(h.states.map((state) => state.phase), ["starting", "ready", "stopping", "exited", "exited"]); |
| 218 | }); |
| 219 | |
| 220 | test("a service that ignores stdin close is killed after the grace period", async () => { |
| 221 | const child = new FakeChild("g-1", { exitOnStdinEnd: false }); |
| 222 | const h = harness({ children: [child] }); |
| 223 | await h.supervisor.start(); |
| 224 | await h.supervisor.shutdown(); |
| 225 | assert.equal(child.alive, false); |
| 226 | assert.equal(h.supervisor.current.phase, "exited"); |
| 227 | }); |
| 228 | |
| 229 | test("a retryable save failure keeps the service alive and retry uses the same request identity", async () => { |
| 230 | const child = new FakeChild("g-1", { |
| 231 | shutdownResults: [ |
| 232 | { |
| 233 | phase: "saving", |
| 234 | outcome: "failed", |
| 235 | completed: false, |
| 236 | retryable: true, |
| 237 | errorCode: "session_save_failed", |
| 238 | error: "disk full", |
| 239 | }, |
| 240 | { |
| 241 | phase: "completed", |
| 242 | outcome: "success", |
| 243 | completed: true, |
| 244 | retryable: false, |
| 245 | }, |
| 246 | ], |
| 247 | }); |
| 248 | const h = harness({ children: [child] }); |
| 249 | await h.supervisor.start(); |
| 250 | await assert.rejects(h.supervisor.shutdown(), /session_save_failed/); |
| 251 | assert.equal(child.alive, true, "failed durable save must not close stdin or kill the service"); |
| 252 | await h.supervisor.shutdown(); |
| 253 | const shutdowns = child.requests.filter((request) => request.method === "desktop/shutdown"); |
| 254 | assert.equal(shutdowns.length, 2); |
| 255 | assert.equal((shutdowns[0].params as { requestId: string }).requestId, (shutdowns[1].params as { requestId: string }).requestId); |
| 256 | assert.equal(child.alive, false); |
| 257 | }); |
| 258 | |
| 259 | test("shutdown status polling waits through in-progress cleanup", async () => { |
| 260 | const child = new FakeChild("g-1", { |
| 261 | shutdownResults: [ |
| 262 | { |
| 263 | phase: "saving", |
| 264 | outcome: "in_progress", |
| 265 | completed: false, |
| 266 | retryable: false, |
| 267 | }, |
| 268 | { |
| 269 | phase: "closing", |
| 270 | outcome: "in_progress", |
| 271 | completed: false, |
| 272 | retryable: false, |
| 273 | }, |
| 274 | { |
| 275 | phase: "completed", |
| 276 | outcome: "success", |
| 277 | completed: true, |
| 278 | retryable: false, |
| 279 | }, |
| 280 | ], |
| 281 | }); |
| 282 | const h = harness({ children: [child] }); |
| 283 | const phases: string[] = []; |
| 284 | await h.supervisor.start(); |
| 285 | await h.supervisor.shutdown("user_quit", (phase) => phases.push(phase)); |
| 286 | assert.deepEqual(phases, ["saving", "closing", "completed"]); |
| 287 | assert.deepEqual( |
| 288 | child.requests.slice(2).map((request) => request.method), |
| 289 | ["desktop/shutdown", "desktop/shutdownStatus", "desktop/shutdownStatus"], |
| 290 | ); |
| 291 | }); |
| 292 | |
| 293 | test("shutdown fences a hello completion queued before shutdown", async () => { |
| 294 | const h = harness(); |
| 295 | const starting = h.supervisor.start(); |
| 296 | const rejected = assert.rejects(starting, /cancelled|exited/); |
| 297 | await h.supervisor.shutdown(); |
| 298 | await rejected; |
| 299 | assert.deepEqual(h.ready, []); |
| 300 | assert.deepEqual(h.failures, []); |
| 301 | assert.equal(h.spawned.length, 1); |
| 302 | await assert.rejects(h.supervisor.start(), /shutting down/); |
| 303 | }); |
| 304 | |
| 305 | test("a destroyed renderer throwing during exit notification cannot prevent shutdown", async () => { |
| 306 | const h = harness({ onState: (state) => { if (state.phase === "exited") throw new Error("Object has been destroyed"); }, }); |
| 307 | await h.supervisor.start(); |
| 308 | await h.supervisor.shutdown(); |
| 309 | assert.equal(h.supervisor.current.phase, "exited"); |
| 310 | assert.equal(h.spawned[0].alive, false); |
| 311 | }); |
| 312 | |
| 313 | test("concurrent restarts share one replacement and shutdown prevents its spawn", async () => { |
| 314 | const h = harness(); |
| 315 | await h.supervisor.start(); |
| 316 | const a = h.supervisor.restart(); |
| 317 | const b = h.supervisor.restart(); |
| 318 | const rejected = Promise.all([assert.rejects(a, /shutting down/), assert.rejects(b, /shutting down/)]); |
| 319 | await h.supervisor.shutdown(); |
| 320 | await rejected; |
| 321 | assert.equal(h.spawned.length, 1); |
| 322 | }); |
| 323 |