| 1 | // Trajectory recording and replay over the real server: files land in an |
| 2 | // isolated recordings dir; replay re-enters the normal tool pipeline and the |
| 3 | // recorder never records itself. |
| 4 | import { test, before, after } from "node:test"; |
| 5 | import assert from "node:assert/strict"; |
| 6 | import fs from "node:fs"; |
| 7 | import os from "node:os"; |
| 8 | import path from "node:path"; |
| 9 | import url from "node:url"; |
| 10 | import { spawn } from "node:child_process"; |
| 11 | |
| 12 | const ROOT = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), ".."); |
| 13 | const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "cu-traj-state-")); |
| 14 | const recDir = fs.mkdtempSync(path.join(os.tmpdir(), "cu-traj-rec-")); |
| 15 | let server; |
| 16 | let buf = ""; |
| 17 | const pending = new Map(); |
| 18 | let nextId = 1; |
| 19 | |
| 20 | function rpc(method, params) { |
| 21 | const id = nextId++; |
| 22 | return new Promise((resolve, reject) => { |
| 23 | const t = setTimeout(() => { pending.delete(id); reject(new Error(`timeout: ${method}`)); }, 20_000); |
| 24 | pending.set(id, (msg) => { clearTimeout(t); resolve(msg); }); |
| 25 | server.stdin.write(JSON.stringify({ jsonrpc: "2.0", id, method, params }) + "\n"); |
| 26 | }); |
| 27 | } |
| 28 | async function tool(name, args = {}) { |
| 29 | const res = await rpc("tools/call", { name, arguments: args }); |
| 30 | return JSON.parse(res.result.content[0].text); |
| 31 | } |
| 32 | |
| 33 | before(() => { |
| 34 | server = spawn("node", [path.join(ROOT, "mcp", "server.mjs")], { |
| 35 | env: { ...process.env, CODEWHALE_CU_STATE_DIR: stateDir, CODEWHALE_CU_RECORDINGS_DIR: recDir, CODEWHALE_CU_APP: "off" }, |
| 36 | stdio: ["pipe", "pipe", "pipe"], |
| 37 | }); |
| 38 | server.stdout.on("data", (c) => { |
| 39 | buf += c.toString(); |
| 40 | let i; |
| 41 | while ((i = buf.indexOf("\n")) !== -1) { |
| 42 | const line = buf.slice(0, i).trim(); |
| 43 | buf = buf.slice(i + 1); |
| 44 | if (!line) continue; |
| 45 | const msg = JSON.parse(line); |
| 46 | if (msg.id != null && pending.has(msg.id)) { pending.get(msg.id)(msg); pending.delete(msg.id); } |
| 47 | } |
| 48 | }); |
| 49 | }); |
| 50 | after(() => { try { server.stdin.end(); } catch {} server?.kill("SIGTERM"); fs.rmSync(stateDir, { recursive: true, force: true }); fs.rmSync(recDir, { recursive: true, force: true }); }); |
| 51 | |
| 52 | test("record → status → stop writes a local JSONL with turns and refusals", async () => { |
| 53 | const started = await tool("trajectory", { action: "start" }); |
| 54 | assert.equal(started.ok, true); |
| 55 | assert.equal(started.recording, true); |
| 56 | assert.ok(fs.existsSync(started.file)); |
| 57 | assert.equal((await tool("trajectory", { action: "status" })).recording, true); |
| 58 | await tool("wait", { seconds: 0.05 }); |
| 59 | await tool("computer", { action: "list" }); |
| 60 | const refused = await tool("click", {}); |
| 61 | assert.equal(refused.error?.code, "bad_args"); |
| 62 | const stopped = await tool("trajectory", { action: "stop" }); |
| 63 | assert.equal(stopped.recording, false); |
| 64 | assert.equal(stopped.turns, 3, `turns=${stopped.turns}`); |
| 65 | const lines = fs.readFileSync(stopped.file, "utf8").trim().split("\n").map(JSON.parse); |
| 66 | assert.equal(lines[0].type, "start"); |
| 67 | assert.equal(lines.at(-1).type, "stop"); |
| 68 | const calls = lines.filter((l) => l.type === "call"); |
| 69 | assert.deepEqual(calls.map((c) => c.tool), ["wait", "computer", "click"]); |
| 70 | assert.equal(calls[2].ok, false, "refusals are part of the record"); |
| 71 | assert.equal(calls[2].code, "bad_args"); |
| 72 | }); |
| 73 | |
| 74 | test("replay dry_run lists the plan without executing", async () => { |
| 75 | const started = await tool("trajectory", { action: "start" }); |
| 76 | assert.equal(started.recording, true); |
| 77 | await tool("wait", { seconds: 0.01 }); |
| 78 | const stopped = await tool("trajectory", { action: "stop" }); |
| 79 | const dry = await tool("trajectory", { action: "replay", id: path.basename(stopped.file), dry_run: true }); |
| 80 | assert.equal(dry.dry_run, true); |
| 81 | assert.equal(dry.replayed, 0); |
| 82 | assert.deepEqual(dry.plan, ["wait"]); |
| 83 | }); |
| 84 | |
| 85 | test("replay re-enters the pipeline, stops at the first refusal, and never records itself", async () => { |
| 86 | const started = await tool("trajectory", { action: "start" }); |
| 87 | assert.equal(started.recording, true); |
| 88 | await tool("wait", { seconds: 0.01 }); |
| 89 | await tool("click", {}); |
| 90 | await tool("wait", { seconds: 0.01 }); |
| 91 | const stopped = await tool("trajectory", { action: "stop" }); |
| 92 | const replay = await tool("trajectory", { action: "replay", id: path.basename(stopped.file) }); |
| 93 | assert.equal(replay.ok, true); |
| 94 | assert.equal(replay.turns_in_file, 3); |
| 95 | assert.equal(replay.replayed, 2, "stops at the refusal instead of continuing"); |
| 96 | assert.deepEqual(replay.results.map((r) => r.tool), ["wait", "click"]); |
| 97 | assert.equal(replay.results[1].ok, false); |
| 98 | const calls = fs.readFileSync(stopped.file, "utf8").trim().split("\n").map(JSON.parse).filter((l) => l.type === "call"); |
| 99 | assert.equal(calls.length, 3, "replayed calls are not re-recorded"); |
| 100 | assert.equal((await tool("trajectory", { action: "status" })).recording, false); |
| 101 | }); |
| 102 | |
| 103 | test("replay refuses escaping ids; the kill switch gates replay but not status", async () => { |
| 104 | const bad = await tool("trajectory", { action: "replay", id: "../escape.jsonl" }); |
| 105 | assert.equal(bad.error?.code, "bad_args"); |
| 106 | const missing = await tool("trajectory", { action: "replay", id: "traj-nope.jsonl" }); |
| 107 | assert.equal(missing.error?.code, "trajectory_not_found"); |
| 108 | await tool("stop_computer_control", { reason: "trajectory test" }); |
| 109 | const afterStop = await tool("trajectory", { action: "replay", dry_run: true }); |
| 110 | assert.equal(afterStop.error?.code, "control_stopped", "replay is an action, not a read"); |
| 111 | const status = await tool("trajectory", { action: "status" }); |
| 112 | assert.equal(status.ok, true, "status stays readable after the stop, like other read-only probes"); |
| 113 | }); |
| 114 |