| 1 | // One request handler for every out-of-process runner of the backends: the |
| 2 | // ssh remote agent (one request per process) and the desktop app daemon (one |
| 3 | // long-lived process). Only tools in ALLOWED execute, so neither the ssh |
| 4 | // transport nor the app socket can ever become a generic shell. |
| 5 | import url from "node:url"; |
| 6 | import { exec } from "./remote-runtime.mjs"; |
| 7 | import { withSignal, throwIfAborted } from "./exec.mjs"; |
| 8 | |
| 9 | export const ALLOWED = new Set([ |
| 10 | "preview", "platform", "probe", "list_displays", "switch_display", "list_apps", "list_sessions", "list_windows", |
| 11 | "open_application", "kill_app", "set_window_frame", "get_app_state", "resolve_element", "screenshot", "zoom", |
| 12 | "browser_start", "browser_status", "browser_navigate", "browser_click", "browser_type", "browser_screenshot", "browser_stop", |
| 13 | "left_click", "double_click", "triple_click", "right_click", "middle_click", |
| 14 | "mouse_move", "left_click_drag", "left_mouse_down", "left_mouse_up", "scroll", |
| 15 | "type", "key", "hold_key", "set_value", "focus", "get_value", "select_text", "perform_action", "invoke_menu", |
| 16 | "read_clipboard", "write_clipboard", "cursor_position", |
| 17 | "recordingStart", "recordingStop", "recordingStatus", "recordingList", |
| 18 | "app_script", |
| 19 | ]); |
| 20 | |
| 21 | /** app_script runs osascript where this handler executes. The remote agent is |
| 22 | * the exception that proves the transport rule: it must never become a shell, |
| 23 | * so scripting is honored on the local computer (computerId "local") only. */ |
| 24 | const LOCAL_ONLY_TOOLS = new Set(["app_script"]); |
| 25 | |
| 26 | const backends = new Map(); |
| 27 | const heldPointers = new Map(); |
| 28 | const INPUT_MUTATIONS = new Set([ |
| 29 | "open_application", "left_click", "double_click", "triple_click", "right_click", "middle_click", "mouse_move", |
| 30 | "left_click_drag", "left_mouse_down", "left_mouse_up", "scroll", "type", "key", "hold_key", "set_value", "focus", "select_text", "perform_action", "invoke_menu", |
| 31 | ]); |
| 32 | let queue = Promise.resolve(); |
| 33 | |
| 34 | async function backend(computerId, sessionId, persistentInputOwner) { |
| 35 | const key = `${computerId}:${sessionId}`; |
| 36 | if (!backends.has(key)) { |
| 37 | // Same test hook as src/transport.mjs, so the out-of-process route can be |
| 38 | // driven end to end against a recording backend (never set in production). |
| 39 | const test = process.env.CODEWHALE_CU_TEST_BACKEND; |
| 40 | const mod = await import(test ? url.pathToFileURL(test).href : `./backends/${process.platform}.mjs`); |
| 41 | backends.set(key, mod.create({ exec: { ...exec, persistentInputOwner }, computer: { id: computerId, transport: "local", platform: process.platform } })); |
| 42 | } |
| 43 | return backends.get(key); |
| 44 | } |
| 45 | |
| 46 | const sessions = new Map(); |
| 47 | let controlMode = "ready"; |
| 48 | let controlGeneration = 0; |
| 49 | let cleanupPending = false; |
| 50 | |
| 51 | /** Human-facing state contains app identity and action names, never task text. */ |
| 52 | export function controlStatus() { |
| 53 | return { mode: controlMode, cleanupPending, sessions: [...sessions.values()] |
| 54 | .filter((s) => !s.closed && s.target) |
| 55 | .map((s) => ({ target: s.target, mode: s.mode, action: s.action ?? null })) }; |
| 56 | } |
| 57 | |
| 58 | // Called only by the launcher's inherited control channel, never an MCP tool. |
| 59 | // Abort before queuing cleanup so even a held gesture yields to the person. |
| 60 | export async function setControlMode(mode) { |
| 61 | if (!["ready", "paused", "stopped"].includes(mode)) throw new Error("Unknown control mode"); |
| 62 | if (mode === "ready") { |
| 63 | if (cleanupPending) throw new Error("Input is still being released; try again in a moment."); |
| 64 | controlMode = mode; |
| 65 | return controlStatus(); |
| 66 | } |
| 67 | controlMode = mode; |
| 68 | const generation = ++controlGeneration; |
| 69 | cleanupPending = true; |
| 70 | const results = await Promise.allSettled([...sessions.keys()].map((key) => { |
| 71 | const colon = key.indexOf(":"); |
| 72 | return releaseSessionInput(key.slice(colon + 1), key.slice(0, colon), { close: mode === "stopped" }); |
| 73 | })); |
| 74 | const failure = results.find((r) => r.status === "rejected"); |
| 75 | // A cleanup failure stays blocked. Resume cannot hide owned input. |
| 76 | if (failure) throw failure.reason; |
| 77 | if (generation === controlGeneration) cleanupPending = false; |
| 78 | return controlStatus(); |
| 79 | } |
| 80 | |
| 81 | function enqueue(fn) { |
| 82 | const next = queue.then(fn); |
| 83 | queue = next.catch(() => {}); |
| 84 | return next; |
| 85 | } |
| 86 | |
| 87 | /** Cancel this host's work, then release only input held by its backend. */ |
| 88 | export function releaseSessionInput(sessionId, computerId = "local", { close = false } = {}) { |
| 89 | const key = `${computerId}:${sessionId}`; |
| 90 | let session = sessions.get(key); |
| 91 | if (!session) { |
| 92 | if (!close) return Promise.resolve(); |
| 93 | session = { requests: new Set(), closed: true, touched: Date.now() }; |
| 94 | sessions.set(key, session); |
| 95 | } |
| 96 | if (close) session.closed = true; |
| 97 | for (const controller of session.requests) controller.abort(); |
| 98 | return enqueue(async () => { |
| 99 | try { |
| 100 | await withSignal(null, () => backends.get(key)?.releaseInput?.()); |
| 101 | if (heldPointers.get(computerId) === key) heldPointers.delete(computerId); |
| 102 | if (close) await withSignal(null, () => backends.get(key)?.closeSession?.()); |
| 103 | if (close) backends.delete(key); |
| 104 | } finally { session.touched = Date.now(); } |
| 105 | }); |
| 106 | } |
| 107 | |
| 108 | export function closeSession(sessionId, computerId = "local") { |
| 109 | return releaseSessionInput(sessionId, computerId, { close: true }); |
| 110 | } |
| 111 | |
| 112 | /** |
| 113 | * A freshly granted session owner supersedes a closed-session tombstone: |
| 114 | * without this, a daemon that re-leases a session id (the old owner socket |
| 115 | * died with the previous daemon) would keep aborting every request on it. |
| 116 | * In-flight requests from the dead owner stay aborted; the tombstone is the |
| 117 | * only thing removed. |
| 118 | */ |
| 119 | export function reopenSession(sessionId, computerId = "local") { |
| 120 | sessions.delete(`${computerId}:${sessionId}`); |
| 121 | } |
| 122 | |
| 123 | export function closeAllSessions() { |
| 124 | return Promise.allSettled([...sessions.keys()].map((key) => { |
| 125 | const colon = key.indexOf(":"); |
| 126 | return closeSession(key.slice(colon + 1), key.slice(0, colon)); |
| 127 | })); |
| 128 | } |
| 129 | |
| 130 | /** |
| 131 | * Content-free session registry view for agents: which sessions are live, |
| 132 | * what each is bound to, what it is doing right now, and whether any session |
| 133 | * holds a pointer. App identity and action names only — task text never |
| 134 | * reaches this process, so none can leak. Read-only by construction. |
| 135 | */ |
| 136 | export function summarizeSessions() { |
| 137 | const live = [...sessions.entries()].filter(([, s]) => !s.closed); |
| 138 | return { |
| 139 | control: controlMode, |
| 140 | count: live.length, |
| 141 | sessions: live.map(([key, s]) => { |
| 142 | const computerId = key.slice(0, key.indexOf(":")); |
| 143 | return { |
| 144 | target: s.target ?? null, |
| 145 | mode: s.mode ?? null, |
| 146 | action: s.action ?? null, |
| 147 | ageSec: Math.max(0, Math.round((Date.now() - s.touched) / 1000)), |
| 148 | inputHeld: heldPointers.get(computerId) === key, |
| 149 | }; |
| 150 | }), |
| 151 | }; |
| 152 | } |
| 153 | |
| 154 | /** |
| 155 | * Execute one {tool, args} request on this machine's backend. Never throws: |
| 156 | * every outcome is a receipt object with `ok`. |
| 157 | */ |
| 158 | export async function handle(req, { computerId = "local", sessionId = "direct", signal, persistentInputOwner = false } = {}) { |
| 159 | const tool = req?.tool; |
| 160 | if (!ALLOWED.has(tool)) { |
| 161 | return { ok: false, error: { code: "tool_not_allowed", message: `tool "${tool}" is not in the remote allow-list` } }; |
| 162 | } |
| 163 | if (LOCAL_ONLY_TOOLS.has(tool) && computerId !== "local") { |
| 164 | return { ok: false, error: { code: "unsupported_on_transport", message: `"${tool}" runs on the local computer only — a remote agent stays a computer-use channel, never a shell` } }; |
| 165 | } |
| 166 | if (tool === "platform") return { ok: true, platform: process.platform }; |
| 167 | if (controlMode !== "ready") return { ok: false, error: { code: `control_${controlMode}`, message: `Computer Use is ${controlMode} by the user. Wait for them to resume it in the menu bar.` } }; |
| 168 | const generation = controlGeneration; |
| 169 | const key = `${computerId}:${sessionId}`; |
| 170 | let session = sessions.get(key); |
| 171 | if (!session) { |
| 172 | // Retain closed-session tombstones briefly, and bound abandoned sessions |
| 173 | // when a host is killed without a graceful MCP disconnect. |
| 174 | for (const [id, old] of sessions) { |
| 175 | if (old.requests.size || Date.now() - old.touched <= (old.closed ? 300_000 : 3_600_000)) continue; |
| 176 | if (backends.has(id)) { |
| 177 | const colon = id.indexOf(":"); |
| 178 | try { await closeSession(id.slice(colon + 1), id.slice(0, colon)); } |
| 179 | catch (err) { return { ok: false, error: { code: "input_release_failed", message: String(err?.message ?? err) } }; } |
| 180 | } else sessions.delete(id); |
| 181 | } |
| 182 | if ([...sessions.values()].filter((entry) => !entry.closed).length >= 256) return { ok: false, error: { code: "session_limit", message: "Too many active computer sessions; close unused hosts or restart the helper." } }; |
| 183 | session = { requests: new Set(), closed: false, touched: Date.now() }; |
| 184 | sessions.set(key, session); |
| 185 | } |
| 186 | const controller = new AbortController(); |
| 187 | const abort = () => controller.abort(); |
| 188 | signal?.addEventListener("abort", abort, { once: true }); |
| 189 | if (signal?.aborted || session.closed) controller.abort(); |
| 190 | session.requests.add(controller); |
| 191 | // One desktop can execute only one input gesture at a time. The queue spans |
| 192 | // sockets and sessions; a disconnected/cancelled request is checked again |
| 193 | // when its slot arrives, before it can post any input. |
| 194 | try { |
| 195 | return await enqueue(() => withSignal(controller.signal, async () => { |
| 196 | throwIfAborted(); |
| 197 | if (generation !== controlGeneration || controlMode !== "ready") throw Object.assign(new Error("Computer control was interrupted by the user."), { code: "cancelled" }); |
| 198 | const instance = await backend(computerId, sessionId, persistentInputOwner); |
| 199 | throwIfAborted(); |
| 200 | session.action = tool; |
| 201 | if (tool === "open_application") { session.target = null; session.mode = null; } |
| 202 | if (INPUT_MUTATIONS.has(tool) && heldPointers.has(computerId) && heldPointers.get(computerId) !== key) { |
| 203 | return { ok: false, error: { code: "input_busy", message: "Another computer session owns a held pointer; release it or close that session before sending input." } }; |
| 204 | } |
| 205 | const fn = instance[tool]; |
| 206 | if (typeof fn !== "function") { |
| 207 | return { ok: false, error: { code: "unsupported_on_platform", message: `"${tool}" is not implemented on ${process.platform}` } }; |
| 208 | } |
| 209 | // Preserve ownership across calls, not just during the serialized |
| 210 | // request. Another host's click/up must not release this host's press. |
| 211 | if (tool === "left_mouse_down") heldPointers.set(computerId, key); |
| 212 | let data; |
| 213 | try { data = await fn(req.args ?? {}); throwIfAborted(); } |
| 214 | catch (error) { |
| 215 | if (["left_mouse_down", "left_mouse_up", "mouse_move"].includes(tool) && heldPointers.get(computerId) === key) { |
| 216 | await withSignal(null, () => instance.releaseInput?.()); |
| 217 | if (heldPointers.get(computerId) === key) heldPointers.delete(computerId); |
| 218 | } |
| 219 | throw error; |
| 220 | } |
| 221 | if (tool === "left_mouse_up" && heldPointers.get(computerId) === key) heldPointers.delete(computerId); |
| 222 | if (tool === "open_application" && data?.resolved) { |
| 223 | session.target = { name: String(data.resolved.name ?? "Application").slice(0, 128), pid: data.resolved.pid }; |
| 224 | session.mode = data.shared_pointer || data.activate ? "foreground" : "background"; |
| 225 | } |
| 226 | return { ok: true, platform: process.platform, tool, data }; |
| 227 | })); |
| 228 | } catch (err) { |
| 229 | return { ok: false, platform: process.platform, tool, error: { code: err?.code ?? "tool_error", message: String(err?.message ?? err) } }; |
| 230 | } finally { |
| 231 | signal?.removeEventListener("abort", abort); |
| 232 | session.requests.delete(controller); |
| 233 | session.action = null; |
| 234 | session.touched = Date.now(); |
| 235 | } |
| 236 | } |
| 237 |