| 1 | #!/usr/bin/env node |
| 2 | // codewhale-cu MCP server — zero-dependency JSON-RPC 2.0 over stdio. |
| 3 | // One tool surface, four platforms (darwin, win32, linux, harmonyos), with |
| 4 | // computer switching as a default: every tool accepts `computer`, and using a |
| 5 | // computer id switches the sticky active computer. |
| 6 | import fs from "node:fs"; |
| 7 | import path from "node:path"; |
| 8 | import crypto from "node:crypto"; |
| 9 | import * as registry from "../src/registry.mjs"; |
| 10 | import * as consent from "../src/consent.mjs"; |
| 11 | import { backendFor, installRemoteAgent, executorFor, closeAppSession, routeFingerprint, closeSshChannel, SESSION_ID } from "../src/transport.mjs"; |
| 12 | import { spawnDockerComputer, destroyDockerComputer, destroySessionSpawns } from "../src/spawn.mjs"; |
| 13 | import { TOOLS, TOOL_NAMES, REQUIRED_ARGS, ELEMENT_ONLY_TARGET, READ_ONLY_TOOLS, REMOTE_TOOLS, BACKEND_METHOD, resolveTool, parseGrant, MERGED_EXPANSION } from "../src/tools.mjs"; |
| 14 | import { tryJson, withSignal, throwIfAborted, wait } from "../src/exec.mjs"; |
| 15 | import { APP_VERSION } from "../src/app-socket.mjs"; |
| 16 | import { createRecorder, readTrajectory, listTrajectories, resolveTrajectory, isTrajectoryTool } from "../src/trajectory.mjs"; |
| 17 | |
| 18 | const SERVER_NAME = "codewhale-cu"; |
| 19 | |
| 20 | // ---------- per-session runtime state ---------- |
| 21 | let controlStopped = false; |
| 22 | // Registered computers are shared; the selected destination belongs to this |
| 23 | // MCP host. Another task must never redirect an implicit input action. |
| 24 | let activeComputerId = "local"; |
| 25 | let stateCounter = 0; |
| 26 | let inFlight = 0; // actions currently dispatching to a backend/executor |
| 27 | /** request ids cancelled via notifications/cancelled */ |
| 28 | const cancelled = new Set(); |
| 29 | const requests = new Map(); |
| 30 | let dispatch = Promise.resolve(); |
| 31 | const recorder = createRecorder(); |
| 32 | let replaying = false; |
| 33 | // Fixed at process start; nothing can widen it. See parseGrant for the form. |
| 34 | const GRANT = parseGrant(process.env.CODEWHALE_CU_GRANT); |
| 35 | /** The active capability grant, as `request_access` reports it on success or refusal. */ |
| 36 | const grantReport = () => (GRANT ? { mode: "narrowed", tools: [...GRANT].sort(), count: GRANT.size, note: "This session's tools were narrowed at launch (CODEWHALE_CU_GRANT); do not work around it." } : null); |
| 37 | /** state_id -> { computerId, app_ref, windowIndex, elements } */ |
| 38 | const appStates = new Map(); |
| 39 | /** computerId -> state_id of its most recent observation */ |
| 40 | const latestStateByComputer = new Map(); |
| 41 | /** computerId -> last raster metadata {file, scale, origin} */ |
| 42 | const lastRasters = new Map(); |
| 43 | /** computerId -> app_ref the computer's input is bound to (set by open_application) */ |
| 44 | const boundApps = new Map(); |
| 45 | /** computerId -> route-bound session resources; the registry owns configuration. */ |
| 46 | const backendCache = new Map(); |
| 47 | const ROUTE_INSPECTION_TOOLS = new Set([ |
| 48 | "request_access", "list_displays", "list_apps", "list_windows", "get_app_state", "screenshot", |
| 49 | "cursor_position", "read_clipboard", "recording_list", "recording_status", |
| 50 | "find_elements", "get_value", "wait_for", |
| 51 | // A script does not act through the observation state this gate protects, |
| 52 | // so it must not be held up waiting for a screenshot it never reads. |
| 53 | "app_script", |
| 54 | ]); |
| 55 | const STATE_CHAR_BUDGET = Number(process.env.CODEWHALE_CU_MAX_STATE_CHARS) > 0 |
| 56 | ? Number(process.env.CODEWHALE_CU_MAX_STATE_CHARS) |
| 57 | : 16_000; |
| 58 | /** |
| 59 | * Largest base64 image payload we will put in one JSON-RPC message. Hosts cap |
| 60 | * how much a stdio server may write between message boundaries (Claude Code |
| 61 | * disconnects at 16MB) and model APIs cap image bytes well below that, so a |
| 62 | * full-screen 5K PNG must degrade rather than take the transport down. |
| 63 | */ |
| 64 | const INLINE_IMAGE_MAX_BYTES = Number(process.env.CODEWHALE_CU_MAX_IMAGE_BYTES) > 0 |
| 65 | ? Number(process.env.CODEWHALE_CU_MAX_IMAGE_BYTES) |
| 66 | : 5_000_000; |
| 67 | |
| 68 | /** Base64 expands 3 bytes to 4, padded to a multiple of 4. */ |
| 69 | const encodedSize = (bytes) => Math.ceil(bytes / 3) * 4; |
| 70 | |
| 71 | function receipt(computer, extra) { |
| 72 | return { |
| 73 | computer: computer ? { id: computer.id, transport: computer.transport, platform: computer.platform ?? computer.platformHint ?? null } : null, |
| 74 | ts: new Date().toISOString(), |
| 75 | ...extra, |
| 76 | }; |
| 77 | } |
| 78 | |
| 79 | function fail(computer, code, message, extra = {}) { |
| 80 | return receipt(computer, { ok: false, error: { code, message }, ...extra }); |
| 81 | } |
| 82 | |
| 83 | function invalidateObservations(id) { |
| 84 | lastRasters.delete(id); |
| 85 | latestStateByComputer.delete(id); |
| 86 | for (const [stateId, state] of appStates) { |
| 87 | if (state.computerId === id) appStates.delete(stateId); |
| 88 | } |
| 89 | } |
| 90 | |
| 91 | async function retireBinding(id) { |
| 92 | const binding = backendCache.get(id); |
| 93 | invalidateObservations(id); |
| 94 | boundApps.delete(id); |
| 95 | // A route change means the computer behind the id changed: grants made for |
| 96 | // the old destination must not ride to the new one. |
| 97 | consent.dropSession(id); |
| 98 | if (!binding) return; |
| 99 | closeSshChannel(binding); |
| 100 | // Mark unusable before awaiting cleanup. A failure, or a catalog rollback, |
| 101 | // must never resurrect this backend or its observations. |
| 102 | binding.retired = true; |
| 103 | binding.needsObservation = true; |
| 104 | await withSignal(null, async () => { |
| 105 | const outcomes = await Promise.allSettled([ |
| 106 | binding.usedApp ? closeAppSession() : Promise.resolve(), |
| 107 | (async () => { |
| 108 | try { await binding.backend?.releaseInput?.(); } |
| 109 | finally { await binding.backend?.closeSession?.(); } |
| 110 | })(), |
| 111 | ]); |
| 112 | const failed = outcomes.find(result => result.status === "rejected"); |
| 113 | if (failed) throw failed.reason; |
| 114 | }); |
| 115 | binding.backend = null; |
| 116 | } |
| 117 | |
| 118 | async function bindComputer(computer) { |
| 119 | const route = routeFingerprint(computer); |
| 120 | let binding = backendCache.get(computer.id); |
| 121 | if (binding && (binding.route !== route || binding.retired)) { |
| 122 | await retireBinding(computer.id); |
| 123 | binding = { route, needsObservation: true }; |
| 124 | backendCache.set(computer.id, binding); |
| 125 | } else if (!binding) { |
| 126 | binding = { route, needsObservation: false }; |
| 127 | backendCache.set(computer.id, binding); |
| 128 | } |
| 129 | return binding; |
| 130 | } |
| 131 | |
| 132 | async function assertCurrentRoute(computer, binding, dispatched = false) { |
| 133 | try { |
| 134 | let current; |
| 135 | try { current = registry.get(computer.id); } |
| 136 | catch (err) { await retireBinding(computer.id); throw err; } |
| 137 | if (binding.retired || routeFingerprint(current) !== binding.route) { |
| 138 | await bindComputer(current); |
| 139 | throw new ServerError("computer_route_changed", "Computer route changed during this request — observe the registered target again before acting"); |
| 140 | } |
| 141 | } catch (err) { |
| 142 | if (dispatched) err.requestDispatched = true; |
| 143 | throw err; |
| 144 | } |
| 145 | } |
| 146 | |
| 147 | async function getBackend(computer, binding) { |
| 148 | if (!binding.backend) binding.backend = (await backendFor(computer)).backend; |
| 149 | return binding.backend; |
| 150 | } |
| 151 | |
| 152 | /** |
| 153 | * Element target -> enriched target with cached app identity and AX path. |
| 154 | * An explicit state_id pins a specific observation; a bare index addresses |
| 155 | * the latest observation on this computer — the flat addressing a caller |
| 156 | * uses when it acts on what it just saw. |
| 157 | */ |
| 158 | function resolveElement(target, computer) { |
| 159 | const stateId = target.state_id ?? latestStateByComputer.get(computer.id); |
| 160 | const st = stateId ? appStates.get(stateId) : null; |
| 161 | if (!st) throw new ServerError("unknown_state", target.state_id |
| 162 | ? `state_id "${target.state_id}" is unknown or expired — call get_app_state again` |
| 163 | : "no observation on this computer yet — call get_app_state first"); |
| 164 | const el = st.elements[target.index]; |
| 165 | if (!el) throw new ServerError("unknown_element", `element index ${target.index} is outside state ${stateId} (0..${st.elements.length - 1})`); |
| 166 | return { state: st, element: el, stateId }; |
| 167 | } |
| 168 | |
| 169 | class ServerError extends Error { |
| 170 | constructor(code, message, extra = null) { super(message); this.code = code; if (extra) this.extra = extra; } |
| 171 | } |
| 172 | |
| 173 | /** Map raster-pixel coordinates to screen points using the bound raster. */ |
| 174 | function rasterToPoints(computerId, x, y) { |
| 175 | const r = lastRasters.get(computerId); |
| 176 | if (!r) throw new ServerError("no_raster", "no screenshot bound on this computer yet — call screenshot first so pixel targets have a frame"); |
| 177 | if (r.pixels?.w != null && r.pixels?.h != null && (x < 0 || y < 0 || x >= r.pixels.w || y >= r.pixels.h)) { |
| 178 | throw new ServerError("target_outside_raster", `target (${x},${y}) is outside the bound raster (${r.pixels.w}x${r.pixels.h} pixels) — take a fresh screenshot`); |
| 179 | } |
| 180 | const scale = r.scale && r.scale > 0 ? r.scale : 1; |
| 181 | return { x: (r.origin?.x ?? 0) + x / scale, y: (r.origin?.y ?? 0) + y / scale }; |
| 182 | } |
| 183 | |
| 184 | /** |
| 185 | * Normalize a target into backend form: points for coordinates, resolved |
| 186 | * element for elements. Element targets are revalidated against the live |
| 187 | * backend when a resolver is available: stale elements throw `element_stale`, |
| 188 | * moved-but-identical elements are re-aimed at their fresh center |
| 189 | * (sink.reacquired = true so the receipt can say target_reacquired). |
| 190 | */ |
| 191 | async function normalizeTarget(computer, target, kind, resolve, sink) { |
| 192 | if (target?.type === "coordinate") { |
| 193 | if (target.space === "screen") { |
| 194 | if (!Number.isFinite(target.x) || !Number.isFinite(target.y)) { |
| 195 | throw new ServerError("bad_target", "screen coordinates must be finite numbers"); |
| 196 | } |
| 197 | return { x: Math.round(target.x), y: Math.round(target.y), strategy: "event", coordinate_space: "screen" }; |
| 198 | } |
| 199 | if (target.x < 0 || target.y < 0) throw new ServerError("bad_target", "raster coordinates must be non-negative"); |
| 200 | const pt = rasterToPoints(computer.id, target.x, target.y); |
| 201 | return { x: Math.round(pt.x), y: Math.round(pt.y), strategy: "event", coordinate_space: "raster" }; |
| 202 | } |
| 203 | if (target?.type === "element") { |
| 204 | const { state, element, stateId } = resolveElement(target, computer); |
| 205 | if (state.computerId && state.computerId !== computer.id) { |
| 206 | throw new ServerError("state_wrong_computer", `state_id "${stateId}" belongs to computer "${state.computerId}", not "${computer.id}" — call get_app_state on that computer again`); |
| 207 | } |
| 208 | // The receipt must name the observation actually resolved — a bare index |
| 209 | // binds the computer's latest state, so reporting `target.state_id` would |
| 210 | // say "undefined" for the common case. |
| 211 | const where = `state ${stateId} (${state.app_ref?.name ?? state.app_ref?.bundle_id ?? `pid ${state.app_ref?.pid}`})`; |
| 212 | let fresh = null; |
| 213 | if (resolve) { |
| 214 | const res = await resolve({ app_ref: state.app_ref, windowIndex: element.windowIndex ?? 0, path: element.path }); |
| 215 | if (!res?.found || !res.element) { |
| 216 | throw new ServerError("element_stale", `element ${target.index} of ${where} no longer resolves (${res?.reason ?? "not_found"}) — the user or the app may have changed it; call get_app_state again`); |
| 217 | } |
| 218 | fresh = res.element; |
| 219 | if (fresh.role !== element.role) { |
| 220 | throw new ServerError("element_stale", `element ${target.index} of ${where} changed role (${element.role} → ${fresh.role}) — call get_app_state again`); |
| 221 | } |
| 222 | // In-place replacement: same role and geometry but a different label is |
| 223 | // still a different element (e.g. "Load" → "Confirm"). |
| 224 | if (fresh.label !== element.label) { |
| 225 | throw new ServerError("element_stale", `element ${target.index} of ${where} changed label (${element.label} → ${fresh.label}) — call get_app_state again`); |
| 226 | } |
| 227 | } |
| 228 | if (kind === "semantic") { |
| 229 | return { |
| 230 | app_ref: state.app_ref, windowIndex: element.windowIndex ?? 0, path: element.path, |
| 231 | strategy: "a11y", role: element.role, label: element.label, reacquired: false, |
| 232 | ...(element.runtime_id ? { runtime_id: element.runtime_id, window_runtime_id: element.window_runtime_id } : {}), |
| 233 | }; |
| 234 | } |
| 235 | const moved = !!fresh && ( |
| 236 | fresh.position?.x !== element.position?.x || fresh.position?.y !== element.position?.y || |
| 237 | fresh.size?.w !== element.size?.w || fresh.size?.h !== element.size?.h); |
| 238 | const pos = fresh?.position ?? element.position; |
| 239 | const sz = fresh?.size ?? element.size; |
| 240 | if (!pos || !sz) throw new ServerError("element_no_geometry", `element ${target.index} of ${where} has no cached geometry — use a coordinate target`); |
| 241 | if (moved && sink) sink.reacquired = true; |
| 242 | // Keep the element identity as well as geometry: semantic clicks must not |
| 243 | // substitute whichever element happens to occupy an oversized AX center. |
| 244 | const c = { x: Math.round(pos.x + sz.w / 2), y: Math.round(pos.y + sz.h / 2) }; |
| 245 | return { ...c, strategy: "a11y-center", role: element.role, label: element.label, app_ref: state.app_ref, |
| 246 | windowIndex: element.windowIndex ?? 0, path: element.path, reacquired: moved }; |
| 247 | } |
| 248 | throw new ServerError("bad_target", "target must be {type:'coordinate',x,y} or {type:'element',index} (state_id optional to pin a specific observation)"); |
| 249 | } |
| 250 | |
| 251 | function bindRaster(computer, shot) { |
| 252 | lastRasters.set(computer.id, { |
| 253 | file: shot.file ?? shot.path, |
| 254 | scale: shot.scale ?? 1, |
| 255 | origin: shot.points ?? { x: 0, y: 0 }, |
| 256 | pixels: shot.pixels ?? null, |
| 257 | capturedAt: shot.capturedAt ?? new Date().toISOString(), |
| 258 | }); |
| 259 | } |
| 260 | |
| 261 | /** A zoom produces a child raster: origin shifted by the crop, parent scale. */ |
| 262 | function bindZoomRaster(computer, parent, region, file) { |
| 263 | const scale = parent.scale && parent.scale > 0 ? parent.scale : 1; |
| 264 | lastRasters.set(computer.id, { |
| 265 | file, |
| 266 | scale, |
| 267 | origin: { |
| 268 | x: (parent.origin?.x ?? 0) + region[0] / scale, |
| 269 | y: (parent.origin?.y ?? 0) + region[1] / scale, |
| 270 | }, |
| 271 | pixels: { w: region[2], h: region[3] }, |
| 272 | parent: parent.file, |
| 273 | capturedAt: new Date().toISOString(), |
| 274 | }); |
| 275 | } |
| 276 | |
| 277 | function rememberState(computer, app_ref, result) { |
| 278 | const id = `s-${++stateCounter}`; |
| 279 | // The observed identity wins over the caller's hint: "chrome" may have |
| 280 | // resolved to "Google Chrome", and later re-resolution has to name the same |
| 281 | // process, not re-run a loose match that could pick a different one. |
| 282 | const resolved = { ...app_ref }; |
| 283 | for (const key of ["pid", "bundle_id", "name"]) if (result[key] != null && result[key] !== "") resolved[key] = result[key]; |
| 284 | appStates.set(id, { computerId: computer.id, app_ref: resolved, elements: result.elements ?? [], ts: Date.now() }); |
| 285 | latestStateByComputer.set(computer.id, id); |
| 286 | if (appStates.size > 24) { |
| 287 | for (const k of appStates.keys()) { appStates.delete(k); break; } |
| 288 | } |
| 289 | return id; |
| 290 | } |
| 291 | |
| 292 | function filterElements(elements, { detail, query, role, limit, offset, compact }) { |
| 293 | const full = detail === "full"; |
| 294 | let rows = (elements ?? []).map((el, i) => ({ ...el, index: el.index ?? i })); |
| 295 | if (!full) { |
| 296 | rows = rows.filter((el) => el.windowIndex !== -1 || !Array.isArray(el.path) || el.path.length <= 1); |
| 297 | } |
| 298 | if (role) rows = rows.filter((el) => el.role === role); |
| 299 | if (query) { |
| 300 | const q = String(query).toLowerCase(); |
| 301 | rows = rows.filter((el) => [el.label, el.value, el.role, el.subrole].some((v) => String(v ?? "").toLowerCase().includes(q))); |
| 302 | } |
| 303 | const matched = rows.length; |
| 304 | const start = Math.max(0, Number(offset) || 0); |
| 305 | const cap = limit != null ? Math.max(1, Math.min(200, Number(limit))) : null; |
| 306 | const sliced = cap != null ? rows.slice(start, start + cap) : rows.slice(start); |
| 307 | const view = sliced.map((el) => { |
| 308 | if (full) return el; |
| 309 | const { path, windowIndex, ...rest } = el; |
| 310 | if (!compact) return rest; |
| 311 | const label = rest.label != null ? String(rest.label).slice(0, 80) : rest.label; |
| 312 | const value = rest.value != null && String(rest.value).length > 200 ? String(rest.value).slice(0, 200) : rest.value; |
| 313 | return { index: rest.index, role: rest.role, label, value, focused: rest.focused, enabled: rest.enabled, actions: rest.actions }; |
| 314 | }); |
| 315 | return { elements: view, matched, offset: start, returned: view.length, truncated: start + view.length < matched }; |
| 316 | } |
| 317 | |
| 318 | function fitStatePayload(data, budget) { |
| 319 | let payload = data; |
| 320 | let json = JSON.stringify(payload); |
| 321 | if (json.length <= budget) return payload; |
| 322 | if (payload.ocr) { |
| 323 | payload = { ...payload, ocr: { status: payload.ocr.status ?? "omitted", omitted: true, reason: "ocr_too_large", note: "OCR omitted so this observation stays readable. Retry include_ocr with ocr_region, query, or a smaller window." } }; |
| 324 | json = JSON.stringify(payload); |
| 325 | if (json.length <= budget) return { ...payload, truncated: true }; |
| 326 | } |
| 327 | let elements = payload.elements ?? []; |
| 328 | const matched = payload.matched ?? elements.length; |
| 329 | while (elements.length > 4 && json.length > budget) { |
| 330 | elements = elements.slice(0, Math.max(4, Math.floor(elements.length / 2))); |
| 331 | payload = { |
| 332 | ...payload, |
| 333 | elements, |
| 334 | truncated: true, |
| 335 | matched, |
| 336 | returned: elements.length, |
| 337 | next_offset: (payload.offset ?? 0) + elements.length, |
| 338 | note: "Observation truncated to keep the transport intact. Pass query, role, limit and offset; do not retry an unfiltered dump.", |
| 339 | }; |
| 340 | json = JSON.stringify(payload); |
| 341 | } |
| 342 | return payload; |
| 343 | } |
| 344 | |
| 345 | async function invokeType(invoke, prepared) { |
| 346 | const text = String(prepared.text ?? ""); |
| 347 | const pressEnter = prepared.press_enter === true; |
| 348 | const parts = text.split(/\r\n|\n|\r/); |
| 349 | const rest = { ...prepared }; |
| 350 | delete rest.press_enter; |
| 351 | if (parts.length === 1 && !pressEnter) return invoke("type", rest); |
| 352 | const steps = []; |
| 353 | for (let i = 0; i < parts.length; i++) { |
| 354 | if (parts[i]) steps.push(await invoke("type", { ...rest, text: parts[i] })); |
| 355 | if (i < parts.length - 1 || (pressEnter && i === parts.length - 1)) { |
| 356 | steps.push(await invoke("key", { text: "return" })); |
| 357 | } |
| 358 | } |
| 359 | const last = steps.at(-1) ?? { action_sent: true }; |
| 360 | return { ...last, newlines_as_return: true, typed_parts: steps.length }; |
| 361 | } |
| 362 | |
| 363 | function observeState(computer, app_ref, result, args = {}) { |
| 364 | // Cache the complete backend records before making the model-facing view. |
| 365 | // Public indices still address those records, including their private AX |
| 366 | // paths; a compact response must never weaken live target revalidation. |
| 367 | // Ephemeral polls (wait_for) share the filter math without churning the |
| 368 | // state cache: only the observation a caller can act on earns a state_id. |
| 369 | const ephemeral = args.ephemeral === true; |
| 370 | const state_id = ephemeral ? null : rememberState(computer, app_ref, result); |
| 371 | const compact = args.detail === "compact" || args.compact === true; |
| 372 | const detail = args.detail === "full" ? "full" : compact ? "compact" : "summary"; |
| 373 | const filtered = filterElements(result.elements, { |
| 374 | detail: args.detail === "full" ? "full" : "summary", |
| 375 | query: args.query, |
| 376 | role: args.role, |
| 377 | limit: args.limit, |
| 378 | offset: args.offset, |
| 379 | compact, |
| 380 | }); |
| 381 | const data = { |
| 382 | ...result, |
| 383 | state_id, |
| 384 | elements: filtered.elements, |
| 385 | detail, |
| 386 | matched: filtered.matched, |
| 387 | offset: filtered.offset, |
| 388 | returned: filtered.returned, |
| 389 | truncated: filtered.truncated, |
| 390 | note: ephemeral |
| 391 | ? "Ephemeral poll: elements are not bound to a state_id." |
| 392 | : "Indices target this observation's cached tree (including rows not shown); pin it with state_id, or re-observe after the app changes.", |
| 393 | }; |
| 394 | if (compact && data.ocr && args.include_ocr !== true) delete data.ocr; |
| 395 | return fitStatePayload(data, STATE_CHAR_BUDGET); |
| 396 | } |
| 397 | |
| 398 | /** |
| 399 | * Poll get_app_state until the query/role predicate holds or the deadline |
| 400 | * passes. Intermediate polls are ephemeral — they share the filter math but |
| 401 | * never churn the state cache; the observation that satisfies the predicate |
| 402 | * is read once more, bound, and its state_id is what the caller targets. |
| 403 | * Errors that can resolve themselves (app not launched yet) count as "no |
| 404 | * match yet"; errors that cannot (stopped, route changed) abort the wait. |
| 405 | */ |
| 406 | async function waitFor(computer, args, switched) { |
| 407 | const { query, role } = args; |
| 408 | if (query == null && role == null) throw new ServerError("bad_args", "wait_for needs a query and/or role to watch for"); |
| 409 | if (query != null && typeof query !== "string") throw new ServerError("bad_args", "query must be a string"); |
| 410 | if (role != null && typeof role !== "string") throw new ServerError("bad_args", "role must be a string"); |
| 411 | const state = args.state ?? "present"; |
| 412 | if (state !== "present" && state !== "absent") throw new ServerError("bad_args", 'state must be "present" or "absent"'); |
| 413 | const timeoutSec = Number(args.timeout ?? 10); |
| 414 | if (!Number.isFinite(timeoutSec) || timeoutSec < 0.5 || timeoutSec > 60) throw new ServerError("bad_args", "timeout must be 0.5..60 seconds"); |
| 415 | const intervalMs = Number(args.interval ?? 400); |
| 416 | if (!Number.isInteger(intervalMs) || intervalMs < 100 || intervalMs > 5000) throw new ServerError("bad_args", "interval must be an integer 100..5000 ms"); |
| 417 | const limit = args.limit ?? 20; |
| 418 | if (!Number.isSafeInteger(limit) || limit < 1 || limit > 100) throw new ServerError("bad_args", "limit must be an integer 1..100"); |
| 419 | |
| 420 | const FATAL = new Set(["cancelled", "control_stopped", "computer_route_changed", "app_upgrade_required"]); |
| 421 | const observe = (ephemeral) => callTool({ name: "get_app_state", arguments: { |
| 422 | app_ref: args.app_ref, window_id: args.window_id, query, role, |
| 423 | limit, detail: "compact", ephemeral, computer: computer.id, |
| 424 | }}); |
| 425 | const started = Date.now(); |
| 426 | const deadline = started + timeoutSec * 1000; |
| 427 | let polls = 0, lastError = null, everObserved = false; |
| 428 | while (true) { |
| 429 | const res = await observe(true); |
| 430 | polls++; |
| 431 | const body = JSON.parse(res.content[0].text); |
| 432 | let usable = false, matchedCount = 0; |
| 433 | if (!res.isError && body.ok !== false) { usable = true; matchedCount = body.matched ?? 0; } |
| 434 | else if (FATAL.has(body?.error?.code)) { |
| 435 | return { content: [{ type: "text", text: JSON.stringify(fail(computer, body.error.code, body.error.message, { tool: "wait_for", switched, polls })) }], isError: true }; |
| 436 | } else if (body?.found === false || /application not found/.test(body?.error?.message ?? "")) { |
| 437 | usable = true; // not running yet, or gone: zero matches either way |
| 438 | } else { |
| 439 | lastError = body?.error ?? { code: "observe_failed", message: "observation failed" }; |
| 440 | } |
| 441 | if (usable) { everObserved = true; lastError = null; } |
| 442 | if (usable && (state === "absent" ? matchedCount === 0 : matchedCount > 0)) { |
| 443 | const bound = await observe(false); |
| 444 | polls++; |
| 445 | const b = JSON.parse(bound.content[0].text); |
| 446 | if (bound.isError || b.ok === false) { |
| 447 | return { content: [{ type: "text", text: JSON.stringify(receipt(computer, { ok: true, tool: "wait_for", switched, matched: true, state, polls, elapsed_ms: Date.now() - started, note: "Condition held but the follow-up observation failed — call get_app_state before targeting." })) }] }; |
| 448 | } |
| 449 | return { content: [{ type: "text", text: JSON.stringify(receipt(computer, { ok: true, tool: "wait_for", switched, matched: true, state, polls, elapsed_ms: Date.now() - started, state_id: b.state_id, matched_count: b.matched ?? 0, elements: b.elements, app: { name: b.name ?? null, pid: b.pid ?? null, bundle_id: b.bundle_id ?? null }, note: "Elements are bound to this observation — target them with {type:'element', index}; add state_id only to pin this snapshot after later observes. Re-observe if the UI changes again." })) }] }; |
| 450 | } |
| 451 | if (Date.now() >= deadline) break; |
| 452 | await wait(Math.min(intervalMs, Math.max(1, deadline - Date.now()))); |
| 453 | throwIfAborted(); |
| 454 | } |
| 455 | if (!everObserved && lastError) { |
| 456 | return { content: [{ type: "text", text: JSON.stringify(fail(computer, lastError.code ?? "observe_failed", lastError.message ?? "observation failed", { tool: "wait_for", switched, polls, elapsed_ms: Date.now() - started })) }], isError: true }; |
| 457 | } |
| 458 | return { content: [{ type: "text", text: JSON.stringify(receipt(computer, { ok: true, tool: "wait_for", switched, matched: false, timed_out: true, state, polls, elapsed_ms: Date.now() - started, ...(lastError ? { last_error: lastError } : {}), note: state === "absent" ? "Matches remained until the deadline." : "No match appeared before the deadline. Observe the app or widen the query." })) }] }; |
| 459 | } |
| 460 | |
| 461 | // ---------- per-app consent ---------- |
| 462 | // The app, not the tool, is the unit of trust on the local computer: the |
| 463 | // first call that targets an application — binding input to it, observing it |
| 464 | // by name, or acting through a bound/element target — refuses |
| 465 | // consent_required until the user records a decision with the consent tool. |
| 466 | // Spawned computers are exempt: a task-owned desktop holds nothing of the |
| 467 | // user's, and remote machines are covered by the transport's own trust. |
| 468 | |
| 469 | /** Tools whose implicit target is the bound app when no explicit app_ref or element is given. */ |
| 470 | const BOUND_TARGET_TOOLS = new Set([ |
| 471 | "get_app_state", "find_elements", "wait_for", "list_windows", "screenshot", "zoom", |
| 472 | "recording_start", "preview", "invoke_menu", |
| 473 | "type", "key", "hold_key", |
| 474 | "left_click", "double_click", "triple_click", "right_click", "middle_click", |
| 475 | "left_click_drag", "mouse_move", "left_mouse_down", "left_mouse_up", "scroll", |
| 476 | "set_value", "focus", "get_value", "select_text", "perform_action", |
| 477 | ]); |
| 478 | |
| 479 | /** App identity the way consent args carry it (app string, or explicit fields). */ |
| 480 | function refFromConsentArgs(args) { |
| 481 | const ref = {}; |
| 482 | if (typeof args.bundle_id === "string" && args.bundle_id.trim()) ref.bundle_id = args.bundle_id.trim(); |
| 483 | if (typeof args.name === "string" && args.name.trim()) ref.name = args.name.trim(); |
| 484 | if (Number.isInteger(args.pid) && args.pid > 0) ref.pid = args.pid; |
| 485 | if (typeof args.app === "string" && args.app.trim() && !Object.keys(ref).length) { |
| 486 | const s = args.app.trim(); |
| 487 | if (/^pid:\d+$/i.test(s)) ref.pid = Number(s.slice(4)); |
| 488 | else if (/^\d+$/.test(s)) ref.pid = Number(s); |
| 489 | // ".app" is a filename spelling and always means a name — checked |
| 490 | // before the reverse-DNS shape it also satisfies. |
| 491 | else if (/\.app$/i.test(s)) ref.name = s.replace(/\.app$/i, ""); |
| 492 | else if (/^[A-Za-z0-9-]+(\.[A-Za-z0-9-]+)+$/.test(s) && !s.includes(" ")) ref.bundle_id = s; |
| 493 | else ref.name = s; |
| 494 | } |
| 495 | return ref; |
| 496 | } |
| 497 | |
| 498 | /** |
| 499 | * Best-effort identity enrichment through list_apps — the same match rules |
| 500 | * the native resolver uses (pid exact; name and bundle id case-insensitive). |
| 501 | * Returns {name, pid, bundle_id} or null. Only consulted when a decision is |
| 502 | * missing, so the ledger sees the same app under every spelling the model |
| 503 | * might use. |
| 504 | */ |
| 505 | async function resolveAppIdentity(computer, ref) { |
| 506 | if (!ref || !Object.keys(ref).length) return null; |
| 507 | let apps = null; |
| 508 | try { |
| 509 | const res = await callTool({ name: "list_apps", arguments: { computer: computer.id } }); |
| 510 | const body = JSON.parse(res.content[0].text); |
| 511 | apps = body?.apps ?? null; |
| 512 | } catch { return null; } |
| 513 | if (!Array.isArray(apps)) return null; |
| 514 | const wantName = ref.name?.toLowerCase(), wantBundle = ref.bundle_id?.toLowerCase(); |
| 515 | const hit = apps.find((a) => |
| 516 | (ref.pid != null && a.pid === ref.pid) || |
| 517 | (wantBundle && String(a.bundle_id ?? "").toLowerCase() === wantBundle) || |
| 518 | (wantName && String(a.name ?? "").toLowerCase() === wantName)); |
| 519 | return hit ? { name: hit.name ?? null, pid: hit.pid ?? null, bundle_id: hit.bundle_id ?? null } : null; |
| 520 | } |
| 521 | |
| 522 | /** |
| 523 | * Check the ledger for one app reference: direct keys first, then — only when |
| 524 | * undecided — the resolved running-app identity so a grant made under one |
| 525 | * spelling covers the others. Returns {verdict, ref} where ref is the richest |
| 526 | * identity known (for the refusal's app field and alias merging). |
| 527 | */ |
| 528 | async function consentForRef(computer, ref) { |
| 529 | const direct = consent.decisionFor(computer.id, consent.appKeys(ref)); |
| 530 | if (direct.state !== "undecided") return { verdict: direct, ref }; |
| 531 | const resolved = await resolveAppIdentity(computer, ref); |
| 532 | if (!resolved) return { verdict: direct, ref }; |
| 533 | const widened = consent.decisionFor(computer.id, consent.appKeys(resolved)); |
| 534 | return { verdict: widened, ref: resolved }; |
| 535 | } |
| 536 | |
| 537 | /** |
| 538 | * The consent gate, run inside dispatch before any backend call. Returns |
| 539 | * {grant} describing the decision that let the call through (used to merge |
| 540 | * aliases after open_application resolves the real identity), or null when |
| 541 | * the call targets no app. Throws ServerError consent_required / app_denied / |
| 542 | * foreground_consent_required / foreground_denied. |
| 543 | */ |
| 544 | async function consentCheck(computer, name, args) { |
| 545 | if (computer.transport !== "local" || computer.owned === true) return null; |
| 546 | const refs = []; |
| 547 | if (name === "open_application" || name === "kill_app") { |
| 548 | // Both name the target app with name/bundle_id/pid args — a denied app |
| 549 | // must not be terminable any more than it must be bindable. |
| 550 | const ref = {}; |
| 551 | if (Number.isInteger(args.pid)) ref.pid = args.pid; |
| 552 | else if (typeof args.bundle_id === "string" && args.bundle_id) ref.bundle_id = args.bundle_id; |
| 553 | else if (typeof args.name === "string" && args.name) ref.name = args.name; |
| 554 | if (Object.keys(ref).length) refs.push(ref); |
| 555 | } else { |
| 556 | if (args.app_ref && typeof args.app_ref === "object") refs.push(args.app_ref); |
| 557 | for (const key of ["target", "from_target", "to"]) { |
| 558 | if (args[key]?.type === "element") { |
| 559 | try { refs.push(resolveElement(args[key], computer).state.app_ref); } catch { /* the element gate reports its own staleness */ } |
| 560 | } |
| 561 | } |
| 562 | if (args.state_id != null) { |
| 563 | const st = appStates.get(args.state_id); |
| 564 | if (st?.computerId === computer.id) refs.push(st.app_ref); |
| 565 | } |
| 566 | const bound = boundApps.get(computer.id); |
| 567 | if (!refs.length && bound && BOUND_TARGET_TOOLS.has(name)) refs.push(bound); |
| 568 | } |
| 569 | let grant = null; |
| 570 | for (const ref of refs) { |
| 571 | // A state or element whose backend reported no identity at all has no app |
| 572 | // to consent to — observation never named one either, so there is nothing |
| 573 | // a recorded decision could match. |
| 574 | if (!ref || !consent.appKeys(ref).length) continue; |
| 575 | const { verdict, ref: known } = await consentForRef(computer, ref); |
| 576 | const desc = known.name ?? known.bundle_id ?? (known.pid ? `pid ${known.pid}` : "the application"); |
| 577 | const arg = known.bundle_id ?? known.name ?? (known.pid ? `pid:${known.pid}` : "the app"); |
| 578 | if (verdict.state === "denied") { |
| 579 | throw new ServerError("app_denied", |
| 580 | `the user denied access to ${desc} on this computer — do not work around it; only they can change it (consent {action:"revoke"}).`, |
| 581 | { app: known }); |
| 582 | } |
| 583 | if (verdict.state === "undecided") { |
| 584 | throw new ServerError("consent_required", |
| 585 | `Codewhale needs the user's permission to use ${desc} on this computer — ask them, then record their answer with consent {action:"allow"|"deny", app:"${arg}"}.`, |
| 586 | { app: known }); |
| 587 | } |
| 588 | grant = { ref: known, persisted: verdict.persisted === true }; |
| 589 | } |
| 590 | // Taking the shared pointer/focus is a second, separate consent: the first |
| 591 | // activate:true is the moment the agent stops being background — on every |
| 592 | // platform, not just macOS. |
| 593 | if (name === "open_application" && args.activate === true) { |
| 594 | const fg = consent.foregroundDecision(computer.id); |
| 595 | if (fg.state === "denied") { |
| 596 | throw new ServerError("foreground_denied", |
| 597 | `the user denied shared-desktop (foreground) control on this computer — continue with open_application activate:false (background control) or ask them to reconsider.`, |
| 598 | { scope: "foreground" }); |
| 599 | } |
| 600 | if (fg.state === "undecided") { |
| 601 | throw new ServerError("foreground_consent_required", |
| 602 | `open_application activate:true would take this computer's shared pointer and focus — ask the user, then record their answer with consent {action:"allow"|"deny", scope:"foreground"}. Background control (activate:false) needs no such consent.`, |
| 603 | { scope: "foreground" }); |
| 604 | } |
| 605 | } |
| 606 | return grant ? { grant } : null; |
| 607 | } |
| 608 | |
| 609 | // ---------- tool dispatch ---------- |
| 610 | async function callTool(params) { |
| 611 | const requested = params.name; |
| 612 | if (!TOOL_NAMES.has(requested)) { |
| 613 | return { content: [{ type: "text", text: JSON.stringify({ ok: false, error: { code: "unknown_tool", message: `unknown tool "${requested}"` } }) }], isError: true }; |
| 614 | } |
| 615 | // Merged tools (click, pointer, clipboard, recording, computer, key+duration) |
| 616 | // resolve to the wire tool they dispatch to before any gate below, so they |
| 617 | // cannot bypass required args, the kill switch or routing. Wire names stay |
| 618 | // callable as aliases. |
| 619 | let name = requested; |
| 620 | let args = params.arguments ?? {}; |
| 621 | try { |
| 622 | ({ name, args } = resolveTool(requested, args)); |
| 623 | } catch (err) { |
| 624 | return { content: [{ type: "text", text: JSON.stringify(fail(null, err.code ?? "bad_args", err.message)) }], isError: true }; |
| 625 | } |
| 626 | // A narrowed session (CODEWHALE_CU_GRANT) refuses anything outside its grant |
| 627 | // before required-arg or routing behavior can leak. stop_computer_control |
| 628 | // stays reachable as the safety valve; the daemon enforces the same set. |
| 629 | if (GRANT && requested !== "stop_computer_control" && !GRANT.has(requested) && !GRANT.has(name)) { |
| 630 | return { content: [{ type: "text", text: JSON.stringify(fail(null, "not_granted", `"${requested}" is outside this session's capability grant (${GRANT.size} tools). The host narrowed this session deliberately; do not look for a workaround.`)) }], isError: true }; |
| 631 | } |
| 632 | // Hosts are not required to enforce inputSchema. Check declared `required` |
| 633 | // fields here so a missing argument becomes bad_args instead of a backend |
| 634 | // crash or an opaque native error. The message names the tool the caller |
| 635 | // asked for, not the wire name it resolved to. |
| 636 | for (const field of REQUIRED_ARGS.get(name) ?? []) { |
| 637 | if (args[field] === undefined || args[field] === null) { |
| 638 | return { content: [{ type: "text", text: JSON.stringify(fail(null, "bad_args", `${requested} requires "${field}"`)) }], isError: true }; |
| 639 | } |
| 640 | } |
| 641 | |
| 642 | if (name === "stop_computer_control") { |
| 643 | controlStopped = true; |
| 644 | for (const request of requests.values()) { |
| 645 | if (request.name && request.name !== "stop_computer_control") request.controller.abort(); |
| 646 | } |
| 647 | try { |
| 648 | await releaseControl({ releaseOnly: true }); |
| 649 | return { content: [{ type: "text", text: JSON.stringify(receipt(null, { ok: true, stopped: true, inFlight, inputReleased: true, note: "Queued input was refused and ongoing requests were cancelled. Input already delivered cannot be undone. Restart this MCP session to resume." })) }] }; |
| 650 | } catch (err) { |
| 651 | return { content: [{ type: "text", text: JSON.stringify(fail(null, "input_release_failed", String(err?.message ?? err), { stopped: true, inFlight })) }], isError: true }; |
| 652 | } |
| 653 | } |
| 654 | if (controlStopped && !READ_ONLY_TOOLS.has(name)) { |
| 655 | return { content: [{ type: "text", text: JSON.stringify(fail(null, "control_stopped", "stop_computer_control is active; no further actions are permitted this session")) }], isError: true }; |
| 656 | } |
| 657 | |
| 658 | if (name === "wait") { |
| 659 | const s = Math.max(0, Math.min(30, Number(args.seconds) || 1)); |
| 660 | await wait(s * 1000); |
| 661 | return { content: [{ type: "text", text: JSON.stringify(receipt(null, { ok: true, waitedSec: s })) }] }; |
| 662 | } |
| 663 | |
| 664 | if (name === "trajectory_start") { |
| 665 | const r = recorder.start(); |
| 666 | return { content: [{ type: "text", text: JSON.stringify(receipt(null, { ok: true, tool: "trajectory_start", ...r, note: "Every tool call this session makes is appended to a local JSONL. Arguments are stored verbatim so replay is faithful — start it only when the person knows it runs." })) }] }; |
| 667 | } |
| 668 | if (name === "trajectory_stop") { |
| 669 | return { content: [{ type: "text", text: JSON.stringify(receipt(null, { ok: true, tool: "trajectory_stop", ...recorder.stop() })) }] }; |
| 670 | } |
| 671 | if (name === "trajectory_status") { |
| 672 | return { content: [{ type: "text", text: JSON.stringify(receipt(null, { ok: true, tool: "trajectory_status", ...recorder.status(), recent: listTrajectories(5) })) }] }; |
| 673 | } |
| 674 | if (name === "trajectory_replay") { |
| 675 | let file; |
| 676 | try { file = resolveTrajectory(args.id); } catch (err) { |
| 677 | return { content: [{ type: "text", text: JSON.stringify(fail(null, err.code ?? "bad_args", err.message)) }], isError: true }; |
| 678 | } |
| 679 | const calls = readTrajectory(file).filter((entry) => entry.type === "call" && typeof entry.tool === "string" && !isTrajectoryTool(entry.tool)); |
| 680 | if (calls.length > 200) { |
| 681 | return { content: [{ type: "text", text: JSON.stringify(fail(null, "replay_too_large", `this trajectory has ${calls.length} calls; replay is limited to 200 at a time`)) }], isError: true }; |
| 682 | } |
| 683 | const dryRun = args.dry_run === true; |
| 684 | const results = []; |
| 685 | if (!dryRun) { |
| 686 | replaying = true; |
| 687 | try { |
| 688 | for (const call of calls) { |
| 689 | if (controlStopped && !READ_ONLY_TOOLS.has(call.tool)) { results.push({ tool: call.tool, ok: false, code: "control_stopped" }); break; } |
| 690 | let body = null; |
| 691 | try { |
| 692 | const r = await callTool({ name: call.tool, arguments: call.args ?? {} }); |
| 693 | body = JSON.parse(r?.content?.[0]?.text ?? "null"); |
| 694 | } catch (err) { |
| 695 | results.push({ tool: call.tool, ok: false, code: err?.code ?? "replay_failed", message: String(err?.message ?? err).slice(0, 200) }); |
| 696 | break; |
| 697 | } |
| 698 | const ok = body?.ok !== false; |
| 699 | results.push({ tool: call.tool, ok, ...(ok ? {} : { code: body?.error?.code ?? "refused" }) }); |
| 700 | if (!ok) break; // a trajectory is a sequence — replay stops where it broke |
| 701 | } |
| 702 | } finally { replaying = false; } |
| 703 | } |
| 704 | const failed = results.filter((r) => r.ok === false).length; |
| 705 | return { content: [{ type: "text", text: JSON.stringify(receipt(null, { ok: true, tool: "trajectory_replay", trajectory: path.basename(file), dry_run: dryRun, turns_in_file: calls.length, replayed: results.length, failed, ...(dryRun ? { plan: calls.map((c) => c.tool) } : { results }), note: dryRun ? "Nothing was executed. Run again without dry_run:true to replay through the normal gates." : "Replay re-entered the normal pipeline; grants, permissions and the kill switch still apply." })) }] }; |
| 706 | } |
| 707 | |
| 708 | if (name === "computer_list") { |
| 709 | const reg = registry.list(); |
| 710 | return { content: [{ type: "text", text: JSON.stringify(receipt(null, { |
| 711 | ok: true, |
| 712 | active: activeComputerId, |
| 713 | computers: Object.values(reg.computers).map((c) => ({ id: c.id, transport: c.transport, platform: c.platform ?? c.platformHint ?? null, label: c.label ?? null, host: c.host ?? null, owned: c.owned === true || undefined, container: c.container ?? undefined })), |
| 714 | note: "Pass `computer` on any tool to switch (sticky), or computer_switch to switch explicitly.", |
| 715 | })) }] }; |
| 716 | } |
| 717 | |
| 718 | if (name === "computer_register") { |
| 719 | try { |
| 720 | const entry = registry.register({ id: args.computer, transport: args.transport, label: args.label, host: args.host, port: args.port, user: args.user, target: args.target }); |
| 721 | await bindComputer(entry); |
| 722 | let installed = null; |
| 723 | if (entry.transport === "ssh" && args.installAgent !== false) { |
| 724 | installed = await installRemoteAgent(entry); |
| 725 | registry.register({ id: entry.id, transport: "ssh", host: entry.host, port: entry.port, user: entry.user, platformHint: installed.remotePlatform, agentPath: installed.agentPath }); |
| 726 | } |
| 727 | if (entry.transport === "ssh" && args.installAgent === false && !entry.platformHint) { |
| 728 | // Probe cheaply through the agent; if it is missing, registration still succeeds. |
| 729 | try { |
| 730 | const ex = await executorFor(entry); |
| 731 | const reply = await ex.remote({ tool: "platform" }); |
| 732 | registry.register({ id: entry.id, transport: "ssh", host: entry.host, port: entry.port, user: entry.user, platformHint: reply.platform }); |
| 733 | } catch {} |
| 734 | } |
| 735 | const fresh = registry.get(entry.id); |
| 736 | await bindComputer(fresh); |
| 737 | return { content: [{ type: "text", text: JSON.stringify(receipt(null, { ok: true, registered: { ...fresh, platform: fresh.platform ?? fresh.platformHint ?? null }, agentInstall: installed })) }] }; |
| 738 | } catch (err) { |
| 739 | // Registration problems (unreachable host, agent push failed) are |
| 740 | // receipts, not protocol errors. |
| 741 | return { content: [{ type: "text", text: JSON.stringify(fail(null, err.code ?? "register_failed", err.message ?? String(err))) }], isError: true }; |
| 742 | } |
| 743 | } |
| 744 | |
| 745 | if (name === "computer_spawn") { |
| 746 | try { |
| 747 | if (args.transport !== "docker") throw new ServerError("bad_args", `spawn transport must be "docker" (got ${JSON.stringify(args.transport)})`); |
| 748 | const spawned = await spawnDockerComputer({ id: args.computer, image: args.image }); |
| 749 | let entry; |
| 750 | try { |
| 751 | entry = registry.register({ id: args.computer, transport: "docker", label: args.label, container: spawned.container, image: spawned.image, platform: "linux", owned: true, spawnedBy: SESSION_ID }); |
| 752 | } catch (err) { |
| 753 | // The container exists but could not be registered — spawn is |
| 754 | // transactional, so take the container down with it. |
| 755 | await destroyDockerComputer({ container: spawned.container }).catch(() => {}); |
| 756 | throw err; |
| 757 | } |
| 758 | await bindComputer(entry); |
| 759 | // A spawned computer is the point of the call — it becomes active so |
| 760 | // subsequent tools act on the disposable desktop without a switch. |
| 761 | activeComputerId = entry.id; |
| 762 | return { content: [{ type: "text", text: JSON.stringify(receipt(entry, { ok: true, active: activeComputerId, spawned: { id: entry.id, transport: entry.transport, platform: entry.platform, container: entry.container, image: entry.image, owned: true, built: spawned.built }, note: "This is a disposable, task-owned desktop — it is destroyed by computer remove or when this session ends. The user's own machine is untouched." })) }] }; |
| 763 | } catch (err) { |
| 764 | return { content: [{ type: "text", text: JSON.stringify(fail(null, err.code ?? "spawn_failed", err.message ?? String(err))) }], isError: true }; |
| 765 | } |
| 766 | } |
| 767 | |
| 768 | if (name === "computer_remove") { |
| 769 | let entry = null; |
| 770 | try { entry = registry.get(args.computer); } catch {} |
| 771 | let teardown = null; |
| 772 | if (entry?.transport === "docker") { |
| 773 | try { teardown = await destroyDockerComputer(entry); } |
| 774 | catch (err) { teardown = { destroyed: false, cleanup_error: err.message ?? String(err) }; } |
| 775 | } |
| 776 | const res = registry.remove(args.computer); |
| 777 | if (activeComputerId === args.computer) activeComputerId = "local"; |
| 778 | res.active = activeComputerId; |
| 779 | await retireBinding(args.computer); |
| 780 | return { content: [{ type: "text", text: JSON.stringify(receipt(null, { ok: true, ...res, ...(teardown ?? {}) })) }] }; |
| 781 | } |
| 782 | |
| 783 | if (name === "computer_switch") { |
| 784 | const c = registry.get(args.computer); |
| 785 | activeComputerId = c.id; |
| 786 | return { content: [{ type: "text", text: JSON.stringify(receipt(c, { ok: true, active: c.id })) }] }; |
| 787 | } |
| 788 | |
| 789 | // Everything below acts on a computer. |
| 790 | let computer; |
| 791 | let switched = false; |
| 792 | try { |
| 793 | if (args.computer && args.computer !== activeComputerId) { |
| 794 | computer = registry.get(args.computer); |
| 795 | activeComputerId = computer.id; |
| 796 | switched = true; |
| 797 | } else { |
| 798 | computer = registry.get(activeComputerId); |
| 799 | } |
| 800 | } catch (err) { |
| 801 | await retireBinding(args.computer || activeComputerId); |
| 802 | return { content: [{ type: "text", text: JSON.stringify(fail(null, err.code ?? "registry_error", err.message)) }], isError: true }; |
| 803 | } |
| 804 | |
| 805 | // Consent tools are the ledger itself — server-side, no backend dispatch. |
| 806 | // They still resolve the target computer the same way every other tool does. |
| 807 | if (name === "consent_status") { |
| 808 | return { content: [{ type: "text", text: JSON.stringify(receipt(computer, { ok: true, tool: name, switched, ...consent.status(computer.id) })) }] }; |
| 809 | } |
| 810 | if (name === "consent_allow" || name === "consent_deny" || name === "consent_revoke") { |
| 811 | try { |
| 812 | const scope = args.scope === "foreground" ? "foreground" : "app"; |
| 813 | const verb = { consent_allow: "allow", consent_deny: "deny" }[name] ?? null; |
| 814 | if (scope === "foreground") { |
| 815 | const r = verb ? consent.recordForeground(computer.id, verb, { remember: args.remember === true }) |
| 816 | : consent.revokeForeground(computer.id); |
| 817 | return { content: [{ type: "text", text: JSON.stringify(receipt(computer, { ok: true, tool: name, switched, scope, ...r, note: verb ? `Shared-desktop (foreground) control ${verb === "allow" ? "allowed" : "denied"} for ${r.persisted ? "this computer until revoked" : "this session"}.` : "Foreground decision removed — the next activate:true asks again." })) }] }; |
| 818 | } |
| 819 | const ref = refFromConsentArgs(args); |
| 820 | const keys = consent.appKeys(ref); |
| 821 | if (!keys.length) throw new ServerError("bad_args", `consent ${name.slice(8)} needs an app identity (app, name, bundle_id or pid) — or scope:"foreground"`); |
| 822 | // Fold in the resolved running-app identity so the decision holds under |
| 823 | // every spelling — and a deny cannot be sidestepped by asking for the |
| 824 | // same app a different way. |
| 825 | const resolved = await resolveAppIdentity(computer, ref); |
| 826 | const allKeys = resolved ? [...new Set([...keys, ...consent.appKeys(resolved)])] : keys; |
| 827 | if (verb) { |
| 828 | const r = consent.record(computer.id, allKeys, verb, { remember: args.remember === true, name: resolved?.name ?? ref.name ?? null }); |
| 829 | return { content: [{ type: "text", text: JSON.stringify(receipt(computer, { ok: true, tool: name, switched, scope, decision: verb, app: resolved ?? ref, keys: allKeys, persisted: r.persisted, note: `${resolved?.name ?? ref.name ?? ref.bundle_id ?? `pid ${ref.pid}`} ${verb === "allow" ? "allowed" : "denied"} ${r.persisted ? "until revoked" : "for this session"}.` })) }] }; |
| 830 | } |
| 831 | const r = consent.revoke(computer.id, allKeys); |
| 832 | return { content: [{ type: "text", text: JSON.stringify(receipt(computer, { ok: true, tool: name, switched, scope, app: resolved ?? ref, keys: allKeys, ...r, note: "Decisions removed — the next call targeting this app asks again." })) }] }; |
| 833 | } catch (err) { |
| 834 | return { content: [{ type: "text", text: JSON.stringify(fail(computer, err.code ?? "consent_error", err.message ?? String(err), { tool: name, switched })) }], isError: true }; |
| 835 | } |
| 836 | } |
| 837 | |
| 838 | let binding; |
| 839 | let dispatched = false; |
| 840 | try { |
| 841 | binding = await bindComputer(computer); |
| 842 | if (binding.needsObservation && !ROUTE_INSPECTION_TOOLS.has(name)) { |
| 843 | throw new ServerError("computer_observation_required", "Computer route changed — call screenshot or get_app_state on the registered target before acting"); |
| 844 | } |
| 845 | // Per-app consent: the first call that targets an application on the local |
| 846 | // computer must carry a recorded user decision. open_application returns |
| 847 | // the grant so its resolved identity can be aliased below. |
| 848 | const gateResult = await consentCheck(computer, name, args); |
| 849 | if (name === "run_actions") { |
| 850 | const steps = args.steps; |
| 851 | if (!Array.isArray(steps) || steps.length < 1 || steps.length > 8) throw new ServerError("bad_args", "run_actions needs 1..8 steps"); |
| 852 | const results = []; |
| 853 | for (const [i, step] of steps.entries()) { |
| 854 | if (!step || typeof step.tool !== "string") throw new ServerError("bad_args", `step ${i} needs a tool name`); |
| 855 | if (step.tool === "run_actions") throw new ServerError("bad_args", "run_actions cannot nest"); |
| 856 | if (!TOOL_NAMES.has(step.tool)) throw new ServerError("unknown_tool", `unknown tool "${step.tool}"`); |
| 857 | const result = await callTool({ name: step.tool, arguments: { ...(step.arguments ?? {}), computer: computer.id } }); |
| 858 | const body = JSON.parse(result.content[0].text); |
| 859 | results.push({ tool: step.tool, ok: body.ok !== false, receipt: body }); |
| 860 | if (body.ok === false || result.isError) { |
| 861 | return { content: [{ type: "text", text: JSON.stringify(fail(computer, body.error?.code ?? "step_failed", body.error?.message ?? "step failed", { tool: "run_actions", switched, stopped_at: i, steps: results })) }], isError: true }; |
| 862 | } |
| 863 | } |
| 864 | return { content: [{ type: "text", text: JSON.stringify(receipt(computer, { ok: true, tool: "run_actions", switched, steps: results })) }] }; |
| 865 | } |
| 866 | if (name === "find_elements") { |
| 867 | const st = args.state_id ? appStates.get(args.state_id) : null; |
| 868 | if (args.state_id && !st) throw new ServerError("unknown_state", `state_id "${args.state_id}" is unknown or expired — call get_app_state again`); |
| 869 | if (st) { |
| 870 | if (st.computerId && st.computerId !== computer.id) { |
| 871 | throw new ServerError("state_wrong_computer", `state_id "${args.state_id}" belongs to computer "${st.computerId}", not "${computer.id}"`); |
| 872 | } |
| 873 | const filtered = filterElements(st.elements, { |
| 874 | detail: "summary", query: args.query, role: args.role, |
| 875 | limit: args.limit ?? 20, offset: args.offset ?? 0, compact: true, |
| 876 | }); |
| 877 | return { content: [{ type: "text", text: JSON.stringify(receipt(computer, { ok: true, tool: "find_elements", switched, state_id: args.state_id, ...filtered, note: "Indices address the cached tree from this state_id." })) }] }; |
| 878 | } |
| 879 | return callTool({ name: "get_app_state", arguments: { ...args, detail: "compact", limit: args.limit ?? 20, computer: computer.id } }); |
| 880 | } |
| 881 | if (name === "wait_for") { |
| 882 | return waitFor(computer, args, switched); |
| 883 | } |
| 884 | // type/key with an element target run the documented focus-then-act idiom |
| 885 | // in one call: the element is revalidated and accessibility-focused first, |
| 886 | // through the same routed path a separate focus call would take. The |
| 887 | // target stays on the args — the backend also uses it to route the input |
| 888 | // into the element's own window, which is how hosted panels (native file |
| 889 | // pickers) receive keys whose handlers live outside the app's process. |
| 890 | if ((name === "type" || name === "key") && args.target != null) { |
| 891 | if (args.target.type !== "element") { |
| 892 | throw new ServerError("bad_target", `${name} accepts element targets only — use left_click for a coordinate, then ${name}`); |
| 893 | } |
| 894 | const focused = await callTool({ name: "focus", arguments: { target: args.target, computer: computer.id } }); |
| 895 | const focusBody = JSON.parse(focused.content[0].text); |
| 896 | // For a chord the element's window is what matters — key equivalents |
| 897 | // dispatch at window level, so a focus refusal must not block delivery. |
| 898 | // Text is different: characters go to the first responder, so a field |
| 899 | // that could not be focused cannot receive the string either. |
| 900 | if (name === "type" && (focused.isError || focusBody.ok === false)) { |
| 901 | return { content: [{ type: "text", text: JSON.stringify(fail(computer, focusBody.error?.code ?? "focus_failed", focusBody.error?.message ?? "element could not be focused", { tool: name, stage: "focus" })) }], isError: true }; |
| 902 | } |
| 903 | args = { ...args }; |
| 904 | } |
| 905 | // Out-of-process runners (the desktop app for the local computer, the |
| 906 | // remote agent for ssh computers) get the request over the wire. |
| 907 | const backendMethod = BACKEND_METHOD[name]; |
| 908 | // Scripting is honored on the local computer only. Remote agents refuse |
| 909 | // it too (their handler gates computerId), so a remote channel can never |
| 910 | // be steered into a shell — fail here first to save the hop. |
| 911 | if (name === "app_script" && computer.transport !== "local") { |
| 912 | throw new ServerError("unsupported_on_transport", `app_script runs on the local computer only — the ${computer.transport} transport stays a computer-use channel, never a shell`); |
| 913 | } |
| 914 | let data; |
| 915 | const ex = computer.transport === "local" || computer.transport === "ssh" || computer.transport === "docker" ? await executorFor(computer, binding) : null; |
| 916 | if (ex?.kind === "app") binding.usedApp = true; |
| 917 | // Zoom needs the bound parent raster up front (server-side check too, not |
| 918 | // only the backend) so it can bind the child raster after success. |
| 919 | let zoomParent = null; |
| 920 | if (name === "zoom") { |
| 921 | zoomParent = lastRasters.get(computer.id); |
| 922 | if (!zoomParent) throw new ServerError("no_raster", "no screenshot bound on this computer yet — call screenshot first so zoom has a source raster"); |
| 923 | if (!Array.isArray(args.region) || args.region.length !== 4) throw new ServerError("bad_args", "zoom needs region [x, y, w, h] in last-raster pixels"); |
| 924 | } |
| 925 | const sink = { reacquired: false }; |
| 926 | |
| 927 | if (typeof ex?.remote === "function" && REMOTE_TOOLS.has(backendMethod)) { |
| 928 | // ssh rides the persistent agent channel when the remote supports |
| 929 | // --serve; a channel that never produced a reply means an old agent, |
| 930 | // so fall back to one-shot for that binding rather than failing. |
| 931 | const remoteCall = async (request, opts = {}) => { |
| 932 | if (typeof ex.persistent === "function" && binding.sshServe !== false) { |
| 933 | try { |
| 934 | return await ex.persistent(request, opts); |
| 935 | } catch (err) { |
| 936 | const ch = binding.sshChannel; |
| 937 | if (err?.code === "remote_session_lost" && ch && !ch.everReplied) { |
| 938 | binding.sshServe = false; |
| 939 | ex.closeChannel?.(); |
| 940 | return ex.remote(request, opts); |
| 941 | } |
| 942 | throw err; |
| 943 | } |
| 944 | } |
| 945 | return ex.remote(request, opts); |
| 946 | }; |
| 947 | const resolve = async (req) => { |
| 948 | const rep = await remoteCall({ tool: "resolve_element", args: req }, { timeoutMs: 30_000 }); |
| 949 | if (!rep?.ok) return { found: false, element: null, reason: rep?.error?.code ?? "remote_error" }; |
| 950 | return rep.data; |
| 951 | }; |
| 952 | const wireArgs = await prepareArgs(computer, name, args, resolve, sink); |
| 953 | throwIfAborted(); |
| 954 | await assertCurrentRoute(computer, binding); |
| 955 | // Re-check the kill switch: a stop that arrived while the executor was |
| 956 | // being resolved still blocks this dispatch. |
| 957 | if (controlStopped && !READ_ONLY_TOOLS.has(name)) throw new ServerError("control_stopped", "stop_computer_control is active; no further actions are permitted this session"); |
| 958 | inFlight++; |
| 959 | try { |
| 960 | dispatched = true; |
| 961 | const timeoutMs = backendMethod.startsWith("recording") || backendMethod === "get_app_state" ? 60_000 : 30_000; |
| 962 | const invoke = async (tool, a) => { |
| 963 | const r = await remoteCall({ tool, args: a }, { timeoutMs }); |
| 964 | if (!r.ok) throw new ServerError(r.error?.code ?? "remote_error", r.error?.message ?? "remote agent failed"); |
| 965 | return r.data; |
| 966 | }; |
| 967 | data = name === "type" ? await invokeType(invoke, wireArgs) : await invoke(backendMethod, wireArgs); |
| 968 | } finally { |
| 969 | inFlight--; |
| 970 | } |
| 971 | await assertCurrentRoute(computer, binding, true); |
| 972 | if (Array.isArray(data)) data = { items: data }; |
| 973 | if ((backendMethod === "screenshot" || backendMethod === "zoom") && data?.file) { |
| 974 | if (ex.filesLocal) bindRaster(computer, data); |
| 975 | else { |
| 976 | // Raster lives on the remote machine; bind geometry for coordinate mapping. |
| 977 | bindRaster(computer, { ...data, file: null }); |
| 978 | data.note = "file lives on the remote computer; pull it with scp if you need the bytes locally"; |
| 979 | } |
| 980 | } |
| 981 | if (backendMethod === "zoom") bindZoomRaster(computer, zoomParent, args.region, ex.filesLocal ? data?.file ?? data?.path : null); |
| 982 | if (name === "get_app_state") { |
| 983 | data = observeState(computer, wireArgs.app_ref, data, args); |
| 984 | } |
| 985 | if (backendMethod === "probe") Object.assign(data, { via: ex.kind, app: ex.app ?? null }); |
| 986 | if (backendMethod === "probe" && data?.app?.version && data.app.version !== APP_VERSION) { |
| 987 | // The helper owns the modules it loaded at start, so a plugin update |
| 988 | // without a helper restart serves the previous build's behavior. Say |
| 989 | // so instead of letting the agent debug a build that is not running. |
| 990 | data.app.bundled_version = APP_VERSION; |
| 991 | data.app.stale = true; |
| 992 | data.note = [data.note, `The running helper reports ${data.app.version} but this plugin is ${APP_VERSION} — restart the Codewhale Computer Use app to load the current build.`].filter(Boolean).join(" "); |
| 993 | } |
| 994 | } else { |
| 995 | const backend = await getBackend(computer, binding); |
| 996 | if (typeof backend[backendMethod] !== "function") { |
| 997 | throw new ServerError("unsupported_on_backend", `"${name}" is not implemented on the ${computer.platform ?? computer.transport} backend`); |
| 998 | } |
| 999 | const resolve = typeof backend.resolve_element === "function" ? (req) => backend.resolve_element(req) : null; |
| 1000 | const prepared = await prepareArgs(computer, name, args, resolve, sink); |
| 1001 | throwIfAborted(); |
| 1002 | await assertCurrentRoute(computer, binding); |
| 1003 | if (controlStopped && !READ_ONLY_TOOLS.has(name)) throw new ServerError("control_stopped", "stop_computer_control is active; no further actions are permitted this session"); |
| 1004 | inFlight++; |
| 1005 | try { |
| 1006 | dispatched = true; |
| 1007 | data = name === "type" |
| 1008 | ? await invokeType((tool, a) => backend[BACKEND_METHOD[tool] ?? tool](a), prepared) |
| 1009 | : await backend[backendMethod](prepared); |
| 1010 | } finally { |
| 1011 | inFlight--; |
| 1012 | } |
| 1013 | await assertCurrentRoute(computer, binding, true); |
| 1014 | if (Array.isArray(data)) data = { items: data }; // keep receipts objects |
| 1015 | if (name === "screenshot") bindRaster(computer, data); |
| 1016 | if (backendMethod === "zoom") bindZoomRaster(computer, zoomParent, args.region, data?.file ?? data?.path); |
| 1017 | if (name === "get_app_state") { |
| 1018 | data = observeState(computer, prepared.app_ref, data, args); |
| 1019 | } |
| 1020 | if (backendMethod === "probe" && computer.transport === "local") { |
| 1021 | // Direct mode: permissions belong to whatever hosts this server. Say so. |
| 1022 | Object.assign(data, { via: "direct", app: null, appHint: ex?.appReason ?? null }); |
| 1023 | } |
| 1024 | } |
| 1025 | |
| 1026 | // Binding a different app retires this computer's element cache: a bare |
| 1027 | // index must never silently address the previous app's observation — |
| 1028 | // under a concurrent user that mistake clicks the wrong window. |
| 1029 | if (name === "open_application" && data?.resolved) { |
| 1030 | boundApps.set(computer.id, data.resolved); |
| 1031 | // The decision that let this open through covers the resolved identity |
| 1032 | // under its other spellings too — a later bundle-id or name request for |
| 1033 | // the same app must not prompt again. |
| 1034 | if (gateResult?.grant) { |
| 1035 | consent.alias(computer.id, consent.appKeys(data.resolved), { persisted: gateResult.grant.persisted, name: data.resolved.name ?? null }); |
| 1036 | } |
| 1037 | const latestId = latestStateByComputer.get(computer.id); |
| 1038 | const latest = latestId ? appStates.get(latestId) : null; |
| 1039 | if (latest) { |
| 1040 | const a = latest.app_ref ?? {}; |
| 1041 | const b = data.resolved; |
| 1042 | const sameApp = a.pid != null && b.pid != null |
| 1043 | ? a.pid === b.pid |
| 1044 | : (a.bundle_id && b.bundle_id ? a.bundle_id === b.bundle_id : a.name === b.name); |
| 1045 | if (!sameApp) { |
| 1046 | latestStateByComputer.delete(computer.id); |
| 1047 | data.note = [data.note, "Element indices from earlier observations belonged to a different app — call get_app_state before targeting."].filter(Boolean).join(" "); |
| 1048 | } |
| 1049 | } |
| 1050 | } |
| 1051 | |
| 1052 | if (name === "get_app_state" && args.include_ocr) { |
| 1053 | data.ocr ??= { status: "unavailable", reason: "Text recognition is not available on this backend", blocks: [] }; |
| 1054 | if (data.ocr.raster) { |
| 1055 | const localFile = typeof ex?.remote !== "function" || ex.filesLocal; |
| 1056 | bindRaster(computer, localFile ? data.ocr.raster : { ...data.ocr.raster, file: null, path: null }); |
| 1057 | } |
| 1058 | data.ocr.note = "Recognized text may be imperfect. These coordinate targets belong to this captured image, not to accessibility elements; observe again after the UI changes. Prefer ocr_region or query over a second full-window OCR."; |
| 1059 | } |
| 1060 | |
| 1061 | // Inline the raster only when it fits the budget. One oversized JSON-RPC |
| 1062 | // message drops the whole stdio transport and every other tool with it, so |
| 1063 | // an over-budget capture degrades to its text receipt: the file is still on |
| 1064 | // disk and still bound, so zoom or a narrower capture returns a viewable |
| 1065 | // image. Never trade the session for one screenshot. |
| 1066 | let imageBlock = null; |
| 1067 | if ((name === "screenshot" || name === "zoom" || name === "browser_screenshot") && computer.transport === "local" && (data.file || data.path)) { |
| 1068 | const file = data.file || data.path; |
| 1069 | const size = fs.statSync(file).size; |
| 1070 | if (encodedSize(size) > INLINE_IMAGE_MAX_BYTES) { |
| 1071 | data.image_omitted = { |
| 1072 | reason: "raster_too_large", |
| 1073 | bytes: size, |
| 1074 | encoded_bytes: encodedSize(size), |
| 1075 | limit_bytes: INLINE_IMAGE_MAX_BYTES, |
| 1076 | note: "The capture is on disk at the returned path, but inlining it would exceed this host's single-message budget and drop the connection. Capture one display, a region, or an app window, or call zoom on this raster to get a viewable image.", |
| 1077 | }; |
| 1078 | } else { |
| 1079 | const bytes = fs.readFileSync(file); |
| 1080 | imageBlock = { type: "image", mimeType: bytes[0] === 0xff ? "image/jpeg" : "image/png", data: bytes.toString("base64") }; |
| 1081 | } |
| 1082 | } |
| 1083 | if (name === "request_access") { |
| 1084 | const grant = grantReport(); |
| 1085 | if (grant) data.grant = grant; |
| 1086 | } |
| 1087 | const content = [{ type: "text", text: JSON.stringify(receipt(computer, { ok: true, tool: name, switched, ...(sink.reacquired ? { target_reacquired: true } : {}), ...data })) }]; |
| 1088 | if (imageBlock) content.push(imageBlock); |
| 1089 | if ((name === "screenshot" && (data?.file || data?.path) && data?.pixels?.w > 0 && data?.pixels?.h > 0) || |
| 1090 | (name === "browser_screenshot" && !!data?.file) || |
| 1091 | (name === "get_app_state" && data?.found !== false && Array.isArray(data?.elements))) { |
| 1092 | binding.needsObservation = false; |
| 1093 | } |
| 1094 | return { content }; |
| 1095 | } catch (err) { |
| 1096 | // A failed open_application cleared the backend's input binding before it |
| 1097 | // attempted anything — the tracked bound app must not claim otherwise. |
| 1098 | if (name === "open_application") boundApps.delete(computer.id); |
| 1099 | let outcomeUnknown = !!err.requestDispatched; |
| 1100 | if (dispatched && !outcomeUnknown) { |
| 1101 | // A transport/backend can fail after delivering input. Reconcile its |
| 1102 | // captured route on failure too, without replacing the original error |
| 1103 | // with a route/cleanup error or claiming an unchanged-route failure sent input. |
| 1104 | try { await assertCurrentRoute(computer, binding, true); } |
| 1105 | catch { outcomeUnknown = true; } |
| 1106 | } |
| 1107 | // The grant is a launch-time server fact: report it on refusal receipts too, |
| 1108 | // so a narrowed session knows its bounds even when the probe itself failed |
| 1109 | // (for example a headless Linux host with no DISPLAY to inspect). |
| 1110 | const grant = name === "request_access" ? grantReport() : null; |
| 1111 | return { content: [{ type: "text", text: JSON.stringify(fail(computer, err.code ?? "tool_error", err.message ?? String(err), { |
| 1112 | tool: name, switched, |
| 1113 | ...(err.extra ?? {}), |
| 1114 | ...(grant ? { grant } : {}), |
| 1115 | ...(outcomeUnknown ? { request_dispatched: true, outcome_unknown: true, |
| 1116 | note: "Dispatch to the previous route was attempted; its effect is unconfirmed. Observe the current target; do not automatically retry the action." } : {}), |
| 1117 | })) }], isError: true }; |
| 1118 | } |
| 1119 | } |
| 1120 | |
| 1121 | /** |
| 1122 | * Convert public tool args into backend args, identically for every route. |
| 1123 | * Element targets carry their revalidated AX path and fresh center; coordinate |
| 1124 | * targets are mapped from raster pixels to screen points here, once. |
| 1125 | * |
| 1126 | * The desktop app and the ssh agent are backends like any other: sending them |
| 1127 | * raw raster pixels would put every click at the wrong place on a scaled |
| 1128 | * display and skip the raster's own fail-closed checks (no_raster, |
| 1129 | * target_outside_raster), which is what happened while this ran per-route. |
| 1130 | */ |
| 1131 | async function prepareArgs(computer, name, args, resolve, sink) { |
| 1132 | const out = { ...args }; |
| 1133 | delete out.computer; |
| 1134 | delete out.ephemeral; // server-internal: never reaches a backend |
| 1135 | // type/key join the semantic set: their element target addresses a window |
| 1136 | // for input routing (hosted panels), not a point for pointer delivery. |
| 1137 | const semantic = new Set(["set_value", "select_text", "perform_action", "focus", "get_value", "type", "key"]); |
| 1138 | for (const key of ["target", "from_target", "to"]) { |
| 1139 | const given = out[key]; |
| 1140 | if (given == null) continue; |
| 1141 | // Hosts that don't enforce inputSchema can hand us any shape. Refuse |
| 1142 | // before it reaches a backend as an opaque native error or a TypeError. |
| 1143 | if (typeof given !== "object" || Array.isArray(given) || (given.type !== "coordinate" && given.type !== "element")) { |
| 1144 | throw new ServerError("bad_target", `${key} must be {type:'coordinate',x,y[,space]} or {type:'element',index[,state_id]} — got ${JSON.stringify(given)?.slice(0, 120)}`); |
| 1145 | } |
| 1146 | if (key === "target" && ELEMENT_ONLY_TARGET.has(name) && given.type !== "element") { |
| 1147 | throw new ServerError("bad_target", `${name} accepts element targets only — observe the control with get_app_state and pass {type:'element',index}`); |
| 1148 | } |
| 1149 | const kind = key === "target" && semantic.has(name) ? "semantic" : "pointer"; |
| 1150 | out[key] = { ...given, ...(await normalizeTarget(computer, given, kind, resolve, sink)) }; |
| 1151 | } |
| 1152 | if (name === "get_app_state" || name === "find_elements") { |
| 1153 | if (out.detail != null && !["summary", "compact", "full"].includes(out.detail)) throw new ServerError("bad_args", "detail must be summary, compact or full"); |
| 1154 | if (name === "get_app_state") { |
| 1155 | out.compact = out.detail === "compact"; |
| 1156 | out.detail = out.detail === "full" ? "full" : "summary"; |
| 1157 | } |
| 1158 | if (out.include_ocr != null && typeof out.include_ocr !== "boolean") throw new ServerError("bad_args", "include_ocr must be true or false"); |
| 1159 | if (out.window_id != null && (!Number.isSafeInteger(out.window_id) || out.window_id < 0)) throw new ServerError("bad_args", "window_id must be a non-negative window index from list_windows"); |
| 1160 | if (out.limit != null && (!Number.isSafeInteger(out.limit) || out.limit < 1 || out.limit > 200)) throw new ServerError("bad_args", "limit must be an integer 1..200"); |
| 1161 | if (out.offset != null && (!Number.isSafeInteger(out.offset) || out.offset < 0)) throw new ServerError("bad_args", "offset must be a non-negative integer"); |
| 1162 | if (out.query != null && typeof out.query !== "string") throw new ServerError("bad_args", "query must be a string"); |
| 1163 | if (out.role != null && typeof out.role !== "string") throw new ServerError("bad_args", "role must be a string"); |
| 1164 | if (out.ocr_region != null && (!Array.isArray(out.ocr_region) || out.ocr_region.length !== 4)) throw new ServerError("bad_args", "ocr_region must be [x, y, w, h] in screen points"); |
| 1165 | } |
| 1166 | if (name === "app_script") { |
| 1167 | if (typeof out.script !== "string" || !out.script.trim()) throw new ServerError("bad_args", "app_script needs a non-empty script string"); |
| 1168 | if (out.language != null && !["applescript", "javascript"].includes(out.language)) throw new ServerError("bad_args", 'app_script language must be "applescript" or "javascript"'); |
| 1169 | if (out.timeout != null && (!Number.isFinite(out.timeout) || out.timeout <= 0 || out.timeout > 120)) throw new ServerError("bad_args", "app_script timeout must be 1..120 seconds"); |
| 1170 | } |
| 1171 | return out; |
| 1172 | } |
| 1173 | |
| 1174 | // ---------- JSON-RPC loop ---------- |
| 1175 | function respond(id, result) { |
| 1176 | process.stdout.write(JSON.stringify({ jsonrpc: "2.0", id, result }) + "\n"); |
| 1177 | } |
| 1178 | function respondError(id, code, message) { |
| 1179 | process.stdout.write(JSON.stringify({ jsonrpc: "2.0", id, error: { code, message } }) + "\n"); |
| 1180 | } |
| 1181 | |
| 1182 | /** JSON-RPC invalid-params error that survives the dispatch catch below. */ |
| 1183 | function paramError(message) { |
| 1184 | return Object.assign(new Error(message), { rpcCode: -32602 }); |
| 1185 | } |
| 1186 | |
| 1187 | // ---------- bundled skill pack ---------- |
| 1188 | // The operating guide travels with the server and is served as MCP resources |
| 1189 | // (skill://codewhale-cu/…) so any host can read the loop, the failure codes and |
| 1190 | // the safety rules without paying for them in every receipt. The pack is loaded |
| 1191 | // once at startup; a trimmed install without skills/ simply serves none. |
| 1192 | const SKILL_NAME = "computer-use"; |
| 1193 | const SKILL_ROOT_URI = `skill://codewhale-cu/SKILL.md`; |
| 1194 | |
| 1195 | function parseFrontmatter(text) { |
| 1196 | text = text.replace(/\r\n/g, "\n"); |
| 1197 | if (!text.startsWith("---\n")) return null; |
| 1198 | const end = text.indexOf("\n---", 4); |
| 1199 | if (end === -1) return null; |
| 1200 | const out = {}; |
| 1201 | let key = null; |
| 1202 | for (const line of text.slice(4, end).split("\n")) { |
| 1203 | const m = /^([A-Za-z0-9_-]+):\s*(.*)$/.exec(line); |
| 1204 | if (m) { key = m[1]; out[key] = [">-", ">"].includes(m[2]) ? "" : m[2].replace(/^["']|["']$/g, ""); continue; } |
| 1205 | if (key && /^\s+\S/.test(line)) out[key] = `${out[key] ? `${out[key]} ` : ""}${line.trim()}`; |
| 1206 | } |
| 1207 | return out; |
| 1208 | } |
| 1209 | |
| 1210 | const skillPack = (() => { |
| 1211 | const root = new URL("../skills/computer-use/", import.meta.url); |
| 1212 | const files = [ |
| 1213 | ["SKILL.md", "text/markdown"], |
| 1214 | ["references/quick-reference.md", "text/markdown"], |
| 1215 | ["references/refusal-codes.md", "text/markdown"], |
| 1216 | ]; |
| 1217 | const pack = []; |
| 1218 | for (const [rel, mime] of files) { |
| 1219 | try { |
| 1220 | const bytes = fs.readFileSync(new URL(rel, root)); |
| 1221 | const text = bytes.toString("utf8"); |
| 1222 | pack.push({ |
| 1223 | rel, uri: `skill://codewhale-cu/${rel}`, mime, size: bytes.length, text, |
| 1224 | frontmatter: rel === "SKILL.md" ? parseFrontmatter(text) : null, |
| 1225 | sha256: crypto.createHash("sha256").update(bytes).digest("hex"), |
| 1226 | }); |
| 1227 | } catch { /* no pack on disk — serve nothing */ } |
| 1228 | } |
| 1229 | return pack; |
| 1230 | })(); |
| 1231 | const SKILL_DESCRIPTION = skillPack.find((f) => f.rel === "SKILL.md")?.frontmatter?.description ?? "Computer-use operating guide"; |
| 1232 | |
| 1233 | /** |
| 1234 | * callTool plus optional trajectory recording. Recording wraps every call the |
| 1235 | * session makes (refusals included — they are part of what happened); the |
| 1236 | * recorder's own tools and replayed calls are never re-recorded. |
| 1237 | */ |
| 1238 | async function callToolRecorded(params) { |
| 1239 | const result = await callTool(params); |
| 1240 | if (recorder.active && !replaying && !isTrajectoryTool(params?.name)) { |
| 1241 | let body = null; |
| 1242 | try { body = JSON.parse(result?.content?.[0]?.text ?? "null"); } catch { /* non-JSON receipts record without an outcome */ } |
| 1243 | recorder.append({ tool: params.name, args: params.arguments ?? {}, ok: body?.ok !== false, code: body?.error?.code ?? null }); |
| 1244 | } |
| 1245 | return result; |
| 1246 | } |
| 1247 | |
| 1248 | const HANDLERS = { |
| 1249 | initialize(params) { |
| 1250 | return { |
| 1251 | protocolVersion: params?.protocolVersion ?? "2025-06-18", |
| 1252 | capabilities: { |
| 1253 | tools: { listChanged: false }, |
| 1254 | resources: { listChanged: false, subscribe: false }, |
| 1255 | experimental: { "io.modelcontextprotocol/skills": {} }, |
| 1256 | }, |
| 1257 | serverInfo: { name: SERVER_NAME, version: APP_VERSION, platforms: ["darwin", "win32", "linux", "harmonyos"], transports: ["local", "ssh", "hdc"] }, |
| 1258 | }; |
| 1259 | }, |
| 1260 | "tools/list"() { |
| 1261 | // The advertised surface is what every session pays for; merged-away wire |
| 1262 | // names stay callable as aliases but are never listed. A capability grant |
| 1263 | // narrows the listing further, never widens it. |
| 1264 | const advertised = TOOLS.filter((t) => t.hidden !== true); |
| 1265 | if (!GRANT) return { tools: advertised }; |
| 1266 | return { tools: advertised.filter((t) => t.name === "stop_computer_control" || GRANT.has(t.name) || (MERGED_EXPANSION[t.name] ?? []).some((wire) => GRANT.has(wire))) }; |
| 1267 | }, |
| 1268 | "resources/list"() { |
| 1269 | return { resources: skillPack.map(({ uri, rel, mime, size }) => ({ uri, name: rel, mimeType: mime, size })) }; |
| 1270 | }, |
| 1271 | "resources/read"(params) { |
| 1272 | const file = skillPack.find((f) => f.uri === params?.uri); |
| 1273 | if (!file) throw paramError(`resource "${params?.uri ?? ""}" is not part of the bundled skill pack — resources/list names the readable URIs`); |
| 1274 | return { contents: [{ uri: file.uri, mimeType: file.mime, text: file.text }] }; |
| 1275 | }, |
| 1276 | "skills/list"() { |
| 1277 | return { |
| 1278 | skills: [{ |
| 1279 | uri: SKILL_ROOT_URI, name: SKILL_NAME, description: SKILL_DESCRIPTION, |
| 1280 | files: skillPack.map(({ uri, sha256, size }) => ({ uri, sha256, bytes: size })), |
| 1281 | }], |
| 1282 | }; |
| 1283 | }, |
| 1284 | "skills/get"(params) { |
| 1285 | const entry = skillPack.find((f) => f.uri === (params?.uri ?? SKILL_ROOT_URI)); |
| 1286 | if (!entry) throw paramError(`skill "${params?.uri ?? ""}" is unknown — skills/list names the catalog`); |
| 1287 | return { |
| 1288 | skill: { uri: entry.uri, name: SKILL_NAME, description: SKILL_DESCRIPTION, frontmatter: entry.frontmatter, content: entry.text }, |
| 1289 | manifest: skillPack.map(({ uri, sha256, size }) => ({ uri, sha256, bytes: size })), |
| 1290 | }; |
| 1291 | }, |
| 1292 | async "tools/call"(params) { |
| 1293 | if (params?.name === "stop_computer_control") return callTool(params); |
| 1294 | const previous = dispatch; |
| 1295 | let release; |
| 1296 | dispatch = new Promise((resolve) => { release = resolve; }); |
| 1297 | try { |
| 1298 | await previous; |
| 1299 | throwIfAborted(); |
| 1300 | return await callToolRecorded(params ?? {}); |
| 1301 | } catch (err) { |
| 1302 | if (err?.code !== "cancelled") throw err; |
| 1303 | return { content: [{ type: "text", text: JSON.stringify(fail(null, controlStopped ? "control_stopped" : "cancelled", err.message)) }], isError: true }; |
| 1304 | } finally { release(); } |
| 1305 | }, |
| 1306 | "notifications/cancelled"(params) { |
| 1307 | const request = requests.get(params?.requestId); |
| 1308 | if (request) { |
| 1309 | cancelled.add(params.requestId); |
| 1310 | request.controller.abort(); |
| 1311 | } |
| 1312 | return {}; |
| 1313 | }, |
| 1314 | ping() { |
| 1315 | return {}; |
| 1316 | }, |
| 1317 | }; |
| 1318 | |
| 1319 | let buffer = ""; |
| 1320 | process.stdin.setEncoding("utf8"); |
| 1321 | process.stdin.on("data", (chunk) => { |
| 1322 | buffer += chunk; |
| 1323 | let idx; |
| 1324 | while ((idx = buffer.indexOf("\n")) !== -1) { |
| 1325 | const line = buffer.slice(0, idx).trim(); |
| 1326 | buffer = buffer.slice(idx + 1); |
| 1327 | if (!line) continue; |
| 1328 | handleLine(line); |
| 1329 | } |
| 1330 | }); |
| 1331 | async function releaseControl({ releaseOnly = false } = {}) { |
| 1332 | let timer; |
| 1333 | try { |
| 1334 | await Promise.race([ |
| 1335 | (async () => { |
| 1336 | await dispatch; |
| 1337 | await withSignal(null, () => Promise.all([ |
| 1338 | closeAppSession({ releaseOnly }), |
| 1339 | ...[...backendCache.values()].map(async ({ backend }) => { |
| 1340 | await backend?.releaseInput?.(); |
| 1341 | if (!releaseOnly) await backend?.closeSession?.(); |
| 1342 | }), |
| 1343 | ])); |
| 1344 | })(), |
| 1345 | new Promise((_, reject) => { timer = setTimeout(() => reject(new Error("Computer input cleanup did not finish within 3 seconds")), 3_000); }), |
| 1346 | ]); |
| 1347 | } finally { clearTimeout(timer); } |
| 1348 | } |
| 1349 | |
| 1350 | let shuttingDown = false; |
| 1351 | async function shutdown() { |
| 1352 | if (shuttingDown) return; |
| 1353 | shuttingDown = true; |
| 1354 | for (const request of requests.values()) request.controller.abort(); |
| 1355 | try { await releaseControl(); } |
| 1356 | catch (err) { process.stderr.write(`Computer input cleanup failed: ${err?.message ?? err}\n`); } |
| 1357 | // Destroy the disposable computers this session spawned. Entries belonging |
| 1358 | // to other (possibly still-running) sessions are left alone — a container |
| 1359 | // belongs to the process that created it. |
| 1360 | try { |
| 1361 | await withSignal(null, async () => { |
| 1362 | await destroySessionSpawns(); |
| 1363 | const reg = registry.list(); |
| 1364 | for (const c of Object.values(reg.computers)) { |
| 1365 | if (c.transport === "docker" && c.owned === true && c.spawnedBy === SESSION_ID) { |
| 1366 | await destroyDockerComputer(c).catch(() => {}); |
| 1367 | try { registry.remove(c.id); } catch {} |
| 1368 | await retireBinding(c.id).catch(() => {}); |
| 1369 | } |
| 1370 | } |
| 1371 | }); |
| 1372 | } catch (err) { process.stderr.write(`Spawned computer cleanup failed: ${err?.message ?? err}\n`); } |
| 1373 | process.exit(0); |
| 1374 | } |
| 1375 | process.stdin.on("end", shutdown); |
| 1376 | for (const signal of ["SIGTERM", "SIGINT", "SIGHUP"]) process.on(signal, shutdown); |
| 1377 | |
| 1378 | async function handleLine(line) { |
| 1379 | if (shuttingDown) return; |
| 1380 | const msg = tryJson(line, null); |
| 1381 | if (!msg || typeof msg !== "object") return; |
| 1382 | const { id, method, params } = msg; |
| 1383 | if (!method) return; // response to a server request — we never issue any |
| 1384 | const handler = HANDLERS[method]; |
| 1385 | if (!handler) { |
| 1386 | if (id != null) respondError(id, -32601, `method not found: ${method}`); |
| 1387 | return; |
| 1388 | } |
| 1389 | // Cancelled before dispatch: per MCP, respond nothing. |
| 1390 | if (id != null && cancelled.has(id)) { cancelled.delete(id); return; } |
| 1391 | const controller = new AbortController(); |
| 1392 | if (id != null) requests.set(id, { controller, name: method === "tools/call" ? params?.name : null }); |
| 1393 | try { |
| 1394 | const result = await withSignal(controller.signal, () => handler(params)); |
| 1395 | // Cancelled mid-flight: drop the completed response. |
| 1396 | if (id != null) { |
| 1397 | if (cancelled.has(id)) { cancelled.delete(id); return; } |
| 1398 | respond(id, result); |
| 1399 | } |
| 1400 | } catch (err) { |
| 1401 | if (id != null && !cancelled.delete(id)) respondError(id, Number.isInteger(err?.rpcCode) ? err.rpcCode : -32603, err?.message ?? String(err)); |
| 1402 | } finally { |
| 1403 | if (id != null) requests.delete(id); |
| 1404 | } |
| 1405 | } |
| 1406 | |
| 1407 | // Notifications we must tolerate |
| 1408 | ["notifications/initialized", "initialized"].forEach((m) => { if (!HANDLERS[m]) HANDLERS[m] = () => ({}); }); |
| 1409 |